diff --git a/catalog/conus404-daily/collection.json b/catalog/conus404-daily/collection.json index 7391d61f99edc8024d0d503ce1b1d1950fe664ba..14da3f747299fe90222aaa16ee208eec658940b4 100644 --- a/catalog/conus404-daily/collection.json +++ b/catalog/conus404-daily/collection.json @@ -1804,7 +1804,7 @@ "license": "CC0-1.0", "assets": { "zarr-s3": { - "href": "s3://hytest/conus404/conus404_daily.zarr/", + "href": "s3://nhgf-development/conus404/conus404_daily.zarr/", "type": "application/vnd+zarr", "description": "S3 access to collection zarr group", "xarray:open_kwargs": { @@ -1820,6 +1820,27 @@ "zarr", "s3" ] + }, + "zarr-s3-osn": { + "href": "s3://hytest/conus404/conus404_daily.zarr/", + "type": "application/vnd+zarr", + "description": "Open Storage Network Pod S3 API access to collection zarr group", + "xarray:open_kwargs": { + "chunks": {}, + "engine": "zarr", + "consolidated": true + }, + "xarray:storage_options": { + "anon": true, + "client_kwargs": { + "endpoint_url": "https://usgs.osn.mghpcc.org/" + } + }, + "roles": [ + "data", + "zarr", + "s3" + ] } } } \ No newline at end of file diff --git a/workflows/create_collection_zarr_exploratory_workflow_conus404-daily.ipynb b/workflows/create_collection_zarr_exploratory_workflow_conus404-daily.ipynb index c092c3969bffa7864c0caa3222ad1e51101e2ca6..7f10149d7d3b1cb4ddd3a9e8165cc3d3f77f270f 100644 --- a/workflows/create_collection_zarr_exploratory_workflow_conus404-daily.ipynb +++ b/workflows/create_collection_zarr_exploratory_workflow_conus404-daily.ipynb @@ -74,6 +74,31 @@ "id": "482d204d-b5b6-40e5-ac42-55b459be1097", "metadata": {}, "outputs": [], + "source": [ + "# name for STAC collection\n", + "collection_id = 'conus404-daily'\n", + "# description of STAC collection\n", + "collection_description = 'CONUS404 40 years of daily values for subset of model output variables derived from hourly values on cloud storage'\n", + "# license for dataset\n", + "collection_license = 'CC0-1.0'" + ] + }, + { + "cell_type": "markdown", + "id": "116b5837-8e85-4ae7-964a-803533ded714", + "metadata": {}, + "source": [ + "## Asset Metadata Input" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd6fa323-132a-4794-8c80-576933f547a0", + "metadata": { + "tags": [] + }, + "outputs": [], "source": [ "# url to zarr store that you want to create a collection for\n", "zarr_url = 's3://hytest/conus404/conus404_daily.zarr/'\n", @@ -81,18 +106,33 @@ "# define keyword arguments needed for opening the dataset with xarray\n", "# ref: https://github.com/stac-extensions/xarray-assets\n", "xarray_opendataset_kwargs = {\"xarray:open_kwargs\":{\"chunks\":{},\"engine\":\"zarr\",\"consolidated\":True},\n", - " \"xarray:storage_options\":{\"requester_pays\":True}}\n", + " \"xarray:storage_options\": {\"anon\": True, \"client_kwargs\": {\"endpoint_url\":\"https://usgs.osn.mghpcc.org/\"}}}\n", "# description for zarr url asset attached to collection (zarr_url)\n", - "asset_description = \"S3 access to collection zarr group\"\n", + "asset_description = \"Open Storage Network Pod S3 API access to collection zarr group\"\n", "# roles to tag zarr url asset with\n", - "asset_roles = [\"data\",\"zarr\",\"s3\"]\n", + "asset_roles = [\"data\",\"zarr\",\"s3\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1441cd4-e94c-4902-af46-8f1af470eb6b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# url to zarr store that you want to create a collection for\n", + "zarr_url2 = 's3://nhgf-development/conus404/conus404_daily.zarr/'\n", "\n", - "# name for STAC collection\n", - "collection_id = 'conus404-daily'\n", - "# description of STAC collection\n", - "collection_description = 'CONUS404 40 years of daily values for subset of model output variables derived from hourly values on cloud storage'\n", - "# license for dataset\n", - "collection_license = 'CC0-1.0'" + "# define keyword arguments needed for opening the dataset with xarray\n", + "# ref: https://github.com/stac-extensions/xarray-assets\n", + "xarray_opendataset_kwargs2 = {\"xarray:open_kwargs\":{\"chunks\":{},\"engine\":\"zarr\",\"consolidated\":True},\n", + " \"xarray:storage_options\":{\"requester_pays\":True}}\n", + "# description for zarr url asset attached to collection (zarr_url)\n", + "asset_description2 = \"S3 access to collection zarr group\"\n", + "# roles to tag zarr url asset with\n", + "asset_roles2 = [\"data\",\"zarr\",\"s3\"]" ] }, { @@ -316,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "asset_id = \"zarr-s3\"\n", + "asset_id = \"zarr-s3-osn\"\n", "asset = pystac.Asset(href=zarr_url,\n", " description=asset_description,\n", " media_type=\"application/vnd+zarr\",\n", @@ -325,6 +365,24 @@ "collection.add_asset(asset_id, asset)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c298d07-f234-4a08-986d-87f4a39e9ae6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "asset_id2 = \"zarr-s3\"\n", + "asset2 = pystac.Asset(href=zarr_url2,\n", + " description=asset_description2,\n", + " media_type=\"application/vnd+zarr\",\n", + " roles=asset_roles2,\n", + " extra_fields = xarray_opendataset_kwargs2)\n", + "collection.add_asset(asset_id2, asset2)" + ] + }, { "cell_type": "markdown", "id": "f67cd5c9-db33-45c2-bc21-480cd67354f4",