diff --git a/workflows/archive/alaska_et_2020_ccsm4_historical_simulation_create_collection_from_zarr.ipynb b/workflows/archive/alaska_et_2020_ccsm4_historical_simulation_create_collection_from_zarr.ipynb
index f9024e8f89c72bd51e43b220860ce58ce1c37a83..6559ede06964e0971c697aa425922267b80b1119 100644
--- a/workflows/archive/alaska_et_2020_ccsm4_historical_simulation_create_collection_from_zarr.ipynb
+++ b/workflows/archive/alaska_et_2020_ccsm4_historical_simulation_create_collection_from_zarr.ipynb
@@ -496,14 +496,23 @@
},
"outputs": [],
"source": [
- "def get_step(dim_name):\n",
+ "def get_step(ds, dim_name, debug=False, step_ix=0):\n",
" dim_vals = ds[dim_name].values\n",
" diffs = [d2 - d1 for d1, d2 in zip(dim_vals, dim_vals[1:])]\n",
- " unique_steps = np.unique(diffs)\n",
+ " unique_steps = np.unique(diffs, return_counts=True)\n",
+ " step_list = unique_steps[0]\n",
+ " # optional - for invesitgating uneven steps\n",
+ " if debug:\n",
+ " print(f'step_list: {step_list}')\n",
+ " print(f'step_count: {unique_steps[1]}')\n",
+ " indices = [i for i, x in enumerate(diffs) if x == step_list[step_ix]]\n",
+ " print(f'index locations of step index {step_ix} in step_list: {indices}')\n",
" # set step - if all steps are the same length\n",
" # datacube spec specifies to use null for irregularly spaced steps\n",
- " if len(unique_steps)==1:\n",
- " step = unique_steps[0].astype(float).item()\n",
+ " if len(step_list)==1:\n",
+ " # make sure time deltas are in np timedelta format\n",
+ " step_list = [np.array([step], dtype=\"timedelta64[ns]\")[0] for step in step_list]\n",
+ " step = step_list[0].astype(float).item()\n",
" else:\n",
" step = None\n",
" return(step)"
@@ -568,6 +577,90 @@
"print(projjson)"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "205a88b5-5d1b-4a0f-9737-ab308d9f4553",
+ "metadata": {},
+ "source": [
+ "#### user review needed - looks at the steps pulled out and make sure they make sense"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1a7f6872-8c5c-4bf1-819c-b0b6f5103741",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "time_step = get_step(ds, dim_names_dict['T'])\n",
+ "print(f'time step: {time_step}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "50d891d3-b269-4ac0-9ae0-61f1c9bed3de",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# debugging for time steps: get all step values and locations\n",
+ "time_step = get_step(ds, dim_names_dict['T'], debug=True, step_ix=2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4c56aca3-79a8-4b0b-873f-c6539805f241",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# debugging for time steps, cont:\n",
+ "# please choose one of the index locations printed above\n",
+ "# this will print the time steps adjacent to it\n",
+ "ix = 182\n",
+ "ds.isel(time=slice(ix-1,ix+3)).time"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f6ab5e50-a6b8-42a4-a34c-733c5ee6d0c1",
+ "metadata": {},
+ "source": [
+ "**Irregular time step**: This dataset covers April 1 to Septemper 30 of each year (daily time step), with no data the rest of the time. Because this is an irregular time step, we must specify null for the time step."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "88e06715-e6ba-45f4-a94f-e3414ecb53aa",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "x_step = get_step(ds, dim_names_dict['X'])\n",
+ "print(f'x step: {x_step}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d5bfded3-5da7-4bbb-9521-4ac0dee51fca",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "y_step = get_step(ds, dim_names_dict['Y'])\n",
+ "print(f'y step: {y_step}')"
+ ]
+ },
{
"cell_type": "markdown",
"id": "00a5e041-081d-428d-ac2e-75d16de205e6",
@@ -576,6 +669,18 @@
"#### user input needed - you will need to copy all of the dimensions from above into the dict and fill in the appropriate attributes(type, axis, extent):"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3e86d82d-9e58-4bc8-a826-2191ed2b7477",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "print(dims)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -589,9 +694,9 @@
"# dimension name should come from the coordinates printed above\n",
"# we do not recommend including redundant dimensions (do not include x,y if you have lon,lat)\n",
"# note that the extent of each dimension should be pulled from the dataset\n",
- "dims_dict = {'time': pystac.extensions.datacube.Dimension({'type': 'temporal', 'description': get_long_name(ds, 'time'), 'extent': [temporal_extent_lower.strftime('%Y-%m-%dT%XZ'), temporal_extent_upper.strftime('%Y-%m-%dT%XZ')], 'step': pd.Timedelta(get_step('time')).isoformat()}),\n",
- " 'x': pystac.extensions.datacube.Dimension({'type': 'spatial', 'description': get_long_name(ds, 'x'), 'axis': 'x', 'extent': [xy_bounds[0], xy_bounds[2]], 'step': get_step('x'), 'reference_system': projjson}),\n",
- " 'y': pystac.extensions.datacube.Dimension({'type': 'spatial', 'axis': 'y', 'description': get_long_name(ds, 'y'), 'extent': [xy_bounds[1], xy_bounds[3]], 'step': get_step('y'), 'reference_system': projjson}),\n",
+ "dims_dict = {'time': pystac.extensions.datacube.Dimension({'type': 'temporal', 'description': get_long_name(ds, 'time'), 'extent': [temporal_extent_lower.strftime('%Y-%m-%dT%XZ'), temporal_extent_upper.strftime('%Y-%m-%dT%XZ')], 'step': pd.Timedelta(time_step).isoformat()}),\n",
+ " 'x': pystac.extensions.datacube.Dimension({'type': 'spatial', 'description': get_long_name(ds, 'x'), 'axis': 'x', 'extent': [xy_bounds[0], xy_bounds[2]], 'step': x_step, 'reference_system': projjson}),\n",
+ " 'y': pystac.extensions.datacube.Dimension({'type': 'spatial', 'axis': 'y', 'description': get_long_name(ds, 'y'), 'extent': [xy_bounds[1], xy_bounds[3]], 'step': y_step, 'reference_system': projjson}),\n",
" }"
]
},
diff --git a/workflows/archive/alaska_et_2020_era-interim_reanalysis_create_collection_from_zarr.ipynb b/workflows/archive/alaska_et_2020_era-interim_reanalysis_create_collection_from_zarr.ipynb
index 6c161d855e7475fa8543d889e5a101f1dfea1e52..fbfb5a5792e5608d1a36520088569002a018a348 100644
--- a/workflows/archive/alaska_et_2020_era-interim_reanalysis_create_collection_from_zarr.ipynb
+++ b/workflows/archive/alaska_et_2020_era-interim_reanalysis_create_collection_from_zarr.ipynb
@@ -496,14 +496,23 @@
},
"outputs": [],
"source": [
- "def get_step(dim_name):\n",
+ "def get_step(ds, dim_name, debug=False, step_ix=0):\n",
" dim_vals = ds[dim_name].values\n",
" diffs = [d2 - d1 for d1, d2 in zip(dim_vals, dim_vals[1:])]\n",
- " unique_steps = np.unique(diffs)\n",
+ " unique_steps = np.unique(diffs, return_counts=True)\n",
+ " step_list = unique_steps[0]\n",
+ " # optional - for invesitgating uneven steps\n",
+ " if debug:\n",
+ " print(f'step_list: {step_list}')\n",
+ " print(f'step_count: {unique_steps[1]}')\n",
+ " indices = [i for i, x in enumerate(diffs) if x == step_list[step_ix]]\n",
+ " print(f'index locations of step index {step_ix} in step_list: {indices}')\n",
" # set step - if all steps are the same length\n",
" # datacube spec specifies to use null for irregularly spaced steps\n",
- " if len(unique_steps)==1:\n",
- " step = unique_steps[0].astype(float).item()\n",
+ " if len(step_list)==1:\n",
+ " # make sure time deltas are in np timedelta format\n",
+ " step_list = [np.array([step], dtype=\"timedelta64[ns]\")[0] for step in step_list]\n",
+ " step = step_list[0].astype(float).item()\n",
" else:\n",
" step = None\n",
" return(step)"
@@ -568,6 +577,90 @@
"print(projjson)"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "899b4f61-692e-4cda-a058-ae161d1c3e86",
+ "metadata": {},
+ "source": [
+ "#### user review needed - looks at the steps pulled out and make sure they make sense"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "07b8af6b-d486-454c-8719-7ce514154dde",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "time_step = get_step(ds, dim_names_dict['T'])\n",
+ "print(f'time step: {time_step}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "936459cc-cfb9-48e7-b1c8-eb2a75449fdf",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# debugging for time steps: get all step values and locations\n",
+ "time_step = get_step(ds, dim_names_dict['T'], debug=True, step_ix=2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cbebf9a9-21ff-423d-8594-5624009b165c",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# debugging for time steps, cont:\n",
+ "# please choose one of the index locations printed above\n",
+ "# this will print the time steps adjacent to it\n",
+ "ix = 182\n",
+ "ds.isel(time=slice(ix-1,ix+3)).time"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5a0bda39-027e-46bd-8ba1-8dcd181be0b5",
+ "metadata": {},
+ "source": [
+ "**Irregular time step**: This dataset covers April 1 to Septemper 30 of each year (daily time step), with no data the rest of the time. Because this is an irregular time step, we must specify null for the time step."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "74a325e1-b797-403d-b9be-74fa541c59bd",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "x_step = get_step(ds, dim_names_dict['X'])\n",
+ "print(f'x step: {x_step}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aa17c4a4-d2b4-408d-afa9-e3e6737aca2e",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "y_step = get_step(ds, dim_names_dict['Y'])\n",
+ "print(f'y step: {y_step}')"
+ ]
+ },
{
"cell_type": "markdown",
"id": "00a5e041-081d-428d-ac2e-75d16de205e6",
@@ -576,6 +669,18 @@
"#### user input needed - you will need to copy all of the dimensions from above into the dict and fill in the appropriate attributes(type, axis, extent):"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c6102284-ef22-41eb-8e3d-65e5d94a9920",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "print(dims)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -589,9 +694,9 @@
"# dimension name should come from the coordinates printed above\n",
"# we do not recommend including redundant dimensions (do not include x,y if you have lon,lat)\n",
"# note that the extent of each dimension should be pulled from the dataset\n",
- "dims_dict = {'time': pystac.extensions.datacube.Dimension({'type': 'temporal', 'description': get_long_name(ds, 'time'), 'extent': [temporal_extent_lower.strftime('%Y-%m-%dT%XZ'), temporal_extent_upper.strftime('%Y-%m-%dT%XZ')], 'step': pd.Timedelta(get_step('time')).isoformat()}),\n",
- " 'x': pystac.extensions.datacube.Dimension({'type': 'spatial', 'description': get_long_name(ds, 'x'), 'axis': 'x', 'extent': [xy_bounds[0], xy_bounds[2]], 'step': get_step('x'), 'reference_system': projjson}),\n",
- " 'y': pystac.extensions.datacube.Dimension({'type': 'spatial', 'axis': 'y', 'description': get_long_name(ds, 'y'), 'extent': [xy_bounds[1], xy_bounds[3]], 'step': get_step('y'), 'reference_system': projjson}),\n",
+ "dims_dict = {'time': pystac.extensions.datacube.Dimension({'type': 'temporal', 'description': get_long_name(ds, 'time'), 'extent': [temporal_extent_lower.strftime('%Y-%m-%dT%XZ'), temporal_extent_upper.strftime('%Y-%m-%dT%XZ')], 'step': pd.Timedelta(time_step).isoformat()}),\n",
+ " 'x': pystac.extensions.datacube.Dimension({'type': 'spatial', 'description': get_long_name(ds, 'x'), 'axis': 'x', 'extent': [xy_bounds[0], xy_bounds[2]], 'step': x_step, 'reference_system': projjson}),\n",
+ " 'y': pystac.extensions.datacube.Dimension({'type': 'spatial', 'axis': 'y', 'description': get_long_name(ds, 'y'), 'extent': [xy_bounds[1], xy_bounds[3]], 'step': y_step, 'reference_system': projjson}),\n",
" }"
]
},
diff --git a/workflows/archive/alaska_et_2020_gfdl_historical_simulation_create_collection_from_zarr.ipynb b/workflows/archive/alaska_et_2020_gfdl_historical_simulation_create_collection_from_zarr.ipynb
index 20d4422756b2c02cab53fd02e87974d204de7b0f..1ded6def536de9ff2bf2865d34fc84f0e9116920 100644
--- a/workflows/archive/alaska_et_2020_gfdl_historical_simulation_create_collection_from_zarr.ipynb
+++ b/workflows/archive/alaska_et_2020_gfdl_historical_simulation_create_collection_from_zarr.ipynb
@@ -496,14 +496,23 @@
},
"outputs": [],
"source": [
- "def get_step(dim_name):\n",
+ "def get_step(ds, dim_name, debug=False, step_ix=0):\n",
" dim_vals = ds[dim_name].values\n",
" diffs = [d2 - d1 for d1, d2 in zip(dim_vals, dim_vals[1:])]\n",
- " unique_steps = np.unique(diffs)\n",
+ " unique_steps = np.unique(diffs, return_counts=True)\n",
+ " step_list = unique_steps[0]\n",
+ " # optional - for invesitgating uneven steps\n",
+ " if debug:\n",
+ " print(f'step_list: {step_list}')\n",
+ " print(f'step_count: {unique_steps[1]}')\n",
+ " indices = [i for i, x in enumerate(diffs) if x == step_list[step_ix]]\n",
+ " print(f'index locations of step index {step_ix} in step_list: {indices}')\n",
" # set step - if all steps are the same length\n",
" # datacube spec specifies to use null for irregularly spaced steps\n",
- " if len(unique_steps)==1:\n",
- " step = unique_steps[0].astype(float).item()\n",
+ " if len(step_list)==1:\n",
+ " # make sure time deltas are in np timedelta format\n",
+ " step_list = [np.array([step], dtype=\"timedelta64[ns]\")[0] for step in step_list]\n",
+ " step = step_list[0].astype(float).item()\n",
" else:\n",
" step = None\n",
" return(step)"
@@ -568,6 +577,90 @@
"print(projjson)"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "067049e1-b5d0-4c4f-8ea9-b3080f74ae4c",
+ "metadata": {},
+ "source": [
+ "#### user review needed - looks at the steps pulled out and make sure they make sense"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2e425eb3-5428-4df7-8859-76cdd55a835e",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "time_step = get_step(ds, dim_names_dict['T'])\n",
+ "print(f'time step: {time_step}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b09f3c09-e635-4db6-80f0-cfae1dfde1f3",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# debugging for time steps: get all step values and locations\n",
+ "time_step = get_step(ds, dim_names_dict['T'], debug=True, step_ix=2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1f970ee5-e74b-406e-8977-9a4d70215045",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# debugging for time steps, cont:\n",
+ "# please choose one of the index locations printed above\n",
+ "# this will print the time steps adjacent to it\n",
+ "ix = 182\n",
+ "ds.isel(time=slice(ix-1,ix+3)).time"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4c83406f-e000-4a50-86dd-aa6f89a6b763",
+ "metadata": {},
+ "source": [
+ "**Irregular time step**: This dataset covers April 1 to Septemper 30 of each year (daily time step), with no data the rest of the time. Because this is an irregular time step, we must specify null for the time step."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bf49514f-4a97-4912-9e92-0916ee3493c6",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "x_step = get_step(ds, dim_names_dict['X'])\n",
+ "print(f'x step: {x_step}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d9c061b9-9c38-4dae-954b-788afca8a410",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "y_step = get_step(ds, dim_names_dict['Y'])\n",
+ "print(f'y step: {y_step}')"
+ ]
+ },
{
"cell_type": "markdown",
"id": "00a5e041-081d-428d-ac2e-75d16de205e6",
@@ -576,6 +669,18 @@
"#### user input needed - you will need to copy all of the dimensions from above into the dict and fill in the appropriate attributes(type, axis, extent):"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0c401e8f-201e-4641-b030-b70b1dbc1bd1",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "print(dims)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -589,9 +694,9 @@
"# dimension name should come from the coordinates printed above\n",
"# we do not recommend including redundant dimensions (do not include x,y if you have lon,lat)\n",
"# note that the extent of each dimension should be pulled from the dataset\n",
- "dims_dict = {'time': pystac.extensions.datacube.Dimension({'type': 'temporal', 'description': get_long_name(ds, 'time'), 'extent': [temporal_extent_lower.strftime('%Y-%m-%dT%XZ'), temporal_extent_upper.strftime('%Y-%m-%dT%XZ')], 'step': pd.Timedelta(get_step('time')).isoformat()}),\n",
- " 'x': pystac.extensions.datacube.Dimension({'type': 'spatial', 'description': get_long_name(ds, 'x'), 'axis': 'x', 'extent': [xy_bounds[0], xy_bounds[2]], 'step': get_step('x'), 'reference_system': projjson}),\n",
- " 'y': pystac.extensions.datacube.Dimension({'type': 'spatial', 'axis': 'y', 'description': get_long_name(ds, 'y'), 'extent': [xy_bounds[1], xy_bounds[3]], 'step': get_step('y'), 'reference_system': projjson}),\n",
+ "dims_dict = {'time': pystac.extensions.datacube.Dimension({'type': 'temporal', 'description': get_long_name(ds, 'time'), 'extent': [temporal_extent_lower.strftime('%Y-%m-%dT%XZ'), temporal_extent_upper.strftime('%Y-%m-%dT%XZ')], 'step': pd.Timedelta(time_step).isoformat()}),\n",
+ " 'x': pystac.extensions.datacube.Dimension({'type': 'spatial', 'description': get_long_name(ds, 'x'), 'axis': 'x', 'extent': [xy_bounds[0], xy_bounds[2]], 'step': x_step, 'reference_system': projjson}),\n",
+ " 'y': pystac.extensions.datacube.Dimension({'type': 'spatial', 'axis': 'y', 'description': get_long_name(ds, 'y'), 'extent': [xy_bounds[1], xy_bounds[3]], 'step': y_step, 'reference_system': projjson}),\n",
" }"
]
},