From 3d53ab5e7ca2c7c340dc660fc0f01dfb3a0702a6 Mon Sep 17 00:00:00 2001
From: David Blodgett <dblodgett@usgs.gov>
Date: Thu, 27 Jul 2023 21:32:31 -0500
Subject: [PATCH] 00 download works now fixes #121
---
workspace/00_get_data.Rmd | 19 ++++++++++++++-----
workspace/cache/data_paths.json | 32 +++++++++++++++++++-------------
2 files changed, 33 insertions(+), 18 deletions(-)
diff --git a/workspace/00_get_data.Rmd b/workspace/00_get_data.Rmd
index a583312..241ef2b 100644
--- a/workspace/00_get_data.Rmd
+++ b/workspace/00_get_data.Rmd
@@ -206,7 +206,7 @@ if(!file.exists(nhdplus_dir)) {
system(paste0(sevenz, " e -o", nhdplus_dir, " ", file.path(nhdplus_dir, xWalk)))
x <- tryCatch(
- download_nhdplusv2(data_dir),
+ download_nhdplusv2(data_dir, url = "https://dmap-data-commons-ow.s3.amazonaws.com/NHDPlusV21/Data/NationalData/NHDPlusV21_NationalData_Seamless_Geodatabase_Lower48_07.7z"),
# Quiet the download, overwrite existing files
error = function(e)
{system(paste0(sevenz, " x ", file.path(data_dir, "NHDPlusV21_NationalData_Seamless_Geodatabase_Lower48_07.7z")
@@ -218,7 +218,7 @@ if(!file.exists(nhdplus_dir)) {
suppressWarnings(staged_nhd <- stage_national_data())
x <- tryCatch(
- download_nhdplusv2(islands_dir, url = paste0("https://edap-ow-data-commons.s3.amazonaws.com/NHDPlusV21/",
+ download_nhdplusv2(islands_dir, url = paste0("https://dmap-data-commons-ow.s3.amazonaws.com/NHDPlusV21/",
"Data/NationalData/NHDPlusV21_NationalData_Seamless", "_Geodatabase_HI_PR_VI_PI_03.7z")),
# Quiet the download, overwrite existing files
@@ -385,14 +385,22 @@ out_list<- c(out_list, out)
```
```{r WBD}
-# Current version of National WBD
+# Snapshot of National WBD
wbd_dir <- file.path(data_dir, "wbd")
wbd_file <- "WBD_National_GDB"
+
+wbd_path <- file.path(wbd_dir, "WBD_National_GDB.zip")
+
if(!dir.exists(wbd_dir)) {
+ if(is.null(sbtools::current_session()))
+ sb <- authenticate_sb()
+
dir.create(wbd_dir, recursive = TRUE)
- wbd <- download_wbd(wbd_dir, "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip")
+
+ sbtools::item_file_download("63cb38b2d34e06fef14f40ad", names = "WBD_National_GDB.zip",
+ destinations = wbd_path, session = sb)
}
out_gdb <- file.path(wbd_dir, paste0(wbd_file, ".gdb"))
@@ -453,6 +461,7 @@ get_sbfile <- function(f, itm) {
sbtools::item_file_download(itm, names = basename(f),
destinations = f)
+ zip::unzip(f, basename(o), exdir = dirname(f))
}
return(o)
}
@@ -518,7 +527,7 @@ targz <- file.path(data_dir, basename(nwm_targz_url))
out <- list(nwm_network = file.path(data_dir, "NWM_parameters_v2.1", "RouteLink_CONUS.nc"))
if(!file.exists(out$nwm_network)) {
-
+ options(timeout = 60000)
download.file(nwm_targz_url, destfile = targz)
utils::untar(targz, exdir = data_dir)
diff --git a/workspace/cache/data_paths.json b/workspace/cache/data_paths.json
index 94643cd..8c08252 100644
--- a/workspace/cache/data_paths.json
+++ b/workspace/cache/data_paths.json
@@ -17,18 +17,6 @@
"waterbodies_path": "data/NHDPlusNationalData/nhdplus_waterbodies.rds",
"fullcats_table": "data/NHDPlusNationalData/nhdcat_full.rds",
"islandcats_table": "data/islands/nhdcat_full.rds",
- "latest_wbd_rds": "data/wbd/WBD.rds",
- "merit_catchments": "data/merged_AK_MERIT_Hydro/cat_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
- "merit_rivers": "data/merged_AK_MERIT_Hydro/riv_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
- "ak_source": "data/AK/ak.gpkg",
- "hi_source": "data/islands/hi.gpkg",
- "GFv11_gages_lyr": "data/GFv11/GFv11_gages.rds",
- "GFv11_gdb": "data/GFv11/GFv1.1.gdb",
- "GFv11_tgf": "data/GFv11/TGF.gdb",
- "gagesii_lyr": "data/SWIM_gage_loc/gagesII_9322_point_shapefile.shp",
- "res_attributes": "data/reservoir_data/ResOpsUS/attributes/reservoir_attributes.csv",
- "istarf": "data/reservoir_data/ISTARF-CONUS.csv",
- "resops_NID_CW": "data/reservoir_data/cw_ResOpsUS_NID.csv",
"fdr": {
"rpu_18a": "data/fdrfac/NHDPlusCA/NHDPlus18/NHDPlusFdrFac18a/fdr",
"rpu_18b": "data/fdrfac/NHDPlusCA/NHDPlus18/NHDPlusFdrFac18b/fdr",
@@ -253,5 +241,23 @@
"rpu_12b": "data/nhdplusv2_elev/NHDPlusTX/NHDPlus12/NEDSnapshot/NED12b/elev_cm",
"rpu_12c": "data/nhdplusv2_elev/NHDPlusTX/NHDPlus12/NEDSnapshot/NED12c/elev_cm",
"rpu_12d": "data/nhdplusv2_elev/NHDPlusTX/NHDPlus12/NEDSnapshot/NED12d/elev_cm"
- }
+ },
+ "latest_wbd_rds": "data/wbd/WBD.rds",
+ "merit_catchments": "data/merged_AK_MERIT_Hydro/cat_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
+ "merit_rivers": "data/merged_AK_MERIT_Hydro/riv_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
+ "aster_dem": "data/merged_AK_MERIT_Hydro/dem.tif",
+ "merit_dem": "data/merged_AK_MERIT_Hydro/ak_merit_dem.tif",
+ "merit_fdr": "data/merged_AK_MERIT_Hydro/ak_merit_fdr.tif",
+ "merit_fac": "data/merged_AK_MERIT_Hydro/ak_merit_fac.tif",
+ "ak_source": "data/AK/ak.gpkg",
+ "hi_source": "data/islands/hi.gpkg",
+ "nwm_network": "data/NWM_parameters_v2.1/RouteLink_CONUS.nc",
+ "nwm_parm": "data/NWM_v2.1_channel_hydrofabric_10262020/nwm_v2_1_hydrofabric.gdb",
+ "GFv11_gages_lyr": "data/GFv11/GFv11_gages.rds",
+ "GFv11_gdb": "data/GFv11/GFv1.1.gdb",
+ "GFv11_tgf": "data/GFv11/TGF.gdb",
+ "gagesii_lyr": "data/SWIM_gage_loc/gagesII_9322_point_shapefile.shp",
+ "res_attributes": "data/reservoir_data/ResOpsUS/attributes/reservoir_attributes.csv",
+ "istarf": "data/reservoir_data/ISTARF-CONUS.csv",
+ "resops_NID_CW": "data/reservoir_data/cw_ResOpsUS_NID.csv"
}
--
GitLab