00 download works now fixes #121

workspace/00_get_data.Rmd

@@ -206,7 +206,7 @@ if(!file.exists(nhdplus_dir)) {
   system(paste0(sevenz, " e -o", nhdplus_dir, " ", file.path(nhdplus_dir, xWalk)))
   
   x <- tryCatch(
-    download_nhdplusv2(data_dir),
+    download_nhdplusv2(data_dir, url = "https://dmap-data-commons-ow.s3.amazonaws.com/NHDPlusV21/Data/NationalData/NHDPlusV21_NationalData_Seamless_Geodatabase_Lower48_07.7z"),
     # Quiet the download, overwrite existing files
     error =  function(e)
   {system(paste0(sevenz, " x ", file.path(data_dir, "NHDPlusV21_NationalData_Seamless_Geodatabase_Lower48_07.7z")
@@ -218,7 +218,7 @@ if(!file.exists(nhdplus_dir)) {
   suppressWarnings(staged_nhd <- stage_national_data())
 
   x <- tryCatch(
-    download_nhdplusv2(islands_dir,  url = paste0("https://edap-ow-data-commons.s3.amazonaws.com/NHDPlusV21/",
+    download_nhdplusv2(islands_dir,  url = paste0("https://dmap-data-commons-ow.s3.amazonaws.com/NHDPlusV21/",
                "Data/NationalData/NHDPlusV21_NationalData_Seamless", "_Geodatabase_HI_PR_VI_PI_03.7z")),
     
     # Quiet the download, overwrite existing files
@@ -385,14 +385,22 @@ out_list<- c(out_list, out)
 ```
 
 ```{r WBD}
-# Current version of National WBD
+# Snapshot of National WBD
 
 wbd_dir <- file.path(data_dir, "wbd")
 
 wbd_file <- "WBD_National_GDB"
+
+wbd_path <- file.path(wbd_dir, "WBD_National_GDB.zip")
+
 if(!dir.exists(wbd_dir)) {
+  if(is.null(sbtools::current_session()))
+    sb <- authenticate_sb()
+  
   dir.create(wbd_dir, recursive = TRUE)
-  wbd <- download_wbd(wbd_dir, "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip")
+  
+  sbtools::item_file_download("63cb38b2d34e06fef14f40ad", names = "WBD_National_GDB.zip", 
+                              destinations = wbd_path, session = sb)
 }
 
 out_gdb <- file.path(wbd_dir, paste0(wbd_file, ".gdb"))
@@ -453,6 +461,7 @@ get_sbfile <- function(f, itm) {
   
     sbtools::item_file_download(itm, names = basename(f), 
                               destinations = f)
+    zip::unzip(f, basename(o), exdir = dirname(f))
   }
   return(o)
 }
@@ -518,7 +527,7 @@ targz <- file.path(data_dir, basename(nwm_targz_url))
 out <- list(nwm_network = file.path(data_dir, "NWM_parameters_v2.1", "RouteLink_CONUS.nc"))
 
 if(!file.exists(out$nwm_network)) {
-  
+  options(timeout = 60000)
   download.file(nwm_targz_url, destfile = targz)
   
   utils::untar(targz, exdir = data_dir)

workspace/cache/data_paths.json

@@ -17,18 +17,6 @@
   "waterbodies_path": "data/NHDPlusNationalData/nhdplus_waterbodies.rds",
   "fullcats_table": "data/NHDPlusNationalData/nhdcat_full.rds",
   "islandcats_table": "data/islands/nhdcat_full.rds",
-  "latest_wbd_rds": "data/wbd/WBD.rds",
-  "merit_catchments": "data/merged_AK_MERIT_Hydro/cat_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
-  "merit_rivers": "data/merged_AK_MERIT_Hydro/riv_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
-  "ak_source": "data/AK/ak.gpkg",
-  "hi_source": "data/islands/hi.gpkg",
-  "GFv11_gages_lyr": "data/GFv11/GFv11_gages.rds",
-  "GFv11_gdb": "data/GFv11/GFv1.1.gdb",
-  "GFv11_tgf": "data/GFv11/TGF.gdb",
-  "gagesii_lyr": "data/SWIM_gage_loc/gagesII_9322_point_shapefile.shp",
-  "res_attributes": "data/reservoir_data/ResOpsUS/attributes/reservoir_attributes.csv",
-  "istarf": "data/reservoir_data/ISTARF-CONUS.csv",
-  "resops_NID_CW": "data/reservoir_data/cw_ResOpsUS_NID.csv",
   "fdr": {
     "rpu_18a": "data/fdrfac/NHDPlusCA/NHDPlus18/NHDPlusFdrFac18a/fdr",
     "rpu_18b": "data/fdrfac/NHDPlusCA/NHDPlus18/NHDPlusFdrFac18b/fdr",
@@ -253,5 +241,23 @@
     "rpu_12b": "data/nhdplusv2_elev/NHDPlusTX/NHDPlus12/NEDSnapshot/NED12b/elev_cm",
     "rpu_12c": "data/nhdplusv2_elev/NHDPlusTX/NHDPlus12/NEDSnapshot/NED12c/elev_cm",
     "rpu_12d": "data/nhdplusv2_elev/NHDPlusTX/NHDPlus12/NEDSnapshot/NED12d/elev_cm"
-  }
+  },
+  "latest_wbd_rds": "data/wbd/WBD.rds",
+  "merit_catchments": "data/merged_AK_MERIT_Hydro/cat_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
+  "merit_rivers": "data/merged_AK_MERIT_Hydro/riv_pfaf_78_81_82_MERIT_Hydro_v07_Basins_v01.shp",
+  "aster_dem": "data/merged_AK_MERIT_Hydro/dem.tif",
+  "merit_dem": "data/merged_AK_MERIT_Hydro/ak_merit_dem.tif",
+  "merit_fdr": "data/merged_AK_MERIT_Hydro/ak_merit_fdr.tif",
+  "merit_fac": "data/merged_AK_MERIT_Hydro/ak_merit_fac.tif",
+  "ak_source": "data/AK/ak.gpkg",
+  "hi_source": "data/islands/hi.gpkg",
+  "nwm_network": "data/NWM_parameters_v2.1/RouteLink_CONUS.nc",
+  "nwm_parm": "data/NWM_v2.1_channel_hydrofabric_10262020/nwm_v2_1_hydrofabric.gdb",
+  "GFv11_gages_lyr": "data/GFv11/GFv11_gages.rds",
+  "GFv11_gdb": "data/GFv11/GFv1.1.gdb",
+  "GFv11_tgf": "data/GFv11/TGF.gdb",
+  "gagesii_lyr": "data/SWIM_gage_loc/gagesII_9322_point_shapefile.shp",
+  "res_attributes": "data/reservoir_data/ResOpsUS/attributes/reservoir_attributes.csv",
+  "istarf": "data/reservoir_data/ISTARF-CONUS.csv",
+  "resops_NID_CW": "data/reservoir_data/cw_ResOpsUS_NID.csv"
 }