get data debug

workspace/00_get_data.Rmd

@@ -21,6 +21,10 @@ source("R/00_get_data_functions.R")
 
 library(sbtools)
 library(jsonlite)
+library(dplyr)
+library(nhdplusTools)
+library(sf)
+library(hyRefactor)
 
 if(!dir.exists("data")) {dir.create("data")}
 if(!dir.exists("bin")) {dir.create("bin")}
@@ -56,22 +60,24 @@ mapview <- FALSE
 HUC12 (Hydrologic Unit Code, Level 12) outlets derived from the Watershed 
 Boundary Dataset indexed to the reference fabricform the baseline and extent of 
 national modeling fabrics.
-```{r HUC12 outlets, eval="huc12" %in% POI_types}
+```{r HUC12 outlets}
 
 #  Blodgett, D.L., 2022, Mainstem Rivers of the Conterminous United States: 
 #  U.S. Geological Survey data release, https://doi.org/10.5066/P9BTKP3T. 
 
 wbd_points_file <- "102020wbd_outlets.gpkg"
 
+if("huc12" %in% POI_types) {
 get_sb_file(item = "63cb38b2d34e06fef14f40ad",
                      item_files = wbd_points_file,
                      out_destination = data_dir)
+}
 
 out_list <- c(
   out_list, 
   list(hu12_points_path = file.path(data_dir, wbd_points_file)))
 
-if(mapview)(mapview(read_sf(out_list$hu12_points_path)))
+if(mapview) try(mapview(read_sf(out_list$hu12_points_path)))
 
 ```
 
@@ -82,12 +88,12 @@ datasets developed for the Streamgage Watershed InforMation (SWIM) project. This
 dataset, which is referred to as “SWIM streamgage locations,” was created in 
 support of the second dataset of basin characteristics and disturbance indexes. 
 
-```{r SWIM, eval="gages" %in% POI_types}
+```{r SWIM}
 #  Hayes, L., Chase, K.J., Wieczorek, M.E., and Jackson, S.E., 2021, 
 #  USGS streamgages in the conterminous United States indexed to NHDPlus v2.1 
 #  flowlines to support Streamgage Watershed InforMation (SWIM), 2021: U.S. 
 #  Geological Survey data release, https://doi.org/10.5066/P9J5CK2Y.
-
+if("gages" %in% POI_types) {
 out_list <- c(
   out_list, 
   list(SWIM_points_path = 
@@ -100,7 +106,8 @@ out_list <- c(
   out_list, 
   list(SWIM_dbf = file.path(data_dir, "SWIMGFinfo.dbf")))
 
-if(mapview)(mapview(read_sf(out_list$SWIM_points_path)))
+if(mapview) try(mapview(read_sf(out_list$SWIM_points_path)))
+}
 ```
 
 Sites associated with Work by the U.S. Geological Survey (USGS)  to estimate 
@@ -108,7 +115,7 @@ the amount of water that is withdrawn and consumed by thermoelectric power
 plants (Diehl and others, 2013; Diehl and Harris, 2014; Harris and Diehl, 2019 
 Galanter and othes, 2023). 
 
-```{r Thermoelectric Facilities, eval="thermo_electric" %in% POI_types}
+```{r Thermoelectric Facilities}
 #   Harris, Melissa A. and Diehl, Timothy H., 2017. A Comparison of Three 
 #   Federal Datasets for Thermoelectric Water Withdrawals in the United States 
 #   for 2010. Journal of the American Water Resources Association (JAWRA) 
@@ -121,7 +128,7 @@ Galanter and othes, 2023).
 #   2008-2020 period by power plant, month, and year for the conterminous 
 #   United States: U.S. Geological Survey data release, 
 #   https://doi.org/10.5066/P9ZE2FVM.
-
+if("thermo_electric" %in% POI_types) {
 TE_points_path <- file.path(data_dir, "TE_points")
 
 dir.create(TE_points_path, recursive = TRUE, showWarnings = FALSE)
@@ -136,7 +143,8 @@ get_sb_file("63adc826d34e92aad3ca5af4",
 
 out_list <- c(out_list, list(TE_points_path = TE_points_path))
 
-if(mapview)(mapview(read_sf(out_list$TE_points_path)))
+if(mapview) try(mapview(read_sf(out_list$TE_points_path)))
+}
 ```
 
 Network locations made to improve the routing capabilities 
@@ -145,17 +153,18 @@ hydrologic analyses. The resulting enhanced network is named E2NHDPlusV2_us.
 This includes the network locations associated with some diversions and 
 water use withdrawals.
 
-```{r e2nhd supplemental data - USGS, eval="addition_removal" %in% POI_types}
+```{r e2nhd supplemental data - USGS}
 #   Schwarz, G.E., 2019, E2NHDPlusV2_us: Database of Ancillary Hydrologic 
 #   Attributes and Modified Routing for NHDPlus Version 2.1 Flowlines: U.S. 
 #   Geological Survey data release, https://doi.org/10.5066/P986KZEM.
-
+if("addition_removal" %in% POI_types) {
 out_list <- c(
   out_list, 
   list(USGS_IT_path = 
          get_sb_file("5d16509ee4b0941bde5d8ffe", 
                      "supplemental_files.zip",
                      file.path(data_dir, "USGS_IT"))))
+}
 ```
 
 Two datasets relate hydro location information from the National Inventory of
@@ -163,7 +172,7 @@ Dams to the NHDPlus network.  One effort is related to the SPARROW work
 (Wieczorek and others, 2018), the other related to work quantifying impacts on
 natural flow (Wieczorek and others, 2021).
 
-```{r National Inventory of Dams, eval="dams" %in% POI_types}
+```{r National Inventory of Dams}
 
 #   Wieczorek, M.E., Jackson, S.E., and Schwarz, G.E., 2018, Select Attributes 
 #   for NHDPlus Version 2.1 Reach Catchments and Modified Network Routed 
@@ -175,6 +184,7 @@ natural flow (Wieczorek and others, 2021).
 #  impact/disturbance metrics for the conterminous United States, 1800 to 2018: 
 #  U.S. Geological Survey data release, https://doi.org/10.5066/P92S9ZX6.
 
+if("dams" %in% POI_types) {
 NID_points_path <- file.path(data_dir, "NID_points")
 
 get_sb_file("5dbc53d4e4b06957974eddae",
@@ -187,7 +197,8 @@ get_sb_file("5fb7e483d34eb413d5e14873",
 
 out_list <- c(out_list, list(NID_points_path = NID_points_path))
 
-if(mapview)(mapview(read_sf(out_list$NID_points_path)))
+if(mapview) try(mapview(read_sf(out_list$NID_points_path)))
+}
 ```
 
 This next section retrieves NHDPlus datasets related to national modeling 
@@ -233,17 +244,6 @@ download_file(paste0(epa_data_root, "NationalData/NHDPlusV21_NationalData_Seamle
               out_path = islands_dir, check_path = islands_gdb)
 }
 
-# cache the huc12 layer in rds format
-hu12_rds <- file.path(nhdplus_dir, "HUC12.rds")
-
-if(!file.exists(hu12_rds)) {
-  read_sf(nhdplus_gdb, layer = "HUC12") |>
-    st_make_valid() |>
-    st_transform(crs = proj_crs) |> 
-    # TODO: convert this to gpkg
-    saveRDS(file = hu12_rds)
-}
-
 get_sb_file("5dcd5f96e4b069579760aedb", "GageLocGFinfo.dbf", data_dir)
 
 download_file(paste0(epa_data_root, "GlobalData/NHDPlusV21_NHDPlusGlobalData_03.7z"),
@@ -322,18 +322,22 @@ out_list <- c(out_list, list(fullcats_table = fullcat_path, islandcats_table = i
 
 Download NHDPlusV2 FDR and FAC grids for refactoring and catcment splitting.
 
-```{r NHDPlusV2 FDR_FAC, eval="CONUS" %in% domain | reg}
+```{r NHDPlusV2 FDR_FAC}
+if("CONUS" %in% domain | reg) {
 # NHDPlus FDR/FAC grids available by raster processing unit
 out_list<- c(out_list, make_fdr_fac_list(file.path(data_dir, "fdrfac"), region))
+}
 
 ```
 
 Download NHDPlusV2 elevation grids for headwater extensions and splitting 
 catchments into left and right banks.
 
-```{r NHDPlusV2 elev, eval="CONUS" %in% domain | reg}
+```{r NHDPlusV2 elev}
+if("CONUS" %in% domain | reg) {
 # NHDPlus elev grids available by raster processing unit
 out_list<- c(out_list, make_nhdplus_elev_list(file.path(data_dir, "nhdplusv2_elev"), region))
+}
 ```
 
 Merrit Topographic and Hydrographic data for deriving GIS Features of the 
@@ -480,8 +484,11 @@ hilarri_dir <- file.path(data_dir, "HILARRI")
 hilarri_out <- list(hilarri_sites = file.path(hilarri_dir, "HILARRI_v2.csv"))
 
 if("reservoirs" %in% POI_types) {
-download_file("https://hydrosource.ornl.gov/sites/default/files/2023-03/HILARRI_v2.zip", 
-              out_path = hilarri_dir, check_path = hilarri_out$hilari_sites)
+  # now behind user tracking
+# download_file("https://hydrosource.ornl.gov/sites/default/files/2023-03/HILARRI_v2.zip", 
+#               out_path = hilarri_dir, check_path = hilarri_out$hilari_sites)
+  
+  get_sb_file("5dbc53d4e4b06957974eddae", "HILARRI_v2.zip", hilarri_dir, check_path = hilarri_out$hilarri_sites)
 }
 
 out_list <- c(out_list, hilarri_out)

workspace/R/00_get_data_functions.R

@@ -6,7 +6,7 @@
 #' @param check_path character path to check, if it exists, it will be returned invisibly
 #' @return path that data was saved to
 get_sb_file <- function(item, item_files, out_destination, unzip = TRUE, check_path = NULL) {
-  
+
   if(!is.null(check_path) && file.exists(check_path)) return(invisible(check_path))
   
   check_auth()
@@ -26,13 +26,24 @@ get_sb_file <- function(item, item_files, out_destination, unzip = TRUE, check_p
                                 names = basename(missing), 
                                 destinations = missing)
   }
+
+  if(!is.null(check_path) && !file.exists(check_path) && unzip) {
+    missing <- out_files[grepl("7z$|zip$", out_files)]
+    warning("check file missing but zip file exists, trying to unzip again")
+  }
   
   if(unzip) {
     
     un7zip_fs <- missing[grepl("7z$|zip$", missing)]
 
     for(f in un7zip_fs) {
-      system(paste0(sevenz, " e -o", out_destination, " ", f))
+      
+      out_destination_use <- out_destination
+      if(!is.null(check_path) && grepl(".gdb.zip", f)) {
+        out_destination_use <- check_path
+      }
+      
+      system(paste0(sevenz, " e -o", out_destination_use, " ", f))
     }
     
   }
@@ -82,7 +93,7 @@ download_file <- function(url, out_path, check_path = NULL, unzip = TRUE, file_n
 #' @return list containing all flow direction and flow accumulation files
 make_fdr_fac_list <- function(fdr_fac_dir, hu2) {
   if(!dir.exists(fdr_fac_dir))
-    download_elev("FDRFAC", fdr_fac_dir)
+    download_elev("FDRFAC", fdr_fac_dir, hu2)
   
   dirs <- unique(dirname(list.files(fdr_fac_dir, recursive = TRUE, full.names = TRUE)))
   
@@ -99,8 +110,8 @@ make_fdr_fac_list <- function(fdr_fac_dir, hu2) {
   out
 }
 
-make_nhdplus_elev_list <- function(elev_dir, region, hu2) {
-  
+make_nhdplus_elev_list <- function(elev_dir, hu2) {
+
   if(!dir.exists(elev_dir))
     download_elev("DEM", elev_dir, hu2)