diff --git a/workspace/00_get_data.Rmd b/workspace/00_get_data.Rmd
index 328abe76df71eb751cd4101830298d1395301e30..1dfaab3aaa35ea8673a4799383d3e3d6f44fbc6b 100644
--- a/workspace/00_get_data.Rmd
+++ b/workspace/00_get_data.Rmd
@@ -49,7 +49,7 @@ if(!file.exists(hu12_points_path)) {
if(is.null(sbtools::current_session()))
sb <- authenticate_sb()
- sbtools::item_file_download("63cb38b2d34e06fef14f40ad", names = "102020wbd_outlets.gpkg",
+ sbtools::item_file_download("60cb5edfd34e86b938a373f4", names = "102020wbd_outlets.gpkg",
destinations = hu12_points_path, session = sb)
}
@@ -222,7 +222,8 @@ if(!file.exists(nhdplus_dir)) {
suppressWarnings(staged_nhd <- stage_national_data())
x <- tryCatch(
- download_nhdplusv2(islands_dir, "https://s3.amazonaws.com/edap-nhdplus/NHDPlusV21/Data/NationalData/NHDPlusV21_NationalData_Seamless_Geodatabase_HI_PR_VI_PI_03.7z"),
+ download_nhdplusv2(islands_dir, url = paste0("https://edap-ow-data-commons.s3.amazonaws.com/NHDPlusV21/",
+ "Data/NationalData/NHDPlusV21_NationalData_Seamless", "_Geodatabase_HI_PR_VI_PI_03.7z")),
# Quiet the download, overwrite existing files
error = function(e)
@@ -262,6 +263,44 @@ out_list <- c(out_list, list(nhdplus_dir = nhdplus_dir, nhdplus_gdb = nhdplus_gd
nhdplus_rpu = rpu))
```
+```{r Reference Fabric}
+ref_fab_path <- file.path(data_dir, "reference_fabric")
+ref_cat <- file.path(ref_fab_path, "reference_catchments.gpkg")
+ref_fl <- file.path(ref_fab_path, "reference_flowline.gpkg")
+nwm_fl <- file.path(ref_fab_path, "nwm_network.gpkg")
+
+if(!dir.exists(ref_fab_path)) {
+ dir.create(ref_fab_path, recursive = TRUE, showWarnings = FALSE)
+}
+
+if(!file.exists(ref_cat)){
+ possible_vpu <- c("01", "08", "10L", "15", "02", "04", "05", "06", "07", "09",
+ "03S", "03W", "03N", "10U", "11", "12", "13", "14", "16",
+ "17", "18")
+ if(is.null(sbtools::current_session()))
+ authenticate_sb()
+
+ for (vpu in possible_vpu){
+ vpu_gpkg <- paste0(vpu, "_reference_features.gpkg")
+ sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = vpu_gpkg,
+ destinations = file.path(ref_fab_path, vpu_gpkg))
+ }
+
+ sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = "reference_catchments.gpkg",
+ destinations = ref_cat)
+
+ sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = "reference_flowline.gpkg",
+ destinations = ref_fl)
+
+ sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = "nwm_network.gpkg",
+ destinations = nwm_fl)
+}
+
+
+out_list <- c(out_list, list(ref_fab_path = ref_fab_path, ref_cat = ref_cat, ref_fl = ref_fl, nwm_fl = nwm_fl))
+```
+
+
```{r NHDPlusV2 Waterbodies}
# Waterbodies - derived after downloading and post-processing NHDPlus Seamless National Geodatabase
waterbodies_path <- file.path(nhdplus_dir, "nhdplus_waterbodies.rds")
@@ -273,7 +312,7 @@ if(!file.exists(waterbodies_path)) {
# Read the feature class
wbSF <- read_sf(nhdplus_gdb, "NHDWaterbody") %>%
- st_transform(crs)
+ st_transform(5070)
# Convert to simple feature and save out
saveRDS(wbSF, waterbodies_path)
@@ -326,7 +365,6 @@ out$fac <- as.list(setNames(fac, paste0("rpu_", rpu)))
out_list<- c(out_list, out)
```
-
```{r NHDPlusV2 elev}
# NHDPlus FDR/FAC grids available by raster processing unit
@@ -355,15 +393,18 @@ out_list<- c(out_list, out)
wbd_dir <- file.path(data_dir, "wbd")
+wbd_file <- "WBD_National_GDB"
if(!dir.exists(wbd_dir)) {
dir.create(wbd_dir, recursive = TRUE)
wbd <- download_wbd(wbd_dir, "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip")
}
-out_gdb <- file.path(wbd_dir, "WBD_National_GDB.gdb")
+out_gdb <- file.path(wbd_dir, paste0(wbd_file, ".gdb"))
out <- list(latest_wbd = file.path(wbd_dir, "WBD.rds"))
if(!file.exists(out$latest_wbd)) {
+ system(paste0(sevenz, " x ", file.path(wbd_dir, paste0(wbd_file, ".zip")), " -o", wbd_dir), ignore.stdout = TRUE, intern = TRUE)
+
# Read the feature class
wbdfc <- sf::read_sf(out_gdb,
"WBDHU12") %>%
@@ -417,7 +458,7 @@ get_sbfile <- function(f, itm) {
sbtools::item_file_download(itm, names = basename(f),
destinations = f)
- zip::unzip(f)
+ #zip::unzip(f)
}
return(o)
}
@@ -470,6 +511,31 @@ if(!file.exists(file.path(islands_dir, "hi.gpkg"))) {
out_list <- c(out_list, out_hi)
```
+```{r e2nhd}
+zip_file <- "e2nhdplusv2_us_csv.zip"
+out_csv <- "e2nhdplusv2_us.csv"
+
+out_zip <- file.path(data_dir, out_csv)
+
+out <- list(e2nhd_network = file.path(data_dir, out_csv))
+
+if(!file.exists(out$e2nhd_network)) {
+
+ if(is.null(sbtools::current_session()))
+ authenticate_sb()
+
+ sbtools::item_file_download("5d16509ee4b0941bde5d8ffe",
+ names = zip_file,
+ destinations = file.path(data_dir, zip_file))
+
+ zip::unzip(file.path(data_dir, zip_file), exdir = data_dir)
+
+}
+
+out_list <- c(out_list, out)
+
+```
+
```{r nwm_topology}
nwm_targz_url <- "https://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/NWM_parameters/NWM_parameter_files_v2.1.tar.gz"
nwm_parm_url <- "https://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_v2.1_channel_hydrofabric.tar.gz"
@@ -508,13 +574,13 @@ out_list <- c(out_list, out)
```{r nhdplus_attributes}
-out <- list(new_nhdp_atts = file.path("data", (sb_f <- "enhd_nhdplusatts.csv")))
+out <- list(new_nhdp_atts = file.path("cache", (sb_f <- "enhd_nhdplusatts.csv")))
if(!file.exists(out$new_nhdp_atts)) {
if(is.null(sbtools::current_session()))
authenticate_sb()
- sbtools::item_file_download("63cb311ed34e06fef14f40a3",
+ sbtools::item_file_download("60c92503d34e86b9389df1c9",
names = sb_f,
destinations = out$new_nhdp_atts)
}
@@ -522,6 +588,22 @@ if(!file.exists(out$new_nhdp_atts)) {
out_list <- c(out_list, out)
```
+```{r nwm_nhdplus_attributes}
+
+out <- list(new_nwm_nhdp_atts = file.path("cache", (sb_f <- "nwm_enhd_nhdplusatts.csv")))
+
+if(!file.exists(out$new_nwm_nhdp_atts)) {
+ if(is.null(sbtools::current_session()))
+ authenticate_sb()
+
+ sbtools::item_file_download("60c92503d34e86b9389df1c9",
+ names = sb_f,
+ destinations = out$new_nwm_nhdp_atts)
+}
+
+out_list <- c(out_list, out)
+```
+
```{r GFv1.1}
GFv11_dir <- file.path(data_dir, "GFv11")
out <- list(GFv11_gages_lyr = file.path(data_dir, "GFv11/GFv11_gages.rds"),
@@ -581,60 +663,6 @@ if(!file.exists(g2_out$gagesii_lyr)) {
out_list <- c(out_list, g2_out)
```
-```{r updated_flowlines}
-
-out <- list(ref_flowline = file.path(out_list$nhdplus_dir, (sb_f <- "reference_flowline.gpkg")))
-
-if(!file.exists(out$ref_flowline)) {
- if(is.null(sbtools::current_session()))
- authenticate_sb()
-
- fs <- sbtools::item_list_files("61295190d34e40dd9c06bcd7")
-
- if(!sb_f %in% fs$fname) {
-
- stop("you have to run this part interactively")
-
- staged_nhd <- stage_national_data(nhdplus_data = out_list$nhdplus_gdb,
- output_path = out_list$nhdplus_dir)
-
- future::plan(future::multisession(workers = 16))
- # This will generate
- fix_headwaters(staged_nhd$flowline,
- out$ref_flowline,
- new_atts = out_list$new_nhdp_atts,
- nhdpath = out_list$nhdplus_gdb)
-
- sbtools::authenticate_sb()
- sbtools::item_replace_files("61295190d34e40dd9c06bcd7", out$ref_flowline)
-
- }
-
- sbtools::item_file_download("61295190d34e40dd9c06bcd7",
- names = sb_f,
- destinations = out$ref_flowline)
-}
-
-out_list <- c(out_list, out)
-```
-
-```{r reference_catchments}
-
-out <- list(ref_catchment = file.path(out_list$nhdplus_dir, (sb_c <- "reference_catchments.gpkg")))
-
-if(!file.exists(out$ref_catchment)) {
- if(is.null(sbtools::current_session()))
- authenticate_sb()
-
- sbtools::item_file_download("61295190d34e40dd9c06bcd7",
- names = sb_c,
- destinations = out$ref_catchment)
-}
-
-out_list <- c(out_list, out)
-```
-
-
```{r res}
# Download Reservoir and dams data: ResOps, ISTARF and GRanDd
#