Skip to content
Snippets Groups Projects
Commit 8586206f authored by Bock, Andy's avatar Bock, Andy
Browse files

Added new RF source geopackages, a few other updates

parent 508aeb8f
No related branches found
No related tags found
1 merge request!169Updates through 07_merge
Hide whitespace changes
Inline Side-by-side
workspace/00_get_data.Rmd
+ 90
− 62
View file @ 8586206f
......@@ -49,7 +49,7 @@ if(!file.exists(hu12_points_path)) {
if(is.null(sbtools::current_session()))
sb <- authenticate_sb()
sbtools::item_file_download("63cb38b2d34e06fef14f40ad", names = "102020wbd_outlets.gpkg",
sbtools::item_file_download("60cb5edfd34e86b938a373f4", names = "102020wbd_outlets.gpkg",
destinations = hu12_points_path, session = sb)
}
......@@ -222,7 +222,8 @@ if(!file.exists(nhdplus_dir)) {
suppressWarnings(staged_nhd <- stage_national_data())
x <- tryCatch(
download_nhdplusv2(islands_dir, "https://s3.amazonaws.com/edap-nhdplus/NHDPlusV21/Data/NationalData/NHDPlusV21_NationalData_Seamless_Geodatabase_HI_PR_VI_PI_03.7z"),
download_nhdplusv2(islands_dir, url = paste0("https://edap-ow-data-commons.s3.amazonaws.com/NHDPlusV21/",
"Data/NationalData/NHDPlusV21_NationalData_Seamless", "_Geodatabase_HI_PR_VI_PI_03.7z")),
# Quiet the download, overwrite existing files
error = function(e)
......@@ -262,6 +263,44 @@ out_list <- c(out_list, list(nhdplus_dir = nhdplus_dir, nhdplus_gdb = nhdplus_gd
nhdplus_rpu = rpu))
```
```{r Reference Fabric}
ref_fab_path <- file.path(data_dir, "reference_fabric")
ref_cat <- file.path(ref_fab_path, "reference_catchments.gpkg")
ref_fl <- file.path(ref_fab_path, "reference_flowline.gpkg")
nwm_fl <- file.path(ref_fab_path, "nwm_network.gpkg")
if(!dir.exists(ref_fab_path)) {
dir.create(ref_fab_path, recursive = TRUE, showWarnings = FALSE)
}
if(!file.exists(ref_cat)){
possible_vpu <- c("01", "08", "10L", "15", "02", "04", "05", "06", "07", "09",
"03S", "03W", "03N", "10U", "11", "12", "13", "14", "16",
"17", "18")
if(is.null(sbtools::current_session()))
authenticate_sb()
for (vpu in possible_vpu){
vpu_gpkg <- paste0(vpu, "_reference_features.gpkg")
sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = vpu_gpkg,
destinations = file.path(ref_fab_path, vpu_gpkg))
}
sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = "reference_catchments.gpkg",
destinations = ref_cat)
sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = "reference_flowline.gpkg",
destinations = ref_fl)
sbtools::item_file_download("61295190d34e40dd9c06bcd7", names = "nwm_network.gpkg",
destinations = nwm_fl)
}
out_list <- c(out_list, list(ref_fab_path = ref_fab_path, ref_cat = ref_cat, ref_fl = ref_fl, nwm_fl = nwm_fl))
```
```{r NHDPlusV2 Waterbodies}
# Waterbodies - derived after downloading and post-processing NHDPlus Seamless National Geodatabase
waterbodies_path <- file.path(nhdplus_dir, "nhdplus_waterbodies.rds")
......@@ -273,7 +312,7 @@ if(!file.exists(waterbodies_path)) {
# Read the feature class
wbSF <- read_sf(nhdplus_gdb, "NHDWaterbody") %>%
st_transform(crs)
st_transform(5070)
# Convert to simple feature and save out
saveRDS(wbSF, waterbodies_path)
......@@ -326,7 +365,6 @@ out$fac <- as.list(setNames(fac, paste0("rpu_", rpu)))
out_list<- c(out_list, out)
```
```{r NHDPlusV2 elev}
# NHDPlus FDR/FAC grids available by raster processing unit
......@@ -355,15 +393,18 @@ out_list<- c(out_list, out)
wbd_dir <- file.path(data_dir, "wbd")
wbd_file <- "WBD_National_GDB"
if(!dir.exists(wbd_dir)) {
dir.create(wbd_dir, recursive = TRUE)
wbd <- download_wbd(wbd_dir, "https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip")
}
out_gdb <- file.path(wbd_dir, "WBD_National_GDB.gdb")
out_gdb <- file.path(wbd_dir, paste0(wbd_file, ".gdb"))
out <- list(latest_wbd = file.path(wbd_dir, "WBD.rds"))
if(!file.exists(out$latest_wbd)) {
system(paste0(sevenz, " x ", file.path(wbd_dir, paste0(wbd_file, ".zip")), " -o", wbd_dir), ignore.stdout = TRUE, intern = TRUE)
# Read the feature class
wbdfc <- sf::read_sf(out_gdb,
"WBDHU12") %>%
......@@ -417,7 +458,7 @@ get_sbfile <- function(f, itm) {
sbtools::item_file_download(itm, names = basename(f),
destinations = f)
zip::unzip(f)
#zip::unzip(f)
}
return(o)
}
......@@ -470,6 +511,31 @@ if(!file.exists(file.path(islands_dir, "hi.gpkg"))) {
out_list <- c(out_list, out_hi)
```
```{r e2nhd}
zip_file <- "e2nhdplusv2_us_csv.zip"
out_csv <- "e2nhdplusv2_us.csv"
out_zip <- file.path(data_dir, out_csv)
out <- list(e2nhd_network = file.path(data_dir, out_csv))
if(!file.exists(out$e2nhd_network)) {
if(is.null(sbtools::current_session()))
authenticate_sb()
sbtools::item_file_download("5d16509ee4b0941bde5d8ffe",
names = zip_file,
destinations = file.path(data_dir, zip_file))
zip::unzip(file.path(data_dir, zip_file), exdir = data_dir)
}
out_list <- c(out_list, out)
```
```{r nwm_topology}
nwm_targz_url <- "https://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/NWM_parameters/NWM_parameter_files_v2.1.tar.gz"
nwm_parm_url <- "https://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_v2.1_channel_hydrofabric.tar.gz"
......@@ -508,13 +574,13 @@ out_list <- c(out_list, out)
```{r nhdplus_attributes}
out <- list(new_nhdp_atts = file.path("data", (sb_f <- "enhd_nhdplusatts.csv")))
out <- list(new_nhdp_atts = file.path("cache", (sb_f <- "enhd_nhdplusatts.csv")))
if(!file.exists(out$new_nhdp_atts)) {
if(is.null(sbtools::current_session()))
authenticate_sb()
sbtools::item_file_download("63cb311ed34e06fef14f40a3",
sbtools::item_file_download("60c92503d34e86b9389df1c9",
names = sb_f,
destinations = out$new_nhdp_atts)
}
......@@ -522,6 +588,22 @@ if(!file.exists(out$new_nhdp_atts)) {
out_list <- c(out_list, out)
```
```{r nwm_nhdplus_attributes}
out <- list(new_nwm_nhdp_atts = file.path("cache", (sb_f <- "nwm_enhd_nhdplusatts.csv")))
if(!file.exists(out$new_nwm_nhdp_atts)) {
if(is.null(sbtools::current_session()))
authenticate_sb()
sbtools::item_file_download("60c92503d34e86b9389df1c9",
names = sb_f,
destinations = out$new_nwm_nhdp_atts)
}
out_list <- c(out_list, out)
```
```{r GFv1.1}
GFv11_dir <- file.path(data_dir, "GFv11")
out <- list(GFv11_gages_lyr = file.path(data_dir, "GFv11/GFv11_gages.rds"),
......@@ -581,60 +663,6 @@ if(!file.exists(g2_out$gagesii_lyr)) {
out_list <- c(out_list, g2_out)
```
```{r updated_flowlines}
out <- list(ref_flowline = file.path(out_list$nhdplus_dir, (sb_f <- "reference_flowline.gpkg")))
if(!file.exists(out$ref_flowline)) {
if(is.null(sbtools::current_session()))
authenticate_sb()
fs <- sbtools::item_list_files("61295190d34e40dd9c06bcd7")
if(!sb_f %in% fs$fname) {
stop("you have to run this part interactively")
staged_nhd <- stage_national_data(nhdplus_data = out_list$nhdplus_gdb,
output_path = out_list$nhdplus_dir)
future::plan(future::multisession(workers = 16))
# This will generate
fix_headwaters(staged_nhd$flowline,
out$ref_flowline,
new_atts = out_list$new_nhdp_atts,
nhdpath = out_list$nhdplus_gdb)
sbtools::authenticate_sb()
sbtools::item_replace_files("61295190d34e40dd9c06bcd7", out$ref_flowline)
}
sbtools::item_file_download("61295190d34e40dd9c06bcd7",
names = sb_f,
destinations = out$ref_flowline)
}
out_list <- c(out_list, out)
```
```{r reference_catchments}
out <- list(ref_catchment = file.path(out_list$nhdplus_dir, (sb_c <- "reference_catchments.gpkg")))
if(!file.exists(out$ref_catchment)) {
if(is.null(sbtools::current_session()))
authenticate_sb()
sbtools::item_file_download("61295190d34e40dd9c06bcd7",
names = sb_c,
destinations = out$ref_catchment)
}
out_list <- c(out_list, out)
```
```{r res}
# Download Reservoir and dams data: ResOps, ISTARF and GRanDd
#
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment