Merge branch 'main' into 'main'

more clean up

See merge request !162

.gitignore

@@ -19,4 +19,5 @@ workspace/bin
 workspace/reports
 *.fst
 *.parquet
-*.Renviron
\ No newline at end of file
+*.Renviron
+*.zip
\ No newline at end of file

workspace/02_NHD_navigate.Rmd

@@ -22,7 +22,6 @@ knitr::opts_chunk$set(
 source("R/utils.R")
 source("R/NHD_navigate.R")
 source("R/hyrefactor_funs.R")
-source("../hyfabric/R/poi_creation.R")
 
 # increase timeout for data downloads
 options(timeout=600)

workspace/08_merge.Rmd

@@ -21,8 +21,8 @@ knitr::opts_chunk$set(
 
 ```{r setup}
 # Load custom functions
-source("R/config.R")
 source("R/utils.R")
+source("R/config.R")
 
 ```
 
@@ -30,7 +30,7 @@ source("R/utils.R")
 
 ```{r unify network}
 
-merge_conus(rpus, rpu_vpu_out, 
+merge_conus(rpu_vpu, rpu_vpu_out, 
             lookup_table_refactor, 
             reconciled_layer,
             divide_layer, 
@@ -38,4 +38,20 @@ merge_conus(rpus, rpu_vpu_out,
             agg_cats_layer,
             mapped_outlets_layer)
 
+vpus <- unique(rpu_vpu$vpuid)
+  
+gf_gpkgs <- paste0("temp/GF_", vpus, ".gpkg")
+
+out_gpkg <- "cache/GF.gpkg"
+
+layers <- sf::st_layers(gf_gpkgs[1])
+
+for(g in gf_gpkgs) {
+
+  for(l in layers$name) {
+    sf::write_sf(sf::read_sf(g, l), out_gpkg, l, append = TRUE) 
+  }
+    
+}
+
 ```

workspace/R/utils.R

@@ -317,11 +317,17 @@ merge_conus <- function(rpu_vpu, rpu_vpu_out,
   
   gf_gpkgs <- paste0("cache/GF_", vpus, ".gpkg")
   
-  stop("not implemented")
-  
-  # open all lookup tables and reconcile just the IDs tracking where the IDs came from.
-  
-  # open all the spatial data and update IDs to match.
+  hydrofab::assign_global_identifiers(
+    gpkgs = gf_gpkgs, 
+    outfiles = gsub("cache", "temp", gf_gpkgs), 
+    flowpath_layer = agg_fline_layer, 
+    mapped_POI_layer = mapped_outlets_layer, 
+    divide_layer = agg_cats_layer, 
+    lookup_table_layer = lookup_table_refactor, 
+    flowpath_edge_list = catchment_network_table, 
+    update_terminals = TRUE, 
+    return_lookup = TRUE, 
+    verbose = TRUE)
   
 }
 
@@ -476,24 +482,52 @@ merge_refactor <- function(rpus, rpu_vpu_out,
       df[[n]]
     }, n = x))
   }, out = out), names(out[[1]]))
-  
-  # blow up so we have unique COMIDs to join on.
-  # need to keep the top most of any splits (the .1 variety)
-  # this makes sure out toCOMID assignments go to the right new id.
-  long_form <- st_drop_geometry(out[[reconciled_layer]]) %>%
-    select(newID, member_COMID) %>%
-    mutate(member_COMID = strsplit(member_COMID, ",")) %>%
-    tidyr::unnest(cols = member_COMID) %>%
-    filter(grepl("\\.1$", member_COMID) | !grepl("\\.", member_COMID)) %>%
-    mutate(NHDPlusV2_COMID = as.integer(member_COMID)) %>%
-    select(-member_COMID, update_newtoID = newID)
-  
+
   if("toCOMID" %in% names(out[[reconciled_layer]])) {
+    
+    # blow up so we have unique COMIDs to join on.
+    # need to keep the top most of any splits (the .1 variety)
+    # this makes sure out toCOMID assignments go to the right new id.
+    long_form <- st_drop_geometry(out[[reconciled_layer]]) %>%
+      select(newID, member_COMID) %>%
+      mutate(member_COMID = strsplit(member_COMID, ",")) %>%
+      tidyr::unnest(cols = member_COMID) %>%
+      filter(grepl("\\.1$", member_COMID) | !grepl("\\.", member_COMID)) %>%
+      mutate(NHDPlusV2_COMID = as.integer(member_COMID)) %>%
+      select(-member_COMID, update_newtoID = newID)
+    
     # NOTE: if the missing tocomid is in the next VPU they will still be NA
     out[[reconciled_layer]] <- left_join(out[[reconciled_layer]], 
                                          long_form, by = c("toCOMID" = "NHDPlusV2_COMID"))
     out[[reconciled_layer]]$newtoID[!is.na(out[[reconciled_layer]]$toCOMID)] <- 
       out[[reconciled_layer]]$update_newtoID[!is.na(out[[reconciled_layer]]$toCOMID)]
+    
+    # now use updates from refactored ids with aggregate
+    long_form <- st_drop_geometry(out[[agg_fline_layer]]) %>%
+      select(ID, set) %>%
+      mutate(set = strsplit(set, ",")) %>%
+      tidyr::unnest(cols = set) %>%
+      mutate(set = as.integer(set)) %>%
+      select(update_newtoID = ID, refactor_ID = set)
+    
+    # join updated toID from refactor so we can update as needed
+    out[[agg_fline_layer]] <- out[[agg_fline_layer]] %>%
+      # this adds a "new_toID" containing the refactor id that the 
+      # aggregate flowpaths we are updating need to go to.
+      left_join(select(st_drop_geometry(out[[reconciled_layer]]), 
+                       ID = newID, new_toID = update_newtoID),
+                by = "ID") %>%
+      # we need to handle sets of refactor ids and get the right aggregate toID
+      left_join(long_form, by = c("new_toID" = "refactor_ID"))
+    # need to update the field to reflect the information joined just above.
+    out[[agg_fline_layer]]$new_toID[!is.na(out[[agg_fline_layer]]$update_newtoID)] <-
+      out[[agg_fline_layer]]$update_newtoID[!is.na(out[[agg_fline_layer]]$update_newtoID)]
+    
+    # now do the actual toID updates
+    out[[agg_fline_layer]]$toID[!is.na(out[[agg_fline_layer]]$new_toID)] <- 
+      out[[agg_fline_layer]]$new_toID[!is.na(out[[agg_fline_layer]]$new_toID)]
+    
+    out[[agg_fline_layer]] <- select(out[[agg_fline_layer]], -new_toID, -update_newtoID)
   }
   
   out[[reconciled_layer]] <- select(out[[reconciled_layer]], ID = newID, 

workspace/runners/cleaner.R

+possible_vpu <- c("01", "08", "10L", "15", "02", "04", "05", "06", 
+                  "07", "09", "03S", "03W", 
+                  "03N", "10U", "11", "12", 
+                  "13", "14",  "16", "17", "18")
+
+
+vpu_codes <- possible_vpu
+
+# remove all reference fabric content
+reference <- FALSE
+# remove al POI content
+POI <- TRUE
+# remove refactored content leaving rpu-based reference content behind
+refactor <- TRUE
+# remove all aggregate content
+aggregate <- FALSE
+
+for(VPU in vpu_codes) {
+  
+  rm(rpu_code)
+  
+  source("R/config.R")
+  source("R/utils.R")
+  
+  if(reference) {
+    unlink(nav_gpkg)
+  }
+  
+  if(POI) {
+    sf::st_delete(nav_gpkg, final_poi_layer)
+    sf::st_delete(nav_gpkg, lookup_table_refactor)
+    unlink(temp_gpkg)    
+  }
+  
+  if(refactor) {
+    unlink(rfc_gpkg)
+  }
+  
+  if(aggregate) {
+    unlink(gf_gpkg)
+    unlink(ND_gpkg)
+  }
+  
+  if(refactor | aggregate) {
+    unlink(vpu_lookup_table_file)
+  }
+  
+  for(r in rpu_codes$rpuid) {
+    f <- paste0("cache/", r, "_refactor.gpkg")
+    if(refactor & !reference) {
+      sf::st_delete(f, lookup_table_refactor)
+      sf::st_delete(f, refactored_layer)
+      sf::st_delete(f, reconciled_layer)
+      sf::st_delete(f, outlets_layer)
+      sf::st_delete(f, divide_layer)
+      unlink(paste0("cache/", r, "_lookup.csv"))
+    }
+    
+    if(refactor & reference) {
+      unlink(f)
+      unlink(paste0("cache/", r, "_lookup.csv"))
+    }
+    
+    if(aggregate) {
+      unlink(paste0("cache/", r, "_aggregate.gpkg"))
+      unlink(paste0("cache/", r, "_lookup.csv"))
+    }
+  }
+  
+}

workspace/runners/local_batch.sh

+#!/bin/bash
+
+docker run --mount type=bind,source=$PWD/workspace,target=/jupyter dblodgett/hydrogeoenv-custom:latest R -e "rmarkdown::render('/jupyter/01_NHD_prep.Rmd', output_file='/jupyter/reports/01_NHD_prep.html')" 
+
+parallel --jobs 5 < workspace/runners/navigate_calls.txt
+
+parallel --jobs 5 < workspace/runners/hyRefactor_calls.txt
+
+parallel --jobs 5 < workspace/runners/finalize_calls.txt

workspace/runners/upload_all.R

@@ -25,14 +25,13 @@ for(VPU in possible_vpu) {
     check_auth()
     message(nav_gpkg)
     sbtools::item_replace_files(sb_reference, nav_gpkg)
-    
-    check_auth()
 
+    check_auth()
     message(rfc_gpkg)
     sbtools::item_replace_files(sb_refactor, rfc_gpkg)
 
 
-    check_auth
+    check_auth()
     message(gf_gpkg)
     sbtools::item_replace_files(sb_gf, gf_gpkg)