national merge works and bug fix for RPU merge

workspace/08_merge.Rmd

@@ -21,8 +21,8 @@ knitr::opts_chunk$set(
 
 ```{r setup}
 # Load custom functions
-source("R/config.R")
 source("R/utils.R")
+source("R/config.R")
 
 ```
 
@@ -30,7 +30,7 @@ source("R/utils.R")
 
 ```{r unify network}
 
-merge_conus(rpus, rpu_vpu_out, 
+merge_conus(rpu_vpu, rpu_vpu_out, 
             lookup_table_refactor, 
             reconciled_layer,
             divide_layer, 
@@ -38,4 +38,20 @@ merge_conus(rpus, rpu_vpu_out,
             agg_cats_layer,
             mapped_outlets_layer)
 
+vpus <- unique(rpu_vpu$vpuid)
+  
+gf_gpkgs <- paste0("temp/GF_", vpus, ".gpkg")
+
+out_gpkg <- "cache/GF.gpkg"
+
+layers <- sf::st_layers(gf_gpkgs[1])
+
+for(g in gf_gpkgs) {
+
+  for(l in layers$name) {
+    sf::write_sf(sf::read_sf(g, l), out_gpkg, l, append = TRUE) 
+  }
+    
+}
+
 ```

workspace/R/utils.R

@@ -317,11 +317,17 @@ merge_conus <- function(rpu_vpu, rpu_vpu_out,
   
   gf_gpkgs <- paste0("cache/GF_", vpus, ".gpkg")
   
-  stop("not implemented")
-  
-  # open all lookup tables and reconcile just the IDs tracking where the IDs came from.
-  
-  # open all the spatial data and update IDs to match.
+  hydrofab::assign_global_identifiers(
+    gpkgs = gf_gpkgs, 
+    outfiles = gsub("cache", "temp", gf_gpkgs), 
+    flowpath_layer = agg_fline_layer, 
+    mapped_POI_layer = mapped_outlets_layer, 
+    divide_layer = agg_cats_layer, 
+    lookup_table_layer = lookup_table_refactor, 
+    flowpath_edge_list = catchment_network_table, 
+    update_terminals = TRUE, 
+    return_lookup = TRUE, 
+    verbose = TRUE)
   
 }
 
@@ -476,24 +482,44 @@ merge_refactor <- function(rpus, rpu_vpu_out,
       df[[n]]
     }, n = x))
   }, out = out), names(out[[1]]))
-  
-  # blow up so we have unique COMIDs to join on.
-  # need to keep the top most of any splits (the .1 variety)
-  # this makes sure out toCOMID assignments go to the right new id.
-  long_form <- st_drop_geometry(out[[reconciled_layer]]) %>%
-    select(newID, member_COMID) %>%
-    mutate(member_COMID = strsplit(member_COMID, ",")) %>%
-    tidyr::unnest(cols = member_COMID) %>%
-    filter(grepl("\\.1$", member_COMID) | !grepl("\\.", member_COMID)) %>%
-    mutate(NHDPlusV2_COMID = as.integer(member_COMID)) %>%
-    select(-member_COMID, update_newtoID = newID)
-  
+
   if("toCOMID" %in% names(out[[reconciled_layer]])) {
+    
+    # blow up so we have unique COMIDs to join on.
+    # need to keep the top most of any splits (the .1 variety)
+    # this makes sure out toCOMID assignments go to the right new id.
+    long_form <- st_drop_geometry(out[[reconciled_layer]]) %>%
+      select(newID, member_COMID) %>%
+      mutate(member_COMID = strsplit(member_COMID, ",")) %>%
+      tidyr::unnest(cols = member_COMID) %>%
+      filter(grepl("\\.1$", member_COMID) | !grepl("\\.", member_COMID)) %>%
+      mutate(NHDPlusV2_COMID = as.integer(member_COMID)) %>%
+      select(-member_COMID, update_newtoID = newID)
+    
     # NOTE: if the missing tocomid is in the next VPU they will still be NA
     out[[reconciled_layer]] <- left_join(out[[reconciled_layer]], 
                                          long_form, by = c("toCOMID" = "NHDPlusV2_COMID"))
     out[[reconciled_layer]]$newtoID[!is.na(out[[reconciled_layer]]$toCOMID)] <- 
       out[[reconciled_layer]]$update_newtoID[!is.na(out[[reconciled_layer]]$toCOMID)]
+    
+    # now use updates from refactored ids with aggregate
+    long_form <- st_drop_geometry(out[[agg_fline_layer]]) %>%
+      select(ID, set) %>%
+      mutate(set = strsplit(set, ",")) %>%
+      tidyr::unnest(cols = set) %>%
+      mutate(set = as.integer(set)) %>%
+      select(update_newtoID = ID, refactor_ID = set)
+    
+    # join updated toID from refactor so we can update as needed
+    out[[agg_fline_layer]] <- 
+      left_join(out[[agg_fline_layer]], 
+                select(st_drop_geometry(out[[reconciled_layer]]), 
+                       ID = newID, new_toID = update_newtoID),
+                by = "ID")
+    out[[agg_fline_layer]]$toID[!is.na(out[[agg_fline_layer]]$new_toID)] <- 
+      out[[agg_fline_layer]]$new_toID[!is.na(out[[agg_fline_layer]]$new_toID)]
+    
+    out[[agg_fline_layer]] <- select(out[[agg_fline_layer]], -new_toID)
   }
   
   out[[reconciled_layer]] <- select(out[[reconciled_layer]], ID = newID, 

workspace/runners/local_batch.sh

+#!/bin/bash
+
+docker run --mount type=bind,source=$PWD/workspace,target=/jupyter dblodgett/hydrogeoenv-custom:latest R -e "rmarkdown::render('/jupyter/01_NHD_prep.Rmd', output_file='/jupyter/reports/01_NHD_prep.html')" 
+
+parallel --jobs 5 < workspace/runners/navigate_calls.txt
+
+parallel --jobs 5 < workspace/runners/hyRefactor_calls.txt
+
+parallel --jobs 5 < workspace/runners/finalize_calls.txt

workspace/runners/upload_all.R

@@ -25,14 +25,13 @@ for(VPU in possible_vpu) {
     check_auth()
     message(nav_gpkg)
     sbtools::item_replace_files(sb_reference, nav_gpkg)
-    
-    check_auth()
 
+    check_auth()
     message(rfc_gpkg)
     sbtools::item_replace_files(sb_refactor, rfc_gpkg)
 
 
-    check_auth
+    check_auth()
     message(gf_gpkg)
     sbtools::item_replace_files(sb_gf, gf_gpkg)