From cdb04fb57a04ea9fa9ab326cde3fd314e4f851da Mon Sep 17 00:00:00 2001
From: David Blodgett <dblodgett@usgs.gov>
Date: Tue, 7 Jan 2025 10:04:34 -0600
Subject: [PATCH] get aggregate and non-dend working with single vpu
---
workspace/_targets/01_prep_targets.R | 4 ++-
workspace/_targets/03_refactor_targets.R | 2 +-
workspace/_targets/04_aggregate_targets.R | 2 +-
workspace/_targets_runner.R | 33 +++++++++++++++++++++++
4 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/workspace/_targets/01_prep_targets.R b/workspace/_targets/01_prep_targets.R
index ec98d9a..7d1eab4 100644
--- a/workspace/_targets/01_prep_targets.R
+++ b/workspace/_targets/01_prep_targets.R
@@ -72,7 +72,9 @@ list(
tar_target(vpu_codes, if(length(domain) == 1 && grepl("^[0-1]", domain))
unique(rpu_vpu$vpuid[grepl(domain, rpu_vpu$vpuid)]) else
unique(rpu_vpu$vpuid)),
- tar_target(all_rpu_codes, unique(rpu_vpu$rpuid)),
+ tar_target(rpu_codes, if(length(domain) == 1 && grepl("^[0-1]", domain))
+ unique(rpu_vpu$rpuid[grepl(domain, rpu_vpu$rpuid)]) else
+ unique(rpu_vpu$rpuid)),
tar_target(full_cat_file, data_paths$fullcats_table, format = "file"),
tar_target(full_cat_table, readRDS(full_cat_file)),
diff --git a/workspace/_targets/03_refactor_targets.R b/workspace/_targets/03_refactor_targets.R
index 6c459e4..8c334f5 100644
--- a/workspace/_targets/03_refactor_targets.R
+++ b/workspace/_targets/03_refactor_targets.R
@@ -16,7 +16,7 @@ prep_path <- ".targets_stores/01_prep/objects/"
list(tar_target(data_paths_file, "cache/data_paths.json", format = "file"),
tar_target(data_paths, jsonlite::read_json(data_paths_file)),
- tar_target(rpu_codes_file, file.path(prep_path, "all_rpu_codes"), format = "file"),
+ tar_target(rpu_codes_file, file.path(prep_path, "rpu_codes"), format = "file"),
tar_target(rpu_codes, readRDS(rpu_codes_file)),
tar_target(rpu_vpu_file, file.path(prep_path, "rpu_vpu"), format = "file"),
tar_target(rpu_vpu, readRDS(rpu_vpu_file)),
diff --git a/workspace/_targets/04_aggregate_targets.R b/workspace/_targets/04_aggregate_targets.R
index 6374f23..f20b439 100644
--- a/workspace/_targets/04_aggregate_targets.R
+++ b/workspace/_targets/04_aggregate_targets.R
@@ -15,7 +15,7 @@ source("R/04_aggregate_functions.R")
prep_path <- ".targets_stores/01_prep/objects/"
-list(tar_target(rpu_codes_file, file.path(prep_path, "all_rpu_codes"), format = "file"),
+list(tar_target(rpu_codes_file, file.path(prep_path, "rpu_codes"), format = "file"),
tar_target(rpu_codes, readRDS(rpu_codes_file)),
tar_target(rpu_gpkg, file.path("cache", paste0(rpu_codes, "_refactor.gpkg")),
pattern = map(rpu_codes), deployment = "worker"),
diff --git a/workspace/_targets_runner.R b/workspace/_targets_runner.R
index b5e17ad..594af6a 100644
--- a/workspace/_targets_runner.R
+++ b/workspace/_targets_runner.R
@@ -21,6 +21,8 @@ tar_make(gage_selection_map_report)
tar_make(vpu_codes)
tar_load(vpu_codes)
+tar_make(rpu_codes)
+
########## Debug help ################
if(FALSE) { # this won't run if you just bang through this file
# To debug a given vpu, first run all to get the dynamic branches established:
@@ -67,6 +69,7 @@ tar_make(vpu_gpkg)
# the POI creation workflow loops over these
tar_read(vpu_gpkg)
+# if running many, each can be run independently.
tar_make_future(huc12_poi, workers = workers)
tar_make_future(gage_pois, workers = workers)
tar_make_future(te_pois, workers = workers)
@@ -89,20 +92,50 @@ tar_make_future(poi_lookup, workers = workers)
tar_make_future(draft_segments, workers = workers)
tar_make_future(write_poi_, workers = workers)
+# Otherwise, just run all with
+tar_make()
+
Sys.setenv(TAR_PROJECT = "03_refactor")
+# iterate of "raster processing units"
+tar_make(rpu_gpkg)
+tar_make(vpu_gpkg) # need to track the vpu for each
+
+tar_read(rpu_gpkg)
+tar_read(vpu_gpkg)
+
workers = 6
+# now make all the rpus that we need
tar_make_future(workers = workers)
Sys.setenv(TAR_PROJECT = "04_aggregate")
+# iterate of "raster processing units"
+tar_make(rpu_gpkg)
+tar_make(agg_gpkg) # need to track the vpu for each
+
+tar_read(rpu_gpkg)
+tar_read(agg_gpkg) # output to "aggregate" rpu gpkgs
+
workers = 6
tar_make_future(workers = workers)
Sys.setenv(TAR_PROJECT = "05_non_dendritic")
+# now we combine raster units back to vector processing units
+tar_make(vpu_gpkg)
+tar_read(vpu_gpkg)
+
+# outputs will be written to a refactor and final "GF" gpkg
+tar_make(rfc_gpkg)
+tar_read(rfc_gpkg)
+tar_make(gf_gpkg)
+tar_read(gf_gpkg)
+
+tar_make()
+
workers = 6
tar_make_future(workers = workers)
--
GitLab