From 8ff10143d3cadd25617b2e396c7dc1bfd3d33d57 Mon Sep 17 00:00:00 2001
From: Peter Powers <pmpowers@usgs.gov>
Date: Fri, 18 Feb 2022 10:27:09 -0700
Subject: [PATCH 1/2] increment lib
---
gradle.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gradle.properties b/gradle.properties
index c0b442fc7..bc1f8a0e2 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -10,7 +10,7 @@ micronautRxVersion = 2.1.1
micronautPluginVersion = 3.1.1
nodePluginVersion = 3.0.1
nodeVersion = 16.3.0
-nshmpLibVersion = 0.9.9
+nshmpLibVersion = 0.9.10
nshmpWsUtilsVersion = 0.1.7
shadowVersion = 7.1.2
spotbugsVersion = 4.7.0
--
GitLab
From b2a07f60109e8d0023a872d5319edaed75cc73ed Mon Sep 17 00:00:00 2001
From: Peter Powers <pmpowers@usgs.gov>
Date: Fri, 18 Feb 2022 13:47:59 -0700
Subject: [PATCH 2/2] program output dir updates
---
.../gov/usgs/earthquake/nshmp/DisaggCalc.java | 50 ++-----
.../gov/usgs/earthquake/nshmp/HazardCalc.java | 136 ++++--------------
.../gov/usgs/earthquake/nshmp/RateCalc.java | 25 ++--
3 files changed, 56 insertions(+), 155 deletions(-)
diff --git a/src/main/java/gov/usgs/earthquake/nshmp/DisaggCalc.java b/src/main/java/gov/usgs/earthquake/nshmp/DisaggCalc.java
index 569ec1a3d..b54668199 100644
--- a/src/main/java/gov/usgs/earthquake/nshmp/DisaggCalc.java
+++ b/src/main/java/gov/usgs/earthquake/nshmp/DisaggCalc.java
@@ -170,20 +170,21 @@ public class DisaggCalc {
Set<Imt> modelImts = model.config().hazard.imts;
+ Path out = HazardCalc.createOutputDir(config.output.directory);
+
/*
* If no IML columns present, disaggregate at IMTs and return period from
* config, otherwise disaggregate at target IMLs are present.
*
- * We've removed support for gejson site files at present.
+ * We've removed support for geojson site files at present.
*/
- Path out;
if (siteColumns.size() == allColumns.size()) {
checkArgument(
modelImts.containsAll(config.hazard.imts),
"Config specifies IMTs not supported by model");
double returnPeriod = config.disagg.returnPeriod;
- out = calcRp(model, config, sites, returnPeriod, log);
+ calcRp(model, config, sites, returnPeriod, out, log);
} else {
@@ -196,7 +197,7 @@ public class DisaggCalc {
sites.size() == imls.size(),
"Sites and spectra lists different sizes");
log.info("Spectra: " + imls.size()); // 1:1 with sites
- out = calcIml(model, config, sites, imls, log);
+ calcIml(model, config, sites, imls, out, log);
}
@@ -266,15 +267,13 @@ public class DisaggCalc {
return imtImlMap;
}
- /*
- * Compute hazard curves using the supplied model, config, and sites. Method
- * returns the path to the directory where results were written.
- */
- private static Path calcRp(
+ /* Compute hazard curves using the supplied model, config, and sites. */
+ private static void calcRp(
HazardModel model,
CalcConfig config,
List<Site> sites,
double returnPeriod,
+ Path out,
Logger log) throws IOException {
ExecutorService exec = null;
@@ -289,8 +288,8 @@ public class DisaggCalc {
log.info(PROGRAM + " (return period): calculating ...");
- HazardExport handler = HazardExport.create(model, config, sites, OptionalDouble.empty());
- Path disaggDir = handler.outputDir().resolve("disagg");
+ HazardExport handler = HazardExport.create(model, config, sites, out);
+ Path disaggDir = out.resolve("disagg");
Files.createDirectory(disaggDir);
Stopwatch stopwatch = Stopwatch.createStarted();
@@ -326,14 +325,10 @@ public class DisaggCalc {
count, sites.size(), stopwatch));
}
}
- handler.expire();
-
log.info(String.format(
PROGRAM + " (return period): %s sites completed in %s",
- sites.size(), stopwatch.stop()));
-
+ sites.size(), stopwatch));
exec.shutdown();
- return handler.outputDir();
}
/* Hazard curves are already in log-x space. */
@@ -360,11 +355,12 @@ public class DisaggCalc {
* Compute hazard curves using the supplied model, config, and sites. Method
* returns the path to the directory where results were written.
*/
- private static Path calcIml(
+ private static void calcIml(
HazardModel model,
CalcConfig config,
List<Site> sites,
List<Map<Imt, Double>> imls,
+ Path out,
Logger log) throws IOException {
ExecutorService exec = null;
@@ -379,8 +375,7 @@ public class DisaggCalc {
log.info(PROGRAM + " (IML): calculating ...");
- Path outDir = createOutputDir(config.output.directory);
- Path disaggDir = outDir.resolve("disagg");
+ Path disaggDir = out.resolve("disagg");
Files.createDirectory(disaggDir);
Stopwatch stopwatch = Stopwatch.createStarted();
@@ -414,13 +409,10 @@ public class DisaggCalc {
count, sites.size(), stopwatch));
}
}
-
log.info(String.format(
PROGRAM + " (IML): %s sites completed in %s",
- sites.size(), stopwatch.stop()));
-
+ sites.size(), stopwatch));
exec.shutdown();
- return outDir;
}
private static final class Response {
@@ -519,18 +511,6 @@ public class DisaggCalc {
}
}
- // duplicate of that in HazardExport
- private static Path createOutputDir(Path dir) throws IOException {
- int i = 1;
- Path incrementedDir = dir;
- while (Files.exists(incrementedDir)) {
- incrementedDir = incrementedDir.resolveSibling(dir.getFileName() + "-" + i);
- i++;
- }
- Files.createDirectories(incrementedDir);
- return incrementedDir;
- }
-
private static String disaggFilename(Site site) {
return site.name().equals(Site.NO_NAME)
? String.format(
diff --git a/src/main/java/gov/usgs/earthquake/nshmp/HazardCalc.java b/src/main/java/gov/usgs/earthquake/nshmp/HazardCalc.java
index 4040521be..f9fc796fb 100644
--- a/src/main/java/gov/usgs/earthquake/nshmp/HazardCalc.java
+++ b/src/main/java/gov/usgs/earthquake/nshmp/HazardCalc.java
@@ -12,12 +12,9 @@ import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.OptionalDouble;
-import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.logging.FileHandler;
import java.util.logging.Logger;
@@ -111,15 +108,15 @@ public class HazardCalc {
.build();
}
log.info(config.toString());
-
log.info("");
- Path out = null;
+ Path out = createOutputDir(config.output.directory);
+
if (config.hazard.vs30s.isEmpty()) {
List<Site> sites = readSites(args[1], model.siteData(), OptionalDouble.empty(), log);
log.info("Sites: " + Sites.toString(sites));
- out = calc(model, config, sites, OptionalDouble.empty(), log);
+ calc(model, config, sites, out, log);
} else {
@@ -127,9 +124,10 @@ public class HazardCalc {
log.info("Vs30 batch: " + vs30);
List<Site> sites = readSites(args[1], model.siteData(), OptionalDouble.of(vs30), log);
log.info("Sites: " + Sites.toString(sites));
- out = calc(model, config, sites, OptionalDouble.of(vs30), log);
+ Path vs30dir = out.resolve("vs30-" + ((int) vs30));
+ Files.createDirectory(vs30dir);
+ calc(model, config, sites, vs30dir, log);
}
- out = checkNotNull(out.getParent());
}
@@ -152,16 +150,16 @@ public class HazardCalc {
}
static List<Site> readSites(
- String arg,
+ String siteFile,
SiteData siteData,
OptionalDouble vs30,
Logger log) {
- Path path = Paths.get(arg);
+ Path path = Paths.get(siteFile);
log.info("Sites file: " + path.toAbsolutePath().normalize());
- String fname = arg.toLowerCase();
+ String fname = siteFile.toLowerCase();
checkArgument(fname.endsWith(".csv") || fname.endsWith(".geojson"),
- "Sites file [%s] must be a path to a *.csv or *.geojson file", arg);
+ "Sites file [%s] must be a path to a *.csv or *.geojson file", siteFile);
try {
return fname.endsWith(".csv")
@@ -173,15 +171,12 @@ public class HazardCalc {
}
}
- /*
- * Compute hazard curves using the supplied model, config, and sites. Method
- * returns the path to the directory where results were written.
- */
- private static Path calc(
+ /* Compute hazard curves using the supplied model, config, and sites. */
+ private static void calc(
HazardModel model,
CalcConfig config,
List<Site> sites,
- OptionalDouble vs30,
+ Path out,
Logger log) throws IOException, InterruptedException, ExecutionException {
int threadCount = config.performance.threadCount.value();
@@ -189,18 +184,14 @@ public class HazardCalc {
log.info("Threads: " + ((ThreadPoolExecutor) exec).getCorePoolSize());
log.info(PROGRAM + ": calculating ...");
- HazardExport handler = HazardExport.create(model, config, sites, vs30);
- CalcTask.Builder calcTask = new CalcTask.Builder(model, config, exec);
- WriteTask.Builder writeTask = new WriteTask.Builder(handler);
-
+ HazardExport handler = HazardExport.create(model, config, sites, out);
Stopwatch stopwatch = Stopwatch.createStarted();
int logInterval = sites.size() < 100 ? 1 : sites.size() < 1000 ? 10 : 100;
- Future<Path> out = null;
for (int i = 0; i < sites.size(); i++) {
Site site = sites.get(i);
- Hazard hazard = calcTask.withSite(site).call();
- out = exec.submit(writeTask.withResult(hazard));
+ Hazard hazard = HazardCalcs.hazard(model, config, site, exec);
+ handler.write(hazard);
int count = i + 1;
if (count % logInterval == 0) {
log.info(String.format(
@@ -208,16 +199,22 @@ public class HazardCalc {
count, sites.size(), stopwatch));
}
}
- /* Block shutdown until last task is returned. */
- Path outputDir = out.get();
-
- handler.expire();
exec.shutdown();
log.info(String.format(
PROGRAM + ": %s sites completed in %s",
- handler.resultCount(), stopwatch.stop()));
+ sites.size(), stopwatch));
+ }
- return outputDir;
+ /* Avoid clobbering exsting result directories via incrementing. */
+ static Path createOutputDir(Path dir) throws IOException {
+ int i = 1;
+ Path outDir = dir;
+ while (Files.exists(outDir)) {
+ outDir = outDir.resolveSibling(dir.getFileName() + "-" + i);
+ i++;
+ }
+ Files.createDirectories(outDir);
+ return outDir;
}
private static ExecutorService initExecutor(int threadCount) {
@@ -228,83 +225,6 @@ public class HazardCalc {
}
}
- private static final class CalcTask implements
- Callable<Hazard> {
-
- final HazardModel model;
- final CalcConfig config;
- final Site site;
- final Executor exec;
-
- CalcTask(
- HazardModel model,
- CalcConfig config,
- Site site,
- Executor exec) {
-
- this.model = model;
- this.config = config;
- this.site = site;
- this.exec = exec;
- }
-
- @Override
- public Hazard call() {
- return HazardCalcs.hazard(model, config, site, exec);
- }
-
- static class Builder {
-
- final HazardModel model;
- final CalcConfig config;
- final Executor exec;
-
- Builder(HazardModel model, CalcConfig config, Executor exec) {
- this.model = model;
- this.config = config;
- this.exec = exec;
- }
-
- /* Builds and returns the task. */
- CalcTask withSite(Site site) {
- return new CalcTask(model, config, site, exec);
- }
- }
- }
-
- private static final class WriteTask implements Callable<Path> {
-
- final HazardExport handler;
- final Hazard hazard;
-
- WriteTask(
- HazardExport handler,
- Hazard hazard) {
- this.handler = handler;
- this.hazard = hazard;
- }
-
- @Override
- public Path call() throws IOException {
- handler.write(hazard);
- return handler.outputDir();
- }
-
- static class Builder {
-
- final HazardExport handler;
-
- Builder(HazardExport handler) {
- this.handler = handler;
- }
-
- /* Builds and returns the task. */
- WriteTask withResult(Hazard hazard) {
- return new WriteTask(handler, hazard);
- }
- }
- }
-
static final String TMP_LOG = "nshmp-haz-log";
static Path createTempLog() {
diff --git a/src/main/java/gov/usgs/earthquake/nshmp/RateCalc.java b/src/main/java/gov/usgs/earthquake/nshmp/RateCalc.java
index d0febbdef..097d10b1b 100644
--- a/src/main/java/gov/usgs/earthquake/nshmp/RateCalc.java
+++ b/src/main/java/gov/usgs/earthquake/nshmp/RateCalc.java
@@ -19,6 +19,7 @@ import java.util.logging.FileHandler;
import java.util.logging.Logger;
import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
@@ -106,13 +107,15 @@ public class RateCalc {
.build();
}
log.info(config.toString());
-
log.info("");
+
+ Path out = HazardCalc.createOutputDir(config.output.directory);
+
List<Site> sites = HazardCalc.readSites(
args[1], model.siteData(), OptionalDouble.empty(), log);
log.info("Sites: " + Sites.toString(sites));
- Path out = calc(model, config, sites, log);
+ calc(model, config, sites, out, log);
log.info(PROGRAM + ": finished");
/* Transfer log and write config, windows requires fh.close() */
@@ -136,12 +139,14 @@ public class RateCalc {
* a single calculation, rate calculations are single threaded. Concurrent
* calculations for multiple sites are handled below.
*/
- private static Path calc(
+ private static void calc(
HazardModel model,
CalcConfig config,
List<Site> sites,
+ Path out,
Logger log) throws IOException, ExecutionException, InterruptedException {
+ Stopwatch stopwatch = Stopwatch.createStarted();
ThreadCount threadCount = config.performance.threadCount;
EqRateExport export = null;
if (threadCount != ThreadCount.ONE) {
@@ -149,35 +154,31 @@ public class RateCalc {
ListeningExecutorService executor = MoreExecutors.listeningDecorator(poolExecutor);
log.info("Threads: " + ((ThreadPoolExecutor) poolExecutor).getCorePoolSize());
log.info(PROGRAM + ": calculating ...");
- export = concurrentCalc(model, config, sites, log, executor);
+ export = concurrentCalc(model, config, sites, out, executor);
executor.shutdown();
} else {
log.info("Threads: Running on calling thread");
log.info(PROGRAM + ": calculating ...");
- export = EqRateExport.create(model, config, sites, log);
+ export = EqRateExport.create(model, config, sites, out);
for (Site site : sites) {
EqRate rate = EqRate.create(model, config, site);
export.write(rate);
}
}
- export.expire();
-
log.info(String.format(
PROGRAM + ": %s sites completed in %s",
- export.resultCount(), export.elapsedTime()));
-
- return export.outputDir();
+ sites.size(), stopwatch));
}
private static EqRateExport concurrentCalc(
HazardModel model,
CalcConfig config,
List<Site> sites,
- Logger log,
+ Path out,
ListeningExecutorService executor)
throws InterruptedException, ExecutionException, IOException {
- EqRateExport export = EqRateExport.create(model, config, sites, log);
+ EqRateExport export = EqRateExport.create(model, config, sites, out);
int submitted = 0;
int batchSize = 10;
--
GitLab