From 743b7f8322a2ab58ed5f3737ecc871926be7d381 Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Thu, 25 Mar 2021 09:55:06 -0600
Subject: [PATCH 1/5] i/o metadata for generate_matrix
---
geomagio/metadata/__init__.py | 10 ++++++--
geomagio/processing/adjusted.py | 44 ++++++++++++++++++++++++++++++---
2 files changed, 48 insertions(+), 6 deletions(-)
diff --git a/geomagio/metadata/__init__.py b/geomagio/metadata/__init__.py
index 592b2d159..4a348bb47 100644
--- a/geomagio/metadata/__init__.py
+++ b/geomagio/metadata/__init__.py
@@ -1,7 +1,13 @@
from .Metadata import Metadata
from .MetadataCategory import MetadataCategory
-from .MetadataFactory import MetadataFactory
+from .MetadataFactory import MetadataFactory, GEOMAG_API_URL
from .MetadataQuery import MetadataQuery
-__all__ = ["Metadata", "MetadataCategory", "MetadataFactory", "MetadataQuery"]
+__all__ = [
+ "GEOMAG_API_URL",
+ "Metadata",
+ "MetadataCategory",
+ "MetadataFactory",
+ "MetadataQuery",
+]
diff --git a/geomagio/processing/adjusted.py b/geomagio/processing/adjusted.py
index 97aa9629e..cdfaee9cf 100644
--- a/geomagio/processing/adjusted.py
+++ b/geomagio/processing/adjusted.py
@@ -4,7 +4,14 @@ from typing import Optional
import typer
from ..adjusted.Affine import Affine
-from ..residual import SpreadsheetSummaryFactory, WebAbsolutesFactory
+from ..residual import Reading, SpreadsheetSummaryFactory, WebAbsolutesFactory
+from ..metadata import (
+ GEOMAG_API_URL,
+ Metadata,
+ MetadataCategory,
+ MetadataFactory,
+ MetadataQuery,
+)
def main():
@@ -17,8 +24,10 @@ def generate_matrix(
endtime: str,
readings_starttime: str,
readings_endtime: str,
- output_file: str,
input_factory: str = "webabsolutes",
+ metadata_url: str = GEOMAG_API_URL,
+ output_file: Optional[str] = None,
+ output_metadata: bool = False,
spreadsheet_directory: Optional[str] = None,
webabsolutes_url: Optional[
str
@@ -38,6 +47,16 @@ def generate_matrix(
starttime=UTCDateTime(readings_starttime),
endtime=UTCDateTime(readings_endtime),
)
+ elif input_factory == "metadata":
+ metadata = MetadataFactory(url=metadata_url).get_metadata(
+ query=MetadataQuery(
+ station=observatory,
+ starttime=readings_starttime,
+ endtime=readings_endtime,
+ category=MetadataCategory.READING,
+ )
+ )
+ readings = [Reading(**m.metadata) for m in metadata]
else:
readings = []
@@ -48,5 +67,22 @@ def generate_matrix(
).calculate(readings=readings)[0]
result.matrix = result.matrix.tolist()
- with open(output_file, "w") as file:
- json.dump(result.dict(), file)
+ if output_metadata:
+ MetadataFactory(url=metadata_url).create_metadata(
+ metadata=Metadata(
+ station=observatory,
+ created_by="generate_matrix",
+ metadata=result.dict(),
+ starttime=starttime,
+ endtime=endtime,
+ network="NT",
+ category=MetadataCategory.ADJUSTED_MATRIX,
+ )
+ )
+
+ if output_file:
+ with open(output_file, "w") as file:
+ json.dump(result.dict(), file)
+
+ if not output_file and not output_metadata:
+ raise ValueError("Output method not provided")
--
GitLab
From b6d434f203c2afce93b93354735fd511063b72fd Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Thu, 25 Mar 2021 10:09:17 -0600
Subject: [PATCH 2/5] Ensure valid readings
---
geomagio/processing/adjusted.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/geomagio/processing/adjusted.py b/geomagio/processing/adjusted.py
index cdfaee9cf..9ab5b7e49 100644
--- a/geomagio/processing/adjusted.py
+++ b/geomagio/processing/adjusted.py
@@ -54,6 +54,8 @@ def generate_matrix(
starttime=readings_starttime,
endtime=readings_endtime,
category=MetadataCategory.READING,
+ data_valid=True,
+ metadata_valid=True,
)
)
readings = [Reading(**m.metadata) for m in metadata]
--
GitLab
From a6f9299e189cdc5dac00ced6945d8639fe07225e Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Fri, 26 Mar 2021 08:51:52 -0600
Subject: [PATCH 3/5] print result, save file as json, set metadata start/end
times with result
---
geomagio/processing/adjusted.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/geomagio/processing/adjusted.py b/geomagio/processing/adjusted.py
index 9ab5b7e49..2b7f748e6 100644
--- a/geomagio/processing/adjusted.py
+++ b/geomagio/processing/adjusted.py
@@ -75,8 +75,8 @@ def generate_matrix(
station=observatory,
created_by="generate_matrix",
metadata=result.dict(),
- starttime=starttime,
- endtime=endtime,
+ starttime=result.starttime,
+ endtime=result.endtime,
network="NT",
category=MetadataCategory.ADJUSTED_MATRIX,
)
@@ -84,7 +84,7 @@ def generate_matrix(
if output_file:
with open(output_file, "w") as file:
- json.dump(result.dict(), file)
+ file.write(result.json())
if not output_file and not output_metadata:
- raise ValueError("Output method not provided")
+ print(result.json(indent=2))
--
GitLab
From b08a989e7cdf4c71e99619cf698d1a988112b0c6 Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Fri, 26 Mar 2021 12:21:38 -0600
Subject: [PATCH 4/5] cast to list in adjusted package, store reading times in
comment
---
geomagio/adjusted/AdjustedMatrix.py | 2 +-
geomagio/adjusted/Affine.py | 2 +-
geomagio/algorithm/AdjustedAlgorithm.py | 4 ++--
geomagio/processing/adjusted.py | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/geomagio/adjusted/AdjustedMatrix.py b/geomagio/adjusted/AdjustedMatrix.py
index 44ca88a77..88e7169bf 100644
--- a/geomagio/adjusted/AdjustedMatrix.py
+++ b/geomagio/adjusted/AdjustedMatrix.py
@@ -22,7 +22,7 @@ class AdjustedMatrix(BaseModel):
Matrix is non-constrained otherwise
"""
- matrix: Optional[Any] = None
+ matrix: Optional[List[List[float]]] = None
pier_correction: float = 0
metrics: Optional[List[Metric]] = None
starttime: Optional[UTCDateTime] = None
diff --git a/geomagio/adjusted/Affine.py b/geomagio/adjusted/Affine.py
index 0a7a979c3..28e14c241 100644
--- a/geomagio/adjusted/Affine.py
+++ b/geomagio/adjusted/Affine.py
@@ -121,7 +121,7 @@ class Affine(BaseModel):
[reading.pier_correction for reading in readings], weights=weights
)
matrix = AdjustedMatrix(
- matrix=M_composed,
+ matrix=M_composed.tolist(),
pier_correction=pier_correction,
)
matrix.metrics = matrix.get_metrics(readings=readings)
diff --git a/geomagio/algorithm/AdjustedAlgorithm.py b/geomagio/algorithm/AdjustedAlgorithm.py
index d103e2d22..fcbe2dceb 100644
--- a/geomagio/algorithm/AdjustedAlgorithm.py
+++ b/geomagio/algorithm/AdjustedAlgorithm.py
@@ -45,7 +45,7 @@ class AdjustedAlgorithm(Algorithm):
"""
# Adjusted matrix defaults to identity matrix
matrix_size = len([c for c in self.get_input_channels() if c != "F"]) + 1
- matrix = np.eye(matrix_size)
+ matrix = np.eye(matrix_size).tolist()
if self.statefile is None:
self.matrix = AdjustedMatrix(matrix=matrix)
return
@@ -61,7 +61,7 @@ class AdjustedAlgorithm(Algorithm):
# read data from legacy format
for row in range(matrix_size):
for col in range(matrix_size):
- matrix[row, col] = np.float64(data[f"M{row+1}{col+1}"])
+ matrix[row][col] = np.float64(data[f"M{row+1}{col+1}"])
pier_correction = np.float64(data["PC"])
self.matrix = AdjustedMatrix(matrix=matrix, pier_correction=pier_correction)
else:
diff --git a/geomagio/processing/adjusted.py b/geomagio/processing/adjusted.py
index 2b7f748e6..66b7e8701 100644
--- a/geomagio/processing/adjusted.py
+++ b/geomagio/processing/adjusted.py
@@ -67,7 +67,6 @@ def generate_matrix(
starttime=UTCDateTime(starttime),
endtime=UTCDateTime(endtime),
).calculate(readings=readings)[0]
- result.matrix = result.matrix.tolist()
if output_metadata:
MetadataFactory(url=metadata_url).create_metadata(
@@ -79,6 +78,7 @@ def generate_matrix(
endtime=result.endtime,
network="NT",
category=MetadataCategory.ADJUSTED_MATRIX,
+ comment=f"calculated from {readings_starttime} to {readings_endtime}",
)
)
--
GitLab
From a686fc91ef612b51e98078f6d16aa66fb623f556 Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Tue, 30 Mar 2021 12:56:21 -0600
Subject: [PATCH 5/5] Add quiet option, enumerate factories, set update
interval to none
---
geomagio/processing/adjusted.py | 47 +++++++++++++++++++--------------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/geomagio/processing/adjusted.py b/geomagio/processing/adjusted.py
index 66b7e8701..a3e8000d6 100644
--- a/geomagio/processing/adjusted.py
+++ b/geomagio/processing/adjusted.py
@@ -1,4 +1,5 @@
-import json
+from enum import Enum
+
from obspy import UTCDateTime
from typing import Optional
import typer
@@ -14,6 +15,12 @@ from ..metadata import (
)
+class InputFactory(str, Enum):
+ METADATA = "metadata"
+ SPREADSHEET = "spreadsheet"
+ WEBABSOLUTES = "webabsolutes"
+
+
def main():
typer.run(generate_matrix)
@@ -24,7 +31,7 @@ def generate_matrix(
endtime: str,
readings_starttime: str,
readings_endtime: str,
- input_factory: str = "webabsolutes",
+ input_factory: InputFactory = InputFactory.WEBABSOLUTES,
metadata_url: str = GEOMAG_API_URL,
output_file: Optional[str] = None,
output_metadata: bool = False,
@@ -32,8 +39,21 @@ def generate_matrix(
webabsolutes_url: Optional[
str
] = "https://geomag.usgs.gov/baselines/observation.json.php",
+ quiet: bool = False,
):
- if input_factory == "spreadsheet":
+ if input_factory == InputFactory.METADATA:
+ metadata = MetadataFactory(url=metadata_url).get_metadata(
+ query=MetadataQuery(
+ station=observatory,
+ starttime=readings_starttime,
+ endtime=readings_endtime,
+ category=MetadataCategory.READING,
+ data_valid=True,
+ metadata_valid=True,
+ )
+ )
+ readings = [Reading(**m.metadata) for m in metadata]
+ elif input_factory == InputFactory.SPREADSHEET:
readings = SpreadsheetSummaryFactory(
base_directory=spreadsheet_directory
).get_readings(
@@ -41,31 +61,18 @@ def generate_matrix(
starttime=UTCDateTime(readings_starttime),
endtime=UTCDateTime(readings_endtime),
)
- elif input_factory == "webabsolutes":
+ elif input_factory == InputFactory.WEBABSOLUTES:
readings = WebAbsolutesFactory(url=webabsolutes_url).get_readings(
observatory=observatory,
starttime=UTCDateTime(readings_starttime),
endtime=UTCDateTime(readings_endtime),
)
- elif input_factory == "metadata":
- metadata = MetadataFactory(url=metadata_url).get_metadata(
- query=MetadataQuery(
- station=observatory,
- starttime=readings_starttime,
- endtime=readings_endtime,
- category=MetadataCategory.READING,
- data_valid=True,
- metadata_valid=True,
- )
- )
- readings = [Reading(**m.metadata) for m in metadata]
- else:
- readings = []
-
+ # calculate one affine matrix between starttime and endtime
result = Affine(
observatory=observatory,
starttime=UTCDateTime(starttime),
endtime=UTCDateTime(endtime),
+ update_interval=None,
).calculate(readings=readings)[0]
if output_metadata:
@@ -86,5 +93,5 @@ def generate_matrix(
with open(output_file, "w") as file:
file.write(result.json())
- if not output_file and not output_metadata:
+ if not quiet:
print(result.json(indent=2))
--
GitLab