From a9dd62f5f8e2894280ef240acb6a30a458eb1eca Mon Sep 17 00:00:00 2001
From: Alex Wernle <awernle@usgs.gov>
Date: Thu, 11 Aug 2022 15:14:19 -0600
Subject: [PATCH] Broke up code into smaller functions, changed
starttime/endtime to be defined by first absolutes reading starttime and last
absolutes reading endtime
---
geomagio/processing/absolutes.py | 82 -------------
geomagio/processing/copy_absolutes.py | 166 ++++++++++++++++++++++++++
pyproject.toml | 1 +
3 files changed, 167 insertions(+), 82 deletions(-)
delete mode 100644 geomagio/processing/absolutes.py
create mode 100644 geomagio/processing/copy_absolutes.py
diff --git a/geomagio/processing/absolutes.py b/geomagio/processing/absolutes.py
deleted file mode 100644
index 714482ece..000000000
--- a/geomagio/processing/absolutes.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on Thu Jul 7 11:05:19 2022
-
-@author: awernle
-"""
-import json
-import typer
-import sys
-from obspy import UTCDateTime
-from datetime import datetime, timedelta
-from geomagio.api.db import database, MetadataDatabaseFactory
-from geomagio.metadata import Metadata, MetadataCategory
-from geomagio.residual import WebAbsolutesFactory
-
-test_metadata = []
-
-app = typer.Typer()
-
-
-@app.command(
- help=f"""
- Get existing readings from web absolutes.
-
- observatory (required): Input 3-letter observatory ID
-
- starttime (option): Input yyyy-mm-dd, default is 24 hours before current time
-
- endtime (option): Input yyyy-mm-dd,, default is current time
-
- """
-)
-def get_readings(observatory: str, starttime: str = None, endtime: str = None):
- if starttime is None:
- starttime = UTCDateTime((datetime.today() - timedelta(days=1)))
- else:
- starttime = UTCDateTime(starttime)
- if endtime is None:
- endtime = UTCDateTime((datetime.today()))
- else:
- endtime = UTCDateTime(endtime)
- readings = WebAbsolutesFactory().get_readings(observatory, starttime, endtime)
-
- if not readings:
- print("No readings available for this period")
- sys.exit()
- else:
- print("Readings uploaded")
-
- for reading in readings:
- json_string = reading.json()
- reading_dict = json.loads(json_string)
- try:
- reviewer = reading.metadata["reviewer"]
- except KeyError:
- reviewer = None
- test_metadata.append(
- Metadata(
- category=MetadataCategory.READING,
- created_by="absolutes_test.py",
- network="NT",
- updated_by=reviewer,
- starttime=reading.time,
- endtime=reading.time,
- station=reading.metadata["station"],
- metadata=reading_dict,
- )
- )
-
-
-async def load_test_metadata():
- await database.connect()
- for meta in test_metadata:
- await MetadataDatabaseFactory(database=database).create_metadata(meta)
- await database.disconnect()
-
-
-if __name__ == "__main__":
- import asyncio
-
- app()
- asyncio.run(load_test_metadata())
diff --git a/geomagio/processing/copy_absolutes.py b/geomagio/processing/copy_absolutes.py
new file mode 100644
index 000000000..27d81eed3
--- /dev/null
+++ b/geomagio/processing/copy_absolutes.py
@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jul 7 11:05:19 2022
+
+@author: awernle
+"""
+import os
+import json
+from datetime import date, datetime, time, timedelta
+
+# from types import NoneType
+from typing import List
+
+import typer
+from obspy import UTCDateTime
+
+"""
+from ..metadata import Metadata, MetadataFactory, MetadataCategory
+from ..residual import Reading, WebAbsolutesFactory
+"""
+from geomagio.metadata import Metadata, MetadataFactory, MetadataCategory
+from geomagio.residual import Reading, WebAbsolutesFactory
+
+TODAY = datetime.combine(date.today(), time(0, 0, 0))
+NoneType = type(None)
+
+
+def copy_absolutes(
+ observatory: str = typer.Option(..., help="Observatory code"),
+ starttime: datetime = typer.Option(
+ default=TODAY - timedelta(days=1), help="Search start time, default "
+ ),
+ endtime: datetime = typer.Option(default=TODAY),
+ metadata_token: str = typer.Option(
+ default=os.getenv("GITLAB_API_TOKEN"),
+ help="Token for metadata web service.",
+ metavar="TOKEN",
+ show_default="environment variable GITLAB_API_TOKEN",
+ ),
+ metadata_url: str = typer.Option(
+ default="https://geomag.usgs.gov/ws/secure", # insert staging
+ help="URL to metadata web service",
+ metavar="URL",
+ ),
+ web_absolutes_url: str = typer.Option(
+ default="https://geomag.usgs.gov/baselines/observation.json.php",
+ help="URL to web absolutes service",
+ metavar="URL",
+ ),
+) -> List[Metadata]: # make this nonetype?
+ """Copy absolutes from the web absolutes service into the metadata service."""
+ # read readings from web absolutes
+ web_absolutes_factory = WebAbsolutesFactory(url=web_absolutes_url)
+ readings = get_readings(
+ factory=web_absolutes_factory,
+ observatory=observatory,
+ starttime=UTCDateTime(starttime),
+ endtime=UTCDateTime(endtime),
+ )
+ typer.echo(f"Found {len(readings)} absolutes")
+ # write readings to metadata service
+ metadata_factory = MetadataFactory(token=metadata_token, url=metadata_url)
+ with typer.progressbar(
+ iterable=readings, label="Uploading to metadata service"
+ ) as progressbar:
+ for reading in progressbar:
+ upload_reading(factory=metadata_factory, reading=reading)
+
+
+def create_reading_metadata(reading: Reading) -> Metadata:
+ """Create reading metadata object.
+
+ Parameters
+ ----------
+ reading:
+ reading to convert to metadata.
+
+ Returns
+ -------
+ metadata object for reading
+ """
+ json_string = reading.json()
+ reading_dict = json.loads(json_string)
+ metadata = []
+ try:
+ reviewer = reading.metadata["reviewer"]
+ except KeyError:
+ reviewer = None
+ metadata.append(
+ Metadata(
+ category=MetadataCategory.READING,
+ created_by="absolutes_test.py",
+ network="NT",
+ updated_by=reviewer,
+ starttime=reading.time,
+ endtime=reading.time,
+ station=reading.metadata["station"],
+ metadata=reading_dict,
+ )
+ )
+ return metadata
+ # TODO: should reading starttime/endtime be for range of measurement times?
+ pass
+
+
+def get_readings(
+ factory: WebAbsolutesFactory,
+ observatory: str,
+ starttime: UTCDateTime,
+ endtime: UTCDateTime,
+) -> List[Reading]:
+ """Get readings from web absolutes.
+
+ Parameters
+ ----------
+ factory
+ configured WebAbsolutesFactory
+ observatory
+ search observatory
+ starttime
+ search start time
+ endtime
+ search end time
+
+ Returns
+ -------
+ list of found readings
+ """
+ readings = factory.get_readings(
+ observatory=observatory,
+ starttime=UTCDateTime(starttime),
+ endtime=UTCDateTime(endtime),
+ include_measurements=True,
+ )
+ return readings
+
+
+def upload_reading(factory: MetadataFactory, reading: Reading) -> Metadata:
+ """Upload reading to metadata service
+
+ Parameters
+ ----------
+ factory
+ factory configured for metadata service
+ reading
+ reading to upload
+
+ Returns
+ -------
+ created metadata object
+ """
+ metadata = create_reading_metadata(reading=reading)
+ # TODO: should this check if metadata was already uploaded?
+ # TODO: should that check occur before calling this method?
+ for meta in metadata:
+ return factory.create_metadata(metadata=meta)
+
+
+def main() -> None:
+ """Entrypoint for copy absolutes method."""
+ # for one command, can use typer.run
+ typer.run(copy_absolutes)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pyproject.toml b/pyproject.toml
index 2a5999441..cf5d931e0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -71,3 +71,4 @@ geomag-py = "geomagio.Controller:main"
magproc-prepfiles = "geomagio.processing.magproc:main"
make-cal = "geomagio.processing.make_cal:main"
geomag-filter = "geomagio.processing.filters:main"
+copy-absolutes = "geomagio.processing.copy_absolutes:main"
\ No newline at end of file
--
GitLab