Merge branch 'support-miniseed' into 'master'

webservice and obsrio enhancements

See merge request !66

geomagio/api/db/metadata_table.py

 from datetime import datetime
-import enum
 
-from obspy import UTCDateTime
 from sqlalchemy import or_, Boolean, Column, Index, Integer, JSON, String, Table, Text
 import sqlalchemy_utc
 

geomagio/api/db/session_table.py

 from datetime import datetime, timedelta, timezone
-import json
-from typing import Dict, Optional
 
 import sqlalchemy
 import sqlalchemy_utc

geomagio/api/secure/MetadataQuery.py

 from datetime import timezone
 
 from obspy import UTCDateTime
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
 
 from ...metadata import MetadataCategory
 from ... import pydantic_utcdatetime

geomagio/api/secure/SessionMiddleware.py

-import base64
 import json
 from typing import Callable, Dict, Mapping
 import uuid
 
 from cryptography.fernet import Fernet
-from starlette.datastructures import MutableHeaders, Secret
+from starlette.datastructures import MutableHeaders
 from starlette.requests import HTTPConnection
 from starlette.types import ASGIApp, Message, Receive, Scope, Send
 

geomagio/api/ws/DataApiQuery.py

 import datetime
 import enum
-from typing import Any, Dict, List, Optional, Union
+import os
+from typing import Dict, List, Optional, Union
 
 from obspy import UTCDateTime
 from pydantic import BaseModel, root_validator, validator
 
 from ... import pydantic_utcdatetime
-from .Element import ELEMENTS, ELEMENT_INDEX
+from .Element import ELEMENTS
 from .Observatory import OBSERVATORY_INDEX
 
 
@@ -38,6 +39,13 @@ class SamplingPeriod(float, enum.Enum):
     HOUR = 3600.0
     DAY = 86400.0
 
+    @property
+    def input_factory(self):
+        if self in [SamplingPeriod.TEN_HERTZ, SamplingPeriod.HOUR, SamplingPeriod.DAY]:
+            return "miniseed"
+        else:
+            return "edge"
+
 
 class DataApiQuery(BaseModel):
     id: str

geomagio/api/ws/Element.py

@@ -10,6 +10,9 @@ class Element(BaseModel):
 
 
 ELEMENTS = [
+    Element(id="U", name="North Component(miniseed)", units="nT"),
+    Element(id="V", name="East Component(miniseed)", units="nT"),
+    Element(id="W", name="Vertical Component(miniseed)", units="nT"),
     Element(id="H", name="North Component", units="nT"),
     Element(id="E", name="East Component", units="nT"),
     Element(id="X", name="Geographic North Magnitude", units="nT"),

geomagio/api/ws/algorithms.py

@@ -15,7 +15,7 @@ def get_dbdt(
     query: DataApiQuery = Depends(get_data_query),
     data_factory: TimeseriesFactory = Depends(get_data_factory),
 ) -> Response:
-    dbdt = DbDtAlgorithm()
+    dbdt = DbDtAlgorithm(period=query.sampling_period)
     # read data
     raw = get_timeseries(data_factory, query)
     # run dbdt

geomagio/api/ws/data.py

 import os
-from typing import Any, Dict, List, Union
+from typing import List, Union
 
 from fastapi import APIRouter, Depends, Query
 from obspy import UTCDateTime, Stream
 from starlette.responses import Response
 
 from ... import TimeseriesFactory, TimeseriesUtility
-from ...edge import EdgeFactory
+from ...edge import EdgeFactory, MiniSeedFactory
 from ...iaga2002 import IAGA2002Writer
 from ...imfjson import IMFJSONWriter
 from .DataApiQuery import (
@@ -18,7 +18,9 @@ from .DataApiQuery import (
 )
 
 
-def get_data_factory() -> TimeseriesFactory:
+def get_data_factory(
+    query: DataApiQuery,
+) -> TimeseriesFactory:
     """Reads environment variable to determine the factory to be used
 
     Returns
@@ -26,11 +28,12 @@ def get_data_factory() -> TimeseriesFactory:
     data_factory
         Edge or miniseed factory object
     """
-    data_type = os.getenv("DATA_TYPE", "edge")
-    data_host = os.getenv("DATA_HOST", "cwbpub.cr.usgs.gov")
-    data_port = int(os.getenv("DATA_PORT", "2060"))
-    if data_type == "edge":
-        return EdgeFactory(host=data_host, port=data_port)
+    host = os.getenv("DATA_HOST", "cwbpub.cr.usgs.gov")
+    factory = query.sampling_period.input_factory
+    if factory == "edge":
+        return EdgeFactory(host=host, port=os.getenv("DATA_EARTHWORM_PORT", "2060"))
+    elif factory == "miniseed":
+        return MiniSeedFactory(host=host, port=os.getenv("DATA_MINISEED_PORT", "2061"))
     else:
         return None
 
@@ -150,8 +153,8 @@ router = APIRouter()
 @router.get("/data/")
 def get_data(
     query: DataApiQuery = Depends(get_data_query),
-    data_factory: TimeseriesFactory = Depends(get_data_factory),
 ) -> Response:
+    data_factory = get_data_factory(query=query)
     # read data
     timeseries = get_timeseries(data_factory, query)
     # output response

geomagio/api/ws/metadata.py

 from typing import List
 
-from fastapi import APIRouter, Body, Response
+from fastapi import APIRouter
 from obspy import UTCDateTime
 
 from ...metadata import Metadata, MetadataCategory

geomagio/processing/factory.py

 import os
-from typing import Callable
 
 from ..TimeseriesFactory import TimeseriesFactory
 from ..edge import EdgeFactory, MiniSeedFactory

geomagio/processing/magproc.py

 from datetime import datetime
 import os
 import sys
-from typing import List, Tuple
+from typing import List
 
 from dateutil.relativedelta import relativedelta
 from obspy.core import UTCDateTime, Stream

geomagio/processing/observatory.py

-import os
 from typing import List, Optional
 
 import numpy
 
 from ..algorithm import (
-    Algorithm,
     AdjustedAlgorithm,
     AverageAlgorithm,
     DeltaFAlgorithm,
     SqDistAlgorithm,
     XYZAlgorithm,
 )
-from ..edge import EdgeFactory, MiniSeedFactory
 from ..Controller import Controller, get_realtime_interval
 from ..TimeseriesFactory import TimeseriesFactory
 from .factory import get_edge_factory

geomagio/processing/obsrio.py

-import os
 from typing import Optional
 
 import typer
@@ -14,7 +13,60 @@ from .factory import get_edge_factory, get_miniseed_factory
 
 
 def main():
-    typer.run(filter_realtime)
+    typer.run(obsrio_filter)
+
+
+def obsrio_filter(
+    interval: str,
+    observatory: str,
+    input_factory: Optional[str] = None,
+    host: str = "127.0.0.1",
+    port: str = 2061,
+    output_factory: Optional[str] = None,
+    output_port: int = typer.Option(
+        2061, help="Port where output factory writes data."
+    ),
+    output_read_port: int = typer.Option(
+        2061, help="Port where output factory reads data"
+    ),
+    realtime_interval: int = 600,
+    update_limit: int = 10,
+):
+    if interval == "realtime":
+        filter_realtime(
+            observatory=observatory,
+            input_factory=input_factory,
+            host=host,
+            port=port,
+            output_factory=output_factory,
+            output_port=output_port,
+            output_read_port=output_read_port,
+            realtime_interval=realtime_interval,
+            update_limit=update_limit,
+        )
+    elif interval in ["hour", "day"]:
+        input_factory = EdgeFactory(host=host, port=port)
+        output_factory = MiniSeedFactory(
+            host=host, port=output_read_port, write_port=output_port
+        )
+        if interval == "hour":
+            obsrio_hour(
+                observatory=observatory,
+                input_factory=input_factory,
+                output_factory=output_factory,
+                realtime_interval=realtime_interval,
+                update_limit=update_limit,
+            )
+        elif interval == "days":
+            obsrio_day(
+                observatory=observatory,
+                input_factory=input_factory,
+                output_factory=output_factory,
+                realtime_interval=realtime_interval,
+                update_limit=update_limit,
+            )
+    else:
+        raise ValueError("Invalid interval")
 
 
 def filter_realtime(
@@ -34,12 +86,10 @@ def filter_realtime(
 ):
     """Filter 10Hz miniseed, 1 second, one minute, and temperature data.
     Defaults set for realtime processing; can also be implemented to update legacy data"""
-
     if input_factory == "miniseed":
         input_factory = MiniSeedFactory(host=host, port=port)
     elif input_factory == "edge":
         input_factory = EdgeFactory(host=host, port=port)
-
     if output_factory == "miniseed":
         output_factory = MiniSeedFactory(
             host=host, port=output_read_port, write_port=output_port

setup.py

@@ -26,8 +26,8 @@ setuptools.setup(
     entry_points={
         "console_scripts": [
             "magproc-prepfiles=geomagio.processing.magproc:main",
-            "filter-realtime=geomagio.processing.obsrio:main",
             "generate-matrix=geomagio.processing.adjusted:main",
+            "obsrio-filter=geomagio.processing.obsrio:main",
         ],
     },
 )