From bb496870bebe5cec1bcf920c3eb299129ae884d7 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Mon, 20 Apr 2020 19:21:49 -0600
Subject: [PATCH] Add preliminary metadata service
---
geomagio/api/db/create.py | 4 +-
.../api/db/{metadata.py => metadata_table.py} | 62 ++++++---
.../api/db/{session.py => session_table.py} | 0
geomagio/api/secure/MetadataQuery.py | 31 +++++
geomagio/api/secure/MetadataResponse.py | 11 ++
geomagio/api/secure/app.py | 4 +-
geomagio/api/secure/metadata.py | 70 ++++++++++
geomagio/metadata/Metadata.py | 14 +-
geomagio/metadata/MetadataCategory.py | 1 +
geomagio/metadata/__init__.py | 5 +
test_metadata.py | 123 ++++++++++++++++++
11 files changed, 304 insertions(+), 21 deletions(-)
rename geomagio/api/db/{metadata.py => metadata_table.py} (65%)
rename geomagio/api/db/{session.py => session_table.py} (100%)
create mode 100644 geomagio/api/secure/MetadataQuery.py
create mode 100644 geomagio/api/secure/MetadataResponse.py
create mode 100644 geomagio/api/secure/metadata.py
create mode 100644 geomagio/metadata/__init__.py
create mode 100644 test_metadata.py
diff --git a/geomagio/api/db/create.py b/geomagio/api/db/create.py
index b2dc79313..80fc17e87 100644
--- a/geomagio/api/db/create.py
+++ b/geomagio/api/db/create.py
@@ -3,8 +3,8 @@ import sqlalchemy
from .common import database, sqlalchemy_metadata
# register models with sqlalchemy_metadata by importing
-from .metadata import metadata
-from .session import session
+from .metadata_table import metadata
+from .session_table import session
def create_db():
diff --git a/geomagio/api/db/metadata.py b/geomagio/api/db/metadata_table.py
similarity index 65%
rename from geomagio/api/db/metadata.py
rename to geomagio/api/db/metadata_table.py
index 79d4b9a35..9b1571a21 100644
--- a/geomagio/api/db/metadata.py
+++ b/geomagio/api/db/metadata_table.py
@@ -1,8 +1,8 @@
-import datetime
+from datetime import datetime
import enum
from obspy import UTCDateTime
-from sqlalchemy import or_, Boolean, Column, Index, Integer, String, Table, Text
+from sqlalchemy import or_, Boolean, Column, Index, Integer, JSON, String, Table, Text
import sqlalchemy_utc
from ...metadata import Metadata, MetadataCategory
@@ -23,7 +23,7 @@ metadata = Table(
Column(
"created_time",
sqlalchemy_utc.UtcDateTime,
- default=sqlalchemy_utc.now,
+ default=sqlalchemy_utc.utcnow(),
index=True,
),
# reviewer
@@ -46,7 +46,7 @@ metadata = Table(
# whether metadata is valid (based on review)
Column("metadata_valid", Boolean, default=True, index=True),
# metadata json blob
- Column("metadata", Text, nullable=True),
+ Column("metadata", JSON, nullable=True),
# general comment
Column("comment", Text, nullable=True),
# review specific comment
@@ -79,34 +79,62 @@ metadata = Table(
)
+async def create_metadata(meta: Metadata) -> Metadata:
+ query = metadata.insert()
+ values = meta.datetime_dict(exclude={"id"}, exclude_none=True)
+ query = query.values(**values)
+ metadata.id = await database.execute(query)
+ return metadata
+
+
+async def delete_metadata(id: int) -> None:
+ query = metadata.delete().where(metadata.c.id == id)
+ await database.execute(query)
+
+
async def get_metadata(
*, # make all params keyword
- network: str,
- station: str,
+ id: int = None,
+ network: str = None,
+ station: str = None,
channel: str = None,
location: str = None,
category: MetadataCategory = None,
- starttime: UTCDateTime = None,
- endtime: UTCDateTime = None,
+ starttime: datetime = None,
+ endtime: datetime = None,
data_valid: bool = None,
metadata_valid: bool = None,
):
- query = (
- metadata.select()
- .where(metadata.c.network.like(network or "%"))
- .where(metadata.c.station.like(station or "%"))
- .where(metadata.c.channel.like(channel or "%"))
- .where(metadata.c.location.like(location or "%"))
- )
+ query = metadata.select()
+ if id is not None:
+ query = query.where(metadata.c.id == id)
+ if category:
+ query = query.where(metadata.c.category == category)
+ if network or station or channel or location:
+ query = (
+ query.where(metadata.c.network.like(network or "%"))
+ .where(metadata.c.station.like(station or "%"))
+ .where(metadata.c.channel.like(channel or "%"))
+ .where(metadata.c.location.like(location or "%"))
+ )
if starttime:
query = query.where(
- or_(metadata.c.endtime == None, metadata.c.endtime > starttime.datetime)
+ or_(metadata.c.endtime == None, metadata.c.endtime > starttime)
)
if endtime:
query = query.where(
- or_(metadata.c.starttime == None, metadata.c.starttime < endtime.datetime)
+ or_(metadata.c.starttime == None, metadata.c.starttime < endtime)
)
if data_valid is not None:
query = query.where(metadata.c.data_valid == data_valid)
if metadata_valid is not None:
query = query.where(metadata.c.metadata_valid == metadata_valid)
+ rows = await database.fetch_all(query)
+ return [Metadata(**row) for row in rows]
+
+
+async def update_metadata(meta: Metadata) -> None:
+ query = metadata.update().where(metadata.c.id == meta.id)
+ values = meta.datetime_dict(exclude={"id"})
+ query = query.values(**values)
+ await database.execute(query)
diff --git a/geomagio/api/db/session.py b/geomagio/api/db/session_table.py
similarity index 100%
rename from geomagio/api/db/session.py
rename to geomagio/api/db/session_table.py
diff --git a/geomagio/api/secure/MetadataQuery.py b/geomagio/api/secure/MetadataQuery.py
new file mode 100644
index 000000000..629ad0149
--- /dev/null
+++ b/geomagio/api/secure/MetadataQuery.py
@@ -0,0 +1,31 @@
+from datetime import timezone
+
+from obspy import UTCDateTime
+from pydantic import BaseModel, validator
+
+from ...metadata import MetadataCategory
+from ... import pydantic_utcdatetime
+
+
+class MetadataQuery(BaseModel):
+ id: int = None
+ category: MetadataCategory = None
+ starttime: UTCDateTime = None
+ endtime: UTCDateTime = None
+ network: str = None
+ station: str = None
+ channel: str = None
+ location: str = None
+ data_valid: bool = None
+ metadata_valid: bool = True
+
+ def datetime_dict(self, **kwargs):
+ values = self.dict(**kwargs)
+ for key in ["starttime", "endtime"]:
+ if key in values and values[key] is not None:
+ values[key] = values[key].datetime.replace(tzinfo=timezone.utc)
+ return values
+
+ @validator("starttime")
+ def set_default_starttime(cls, starttime: UTCDateTime = None) -> UTCDateTime:
+ return starttime or UTCDateTime() - 30 * 86400
diff --git a/geomagio/api/secure/MetadataResponse.py b/geomagio/api/secure/MetadataResponse.py
new file mode 100644
index 000000000..5c0516349
--- /dev/null
+++ b/geomagio/api/secure/MetadataResponse.py
@@ -0,0 +1,11 @@
+from typing import Dict, List
+
+from pydantic import BaseModel
+
+from ...metadata import Metadata
+from .MetadataQuery import MetadataQuery
+
+
+class MetadataResponse(BaseModel):
+ query: MetadataQuery
+ results: List[Metadata]
diff --git a/geomagio/api/secure/app.py b/geomagio/api/secure/app.py
index e6508cef7..0edaec59c 100644
--- a/geomagio/api/secure/app.py
+++ b/geomagio/api/secure/app.py
@@ -4,9 +4,10 @@ import uuid
from fastapi import Depends, FastAPI, Request, Response
-from ..db.session import delete_session, get_session, save_session
+from ..db.session_table import delete_session, get_session, save_session
from .encryption import get_fernet
from .login import current_user, router as login_router, User
+from .metadata import router as metadata_router
from .SessionMiddleware import SessionMiddleware
@@ -29,6 +30,7 @@ app.add_middleware(
# include login routes to manage user
app.include_router(login_router)
+app.include_router(metadata_router)
@app.get("/")
diff --git a/geomagio/api/secure/metadata.py b/geomagio/api/secure/metadata.py
new file mode 100644
index 000000000..bdd998af7
--- /dev/null
+++ b/geomagio/api/secure/metadata.py
@@ -0,0 +1,70 @@
+from typing import List
+
+from fastapi import APIRouter, Body, Depends, Request, Response
+from obspy import UTCDateTime
+
+from ...metadata import Metadata, MetadataCategory
+from ..db import metadata_table
+from .login import require_user, User
+from .MetadataQuery import MetadataQuery
+from .MetadataResponse import MetadataResponse
+from ... import pydantic_utcdatetime
+
+# routes for login/logout
+router = APIRouter()
+
+
+@router.post("/metadata", response_model=Metadata)
+async def create_metadata(
+ request: Request, metadata: Metadata, user: User = Depends(require_user()),
+):
+ metadata = await metadata_table.create_metadata(metadata)
+ return Response(metadata, status_code=201, media_type="application/json")
+
+
+@router.delete("/metadata/{id}")
+async def delete_metadata(id: int, user: User = Depends(require_user())):
+ await metadata_table.delete_metadata(id)
+
+
+@router.get("/metadata", response_model=List[Metadata])
+async def get_metadata(
+ category: MetadataCategory = None,
+ starttime: UTCDateTime = None,
+ endtime: UTCDateTime = None,
+ network: str = None,
+ station: str = None,
+ channel: str = None,
+ location: str = None,
+ data_valid: bool = None,
+ metadata_valid: bool = True,
+):
+ query = MetadataQuery(
+ category=category,
+ starttime=starttime,
+ endtime=endtime,
+ network=network,
+ station=station,
+ channel=channel,
+ location=location,
+ data_valid=data_valid,
+ metadata_valid=metadata_valid,
+ )
+ metas = await metadata_table.get_metadata(**query.datetime_dict(exclude={"id"}))
+ return metas
+
+
+@router.get("/metadata/{id}", response_model=Metadata)
+async def get_metadata_by_id(id: int):
+ meta = await metadata_table.get_metadata(id=id)
+ if len(meta) != 1:
+ return Response(status_code=404)
+ else:
+ return meta[0]
+
+
+@router.put("/metadata/{id}")
+async def update_metadata(
+ id: int, metadata: Metadata = Body(...), user: User = Depends(require_user()),
+):
+ await metadata_table.update_metadata(metadata)
diff --git a/geomagio/metadata/Metadata.py b/geomagio/metadata/Metadata.py
index b1962fdd2..ce2ad5211 100644
--- a/geomagio/metadata/Metadata.py
+++ b/geomagio/metadata/Metadata.py
@@ -1,7 +1,8 @@
+from datetime import timezone
from typing import Dict
from obspy import UTCDateTime
-from pydantic import BaseModel
+from pydantic import BaseModel, validator
from .. import pydantic_utcdatetime
from .MetadataCategory import MetadataCategory
@@ -75,3 +76,14 @@ class Metadata(BaseModel):
comment: str = None
# review specific comment
review_comment: str = None
+
+ def datetime_dict(self, **kwargs):
+ values = self.dict(**kwargs)
+ for key in ["created_time", "reviewed_time", "starttime", "endtime"]:
+ if key in values and values[key] is not None:
+ values[key] = values[key].datetime.replace(tzinfo=timezone.utc)
+ return values
+
+ @validator("created_time")
+ def set_default_created_time(cls, created_time: UTCDateTime = None) -> UTCDateTime:
+ return created_time or UTCDateTime()
diff --git a/geomagio/metadata/MetadataCategory.py b/geomagio/metadata/MetadataCategory.py
index d70092f84..44c5608b6 100644
--- a/geomagio/metadata/MetadataCategory.py
+++ b/geomagio/metadata/MetadataCategory.py
@@ -5,4 +5,5 @@ class MetadataCategory(str, Enum):
ADJUSTED_MATRIX = "adjusted-matrix"
FLAG = "flag"
INSTRUMENT = "instrument"
+ OBSERVATORY = "observatory"
READING = "reading"
diff --git a/geomagio/metadata/__init__.py b/geomagio/metadata/__init__.py
new file mode 100644
index 000000000..7502db024
--- /dev/null
+++ b/geomagio/metadata/__init__.py
@@ -0,0 +1,5 @@
+from .Metadata import Metadata
+from .MetadataCategory import MetadataCategory
+
+
+__all__ = ["Metadata", "MetadataCategory"]
diff --git a/test_metadata.py b/test_metadata.py
new file mode 100644
index 000000000..5d597ed8b
--- /dev/null
+++ b/test_metadata.py
@@ -0,0 +1,123 @@
+from obspy import UTCDateTime
+
+from geomagio.api.db import database, metadata_table
+from geomagio.api.ws.Observatory import OBSERVATORIES
+from geomagio.metadata import Metadata, MetadataCategory
+
+
+test_metadata = [
+ Metadata(
+ category=MetadataCategory.INSTRUMENT,
+ created_by="test_metadata.py",
+ network="NT",
+ station="BDT",
+ metadata={
+ "type": "FGE",
+ "channels": {
+ # each channel maps to a list of components to calculate nT
+ # TODO: calculate these lists based on "FGE" type
+ "U": [{"channel": "U_Volt", "offset": 0, "scale": 313.2}],
+ "V": [{"channel": "V_Volt", "offset": 0, "scale": 312.3}],
+ "W": [{"channel": "W_Volt", "offset": 0, "scale": 312.0}],
+ },
+ "electronics": {
+ "serial": "E0542",
+ # these scale values are used to convert voltage
+ "x-scale": 313.2, # V/nT
+ "y-scale": 312.3, # V/nT
+ "z-scale": 312.0, # V/nT
+ "temperature-scale": 0.01, # V/K
+ },
+ "sensor": {
+ "serial": "S0419",
+ # these constants combine with instrument setting for offset
+ "x-constant": 36958, # nT/mA
+ "y-constant": 36849, # nT/mA
+ "z-constant": 36811, # nT/mA
+ },
+ },
+ ),
+ Metadata(
+ category=MetadataCategory.INSTRUMENT,
+ created_by="test_metadata.py",
+ network="NT",
+ station="NEW",
+ metadata={
+ "type": "Narod",
+ "channels": {
+ "U": [
+ {"channel": "U_Volt", "offset": 0, "scale": 100},
+ {"channel": "U_Bin", "offset": 0, "scale": 500},
+ ],
+ "V": [
+ {"channel": "V_Volt", "offset": 0, "scale": 100},
+ {"channel": "V_Bin", "offset": 0, "scale": 500},
+ ],
+ "W": [
+ {"channel": "W_Volt", "offset": 0, "scale": 100},
+ {"channel": "W_Bin", "offset": 0, "scale": 500},
+ ],
+ },
+ },
+ ),
+ Metadata(
+ category=MetadataCategory.INSTRUMENT,
+ created_by="test_metadata.py",
+ network="NT",
+ station="LLO",
+ metadata={
+ "type": "Narod",
+ "channels": {
+ "U": [
+ {"channel": "U_Volt", "offset": 0, "scale": 100},
+ {"channel": "U_Bin", "offset": 0, "scale": 500},
+ ],
+ "V": [
+ {"channel": "V_Volt", "offset": 0, "scale": 100},
+ {"channel": "V_Bin", "offset": 0, "scale": 500},
+ ],
+ "W": [
+ {"channel": "W_Volt", "offset": 0, "scale": 100},
+ {"channel": "W_Bin", "offset": 0, "scale": 500},
+ ],
+ },
+ },
+ ),
+]
+
+# add observatories
+for observatory in OBSERVATORIES:
+ network = "NT"
+ if observatory.agency == "USGS":
+ network = "NT"
+ # rest alphabetical by agency
+ elif observatory.agency == "BGS":
+ network = "GB"
+ elif observatory.agency == "GSC":
+ network = "C2"
+ elif observatory.agency == "JMA":
+ network = "JP"
+ elif observatory.agency == "SANSA":
+ network = "AF"
+ test_metadata.append(
+ Metadata(
+ category=MetadataCategory.OBSERVATORY,
+ created_by="test_metadata.py",
+ network=network,
+ station=observatory.id,
+ metadata=observatory.dict(),
+ )
+ )
+
+
+async def load_test_metadata():
+ await database.connect()
+ for meta in test_metadata:
+ await metadata_table.create_metadata(meta)
+ await database.disconnect()
+
+
+if __name__ == "__main__":
+ import asyncio
+
+ asyncio.run(load_test_metadata())
--
GitLab