diff --git a/geomagio/api/db/create.py b/geomagio/api/db/create.py
index b2dc79313aa132b8bc510a87cc5900e1e60d6d36..80fc17e878bde68070f327f8e79278fa06a7dfdc 100644
--- a/geomagio/api/db/create.py
+++ b/geomagio/api/db/create.py
@@ -3,8 +3,8 @@ import sqlalchemy
from .common import database, sqlalchemy_metadata
# register models with sqlalchemy_metadata by importing
-from .metadata import metadata
-from .session import session
+from .metadata_table import metadata
+from .session_table import session
def create_db():
diff --git a/geomagio/api/db/metadata.py b/geomagio/api/db/metadata_table.py
similarity index 65%
rename from geomagio/api/db/metadata.py
rename to geomagio/api/db/metadata_table.py
index 79d4b9a35e2e15b421300c102401b8b8611478ad..9b1571a21c1abf54a7aed5e644e0974d7c001c99 100644
--- a/geomagio/api/db/metadata.py
+++ b/geomagio/api/db/metadata_table.py
@@ -1,8 +1,8 @@
-import datetime
+from datetime import datetime
import enum
from obspy import UTCDateTime
-from sqlalchemy import or_, Boolean, Column, Index, Integer, String, Table, Text
+from sqlalchemy import or_, Boolean, Column, Index, Integer, JSON, String, Table, Text
import sqlalchemy_utc
from ...metadata import Metadata, MetadataCategory
@@ -23,7 +23,7 @@ metadata = Table(
Column(
"created_time",
sqlalchemy_utc.UtcDateTime,
- default=sqlalchemy_utc.now,
+ default=sqlalchemy_utc.utcnow(),
index=True,
),
# reviewer
@@ -46,7 +46,7 @@ metadata = Table(
# whether metadata is valid (based on review)
Column("metadata_valid", Boolean, default=True, index=True),
# metadata json blob
- Column("metadata", Text, nullable=True),
+ Column("metadata", JSON, nullable=True),
# general comment
Column("comment", Text, nullable=True),
# review specific comment
@@ -79,34 +79,62 @@ metadata = Table(
)
+async def create_metadata(meta: Metadata) -> Metadata:
+ query = metadata.insert()
+ values = meta.datetime_dict(exclude={"id"}, exclude_none=True)
+ query = query.values(**values)
+ metadata.id = await database.execute(query)
+ return metadata
+
+
+async def delete_metadata(id: int) -> None:
+ query = metadata.delete().where(metadata.c.id == id)
+ await database.execute(query)
+
+
async def get_metadata(
*, # make all params keyword
- network: str,
- station: str,
+ id: int = None,
+ network: str = None,
+ station: str = None,
channel: str = None,
location: str = None,
category: MetadataCategory = None,
- starttime: UTCDateTime = None,
- endtime: UTCDateTime = None,
+ starttime: datetime = None,
+ endtime: datetime = None,
data_valid: bool = None,
metadata_valid: bool = None,
):
- query = (
- metadata.select()
- .where(metadata.c.network.like(network or "%"))
- .where(metadata.c.station.like(station or "%"))
- .where(metadata.c.channel.like(channel or "%"))
- .where(metadata.c.location.like(location or "%"))
- )
+ query = metadata.select()
+ if id is not None:
+ query = query.where(metadata.c.id == id)
+ if category:
+ query = query.where(metadata.c.category == category)
+ if network or station or channel or location:
+ query = (
+ query.where(metadata.c.network.like(network or "%"))
+ .where(metadata.c.station.like(station or "%"))
+ .where(metadata.c.channel.like(channel or "%"))
+ .where(metadata.c.location.like(location or "%"))
+ )
if starttime:
query = query.where(
- or_(metadata.c.endtime == None, metadata.c.endtime > starttime.datetime)
+ or_(metadata.c.endtime == None, metadata.c.endtime > starttime)
)
if endtime:
query = query.where(
- or_(metadata.c.starttime == None, metadata.c.starttime < endtime.datetime)
+ or_(metadata.c.starttime == None, metadata.c.starttime < endtime)
)
if data_valid is not None:
query = query.where(metadata.c.data_valid == data_valid)
if metadata_valid is not None:
query = query.where(metadata.c.metadata_valid == metadata_valid)
+ rows = await database.fetch_all(query)
+ return [Metadata(**row) for row in rows]
+
+
+async def update_metadata(meta: Metadata) -> None:
+ query = metadata.update().where(metadata.c.id == meta.id)
+ values = meta.datetime_dict(exclude={"id"})
+ query = query.values(**values)
+ await database.execute(query)
diff --git a/geomagio/api/db/session.py b/geomagio/api/db/session_table.py
similarity index 100%
rename from geomagio/api/db/session.py
rename to geomagio/api/db/session_table.py
diff --git a/geomagio/api/secure/MetadataQuery.py b/geomagio/api/secure/MetadataQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..629ad014907dbf7d35c76acd4781caad98ea184e
--- /dev/null
+++ b/geomagio/api/secure/MetadataQuery.py
@@ -0,0 +1,31 @@
+from datetime import timezone
+
+from obspy import UTCDateTime
+from pydantic import BaseModel, validator
+
+from ...metadata import MetadataCategory
+from ... import pydantic_utcdatetime
+
+
+class MetadataQuery(BaseModel):
+ id: int = None
+ category: MetadataCategory = None
+ starttime: UTCDateTime = None
+ endtime: UTCDateTime = None
+ network: str = None
+ station: str = None
+ channel: str = None
+ location: str = None
+ data_valid: bool = None
+ metadata_valid: bool = True
+
+ def datetime_dict(self, **kwargs):
+ values = self.dict(**kwargs)
+ for key in ["starttime", "endtime"]:
+ if key in values and values[key] is not None:
+ values[key] = values[key].datetime.replace(tzinfo=timezone.utc)
+ return values
+
+ @validator("starttime")
+ def set_default_starttime(cls, starttime: UTCDateTime = None) -> UTCDateTime:
+ return starttime or UTCDateTime() - 30 * 86400
diff --git a/geomagio/api/secure/SessionMiddleware.py b/geomagio/api/secure/SessionMiddleware.py
index ed978a4a2ffab76d1d465a69f71766c6331ad594..1138e0797e0635e00c5ec2ac17ea28208594b178 100644
--- a/geomagio/api/secure/SessionMiddleware.py
+++ b/geomagio/api/secure/SessionMiddleware.py
@@ -113,6 +113,7 @@ class SessionMiddleware:
self, message: Message, value: str, max_age: int = None,
):
headers = MutableHeaders(scope=message)
+ headers.append("Cache-Control", "no-cache")
headers.append(
"Set-Cookie",
f"{self.session_cookie}={value};"
diff --git a/geomagio/api/secure/app.py b/geomagio/api/secure/app.py
index e6508cef732e3d445f2540593fb82a257d233828..0edaec59c82254689011d97c8ec43701d67d2321 100644
--- a/geomagio/api/secure/app.py
+++ b/geomagio/api/secure/app.py
@@ -4,9 +4,10 @@ import uuid
from fastapi import Depends, FastAPI, Request, Response
-from ..db.session import delete_session, get_session, save_session
+from ..db.session_table import delete_session, get_session, save_session
from .encryption import get_fernet
from .login import current_user, router as login_router, User
+from .metadata import router as metadata_router
from .SessionMiddleware import SessionMiddleware
@@ -29,6 +30,7 @@ app.add_middleware(
# include login routes to manage user
app.include_router(login_router)
+app.include_router(metadata_router)
@app.get("/")
diff --git a/geomagio/api/secure/metadata.py b/geomagio/api/secure/metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e4f084622ff72c54eadfdb95e82838bf788e3cc
--- /dev/null
+++ b/geomagio/api/secure/metadata.py
@@ -0,0 +1,89 @@
+"""Module for metadata service.
+
+Uses login.py for user management.
+
+Anyone can access metadata.
+Logged in users can create new metadata.
+Update and delete are restricted based on group membership.
+
+
+Configuration:
+ uses environment variables:
+
+ ADMIN_GROUP - delete is restricted the admin group.
+ REVIEWER_GROUP - update is restricted the reviewer group.
+"""
+import os
+from typing import List
+
+from fastapi import APIRouter, Body, Depends, Request, Response
+from obspy import UTCDateTime
+
+from ...metadata import Metadata, MetadataCategory
+from ..db import metadata_table
+from .login import require_user, User
+from .MetadataQuery import MetadataQuery
+from ... import pydantic_utcdatetime
+
+# routes for login/logout
+router = APIRouter()
+
+
+@router.post("/metadata", response_model=Metadata)
+async def create_metadata(
+ request: Request, metadata: Metadata, user: User = Depends(require_user()),
+):
+ metadata = await metadata_table.create_metadata(metadata)
+ return Response(metadata, status_code=201, media_type="application/json")
+
+
+@router.delete("/metadata/{id}")
+async def delete_metadata(
+ id: int, user: User = Depends(require_user(os.getenv("ADMIN_GROUP", "admin")))
+):
+ await metadata_table.delete_metadata(id)
+
+
+@router.get("/metadata", response_model=List[Metadata])
+async def get_metadata(
+ category: MetadataCategory = None,
+ starttime: UTCDateTime = None,
+ endtime: UTCDateTime = None,
+ network: str = None,
+ station: str = None,
+ channel: str = None,
+ location: str = None,
+ data_valid: bool = None,
+ metadata_valid: bool = True,
+):
+ query = MetadataQuery(
+ category=category,
+ starttime=starttime,
+ endtime=endtime,
+ network=network,
+ station=station,
+ channel=channel,
+ location=location,
+ data_valid=data_valid,
+ metadata_valid=metadata_valid,
+ )
+ metas = await metadata_table.get_metadata(**query.datetime_dict(exclude={"id"}))
+ return metas
+
+
+@router.get("/metadata/{id}", response_model=Metadata)
+async def get_metadata_by_id(id: int):
+ meta = await metadata_table.get_metadata(id=id)
+ if len(meta) != 1:
+ return Response(status_code=404)
+ else:
+ return meta[0]
+
+
+@router.put("/metadata/{id}")
+async def update_metadata(
+ id: int,
+ metadata: Metadata = Body(...),
+ user: User = Depends(require_user([os.getenv("REVIEWER_GROUP", "reviewer")])),
+):
+ await metadata_table.update_metadata(metadata)
diff --git a/geomagio/api/ws/app.py b/geomagio/api/ws/app.py
index 279e21229ff313854e46b38701d15ac48f8d5613..63819226e35e573593034497435a58ef6ba0ce89 100644
--- a/geomagio/api/ws/app.py
+++ b/geomagio/api/ws/app.py
@@ -2,6 +2,7 @@ from typing import Dict, Union
from fastapi import FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, PlainTextResponse, RedirectResponse
from obspy import UTCDateTime
@@ -22,6 +23,9 @@ VERSION = "version"
app = FastAPI(docs_url="/docs", openapi_prefix="/ws")
+
+app.add_middleware(CORSMiddleware, allow_origins=["*"], max_age=86400)
+
app.include_router(data.router)
app.include_router(elements.router)
app.include_router(observatories.router)
diff --git a/geomagio/metadata/Metadata.py b/geomagio/metadata/Metadata.py
index b1962fdd243562debc2d844d340d6375799aeb59..ce2ad5211ac07c5023c4821396a8abe1ed1fd4f5 100644
--- a/geomagio/metadata/Metadata.py
+++ b/geomagio/metadata/Metadata.py
@@ -1,7 +1,8 @@
+from datetime import timezone
from typing import Dict
from obspy import UTCDateTime
-from pydantic import BaseModel
+from pydantic import BaseModel, validator
from .. import pydantic_utcdatetime
from .MetadataCategory import MetadataCategory
@@ -75,3 +76,14 @@ class Metadata(BaseModel):
comment: str = None
# review specific comment
review_comment: str = None
+
+ def datetime_dict(self, **kwargs):
+ values = self.dict(**kwargs)
+ for key in ["created_time", "reviewed_time", "starttime", "endtime"]:
+ if key in values and values[key] is not None:
+ values[key] = values[key].datetime.replace(tzinfo=timezone.utc)
+ return values
+
+ @validator("created_time")
+ def set_default_created_time(cls, created_time: UTCDateTime = None) -> UTCDateTime:
+ return created_time or UTCDateTime()
diff --git a/geomagio/metadata/MetadataCategory.py b/geomagio/metadata/MetadataCategory.py
index d70092f842e83753b5121c7ea093cabcc2cfd36b..44c5608b6febabc9a4c42b7fe28c798dd2fe6192 100644
--- a/geomagio/metadata/MetadataCategory.py
+++ b/geomagio/metadata/MetadataCategory.py
@@ -5,4 +5,5 @@ class MetadataCategory(str, Enum):
ADJUSTED_MATRIX = "adjusted-matrix"
FLAG = "flag"
INSTRUMENT = "instrument"
+ OBSERVATORY = "observatory"
READING = "reading"
diff --git a/geomagio/metadata/__init__.py b/geomagio/metadata/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..7502db0243cac6af13d38c8386c3dd292ae386c2
--- /dev/null
+++ b/geomagio/metadata/__init__.py
@@ -0,0 +1,5 @@
+from .Metadata import Metadata
+from .MetadataCategory import MetadataCategory
+
+
+__all__ = ["Metadata", "MetadataCategory"]
diff --git a/test_metadata.py b/test_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d597ed8b74ba94e713c50989cbb7b5cded6fe62
--- /dev/null
+++ b/test_metadata.py
@@ -0,0 +1,123 @@
+from obspy import UTCDateTime
+
+from geomagio.api.db import database, metadata_table
+from geomagio.api.ws.Observatory import OBSERVATORIES
+from geomagio.metadata import Metadata, MetadataCategory
+
+
+test_metadata = [
+ Metadata(
+ category=MetadataCategory.INSTRUMENT,
+ created_by="test_metadata.py",
+ network="NT",
+ station="BDT",
+ metadata={
+ "type": "FGE",
+ "channels": {
+ # each channel maps to a list of components to calculate nT
+ # TODO: calculate these lists based on "FGE" type
+ "U": [{"channel": "U_Volt", "offset": 0, "scale": 313.2}],
+ "V": [{"channel": "V_Volt", "offset": 0, "scale": 312.3}],
+ "W": [{"channel": "W_Volt", "offset": 0, "scale": 312.0}],
+ },
+ "electronics": {
+ "serial": "E0542",
+ # these scale values are used to convert voltage
+ "x-scale": 313.2, # V/nT
+ "y-scale": 312.3, # V/nT
+ "z-scale": 312.0, # V/nT
+ "temperature-scale": 0.01, # V/K
+ },
+ "sensor": {
+ "serial": "S0419",
+ # these constants combine with instrument setting for offset
+ "x-constant": 36958, # nT/mA
+ "y-constant": 36849, # nT/mA
+ "z-constant": 36811, # nT/mA
+ },
+ },
+ ),
+ Metadata(
+ category=MetadataCategory.INSTRUMENT,
+ created_by="test_metadata.py",
+ network="NT",
+ station="NEW",
+ metadata={
+ "type": "Narod",
+ "channels": {
+ "U": [
+ {"channel": "U_Volt", "offset": 0, "scale": 100},
+ {"channel": "U_Bin", "offset": 0, "scale": 500},
+ ],
+ "V": [
+ {"channel": "V_Volt", "offset": 0, "scale": 100},
+ {"channel": "V_Bin", "offset": 0, "scale": 500},
+ ],
+ "W": [
+ {"channel": "W_Volt", "offset": 0, "scale": 100},
+ {"channel": "W_Bin", "offset": 0, "scale": 500},
+ ],
+ },
+ },
+ ),
+ Metadata(
+ category=MetadataCategory.INSTRUMENT,
+ created_by="test_metadata.py",
+ network="NT",
+ station="LLO",
+ metadata={
+ "type": "Narod",
+ "channels": {
+ "U": [
+ {"channel": "U_Volt", "offset": 0, "scale": 100},
+ {"channel": "U_Bin", "offset": 0, "scale": 500},
+ ],
+ "V": [
+ {"channel": "V_Volt", "offset": 0, "scale": 100},
+ {"channel": "V_Bin", "offset": 0, "scale": 500},
+ ],
+ "W": [
+ {"channel": "W_Volt", "offset": 0, "scale": 100},
+ {"channel": "W_Bin", "offset": 0, "scale": 500},
+ ],
+ },
+ },
+ ),
+]
+
+# add observatories
+for observatory in OBSERVATORIES:
+ network = "NT"
+ if observatory.agency == "USGS":
+ network = "NT"
+ # rest alphabetical by agency
+ elif observatory.agency == "BGS":
+ network = "GB"
+ elif observatory.agency == "GSC":
+ network = "C2"
+ elif observatory.agency == "JMA":
+ network = "JP"
+ elif observatory.agency == "SANSA":
+ network = "AF"
+ test_metadata.append(
+ Metadata(
+ category=MetadataCategory.OBSERVATORY,
+ created_by="test_metadata.py",
+ network=network,
+ station=observatory.id,
+ metadata=observatory.dict(),
+ )
+ )
+
+
+async def load_test_metadata():
+ await database.connect()
+ for meta in test_metadata:
+ await metadata_table.create_metadata(meta)
+ await database.disconnect()
+
+
+if __name__ == "__main__":
+ import asyncio
+
+ asyncio.run(load_test_metadata())