diff --git a/geomagio/api/db/MetadataDatabaseFactory.py b/geomagio/api/db/MetadataDatabaseFactory.py
new file mode 100644
index 0000000000000000000000000000000000000000..748bbabc8d3f01cf78ff435b457560280840ddc0
--- /dev/null
+++ b/geomagio/api/db/MetadataDatabaseFactory.py
@@ -0,0 +1,117 @@
+from datetime import datetime
+from typing import List
+
+from databases import Database
+from obspy import UTCDateTime
+from sqlalchemy import or_
+
+from ...metadata import Metadata, MetadataCategory
+from .metadata_history_table import metadata_history
+from .metadata_table import metadata
+
+
+class MetadataDatabaseFactory(object):
+ def __init__(self, database: Database):
+ self.database = database
+
+ async def create_metadata(self, meta: Metadata) -> Metadata:
+ query = metadata.insert()
+ values = meta.datetime_dict(exclude={"id", "metadata_id"}, exclude_none=True)
+ query = query.values(**values)
+ meta.id = await self.database.execute(query)
+ return meta
+
+ async def get_metadata(
+ self,
+ *, # make all params keyword
+ id: int = None,
+ network: str = None,
+ station: str = None,
+ channel: str = None,
+ location: str = None,
+ category: MetadataCategory = None,
+ starttime: datetime = None,
+ endtime: datetime = None,
+ created_after: datetime = None,
+ created_before: datetime = None,
+ data_valid: bool = None,
+ metadata_valid: bool = None,
+ status: str = None,
+ ):
+ query = metadata.select()
+ if id:
+ query = query.where(metadata.c.id == id)
+ if category:
+ query = query.where(metadata.c.category == category)
+ if network:
+ query = query.where(metadata.c.network == network)
+ if station:
+ query = query.where(metadata.c.station == station)
+ if channel:
+ query = query.where(metadata.c.channel.like(channel))
+ if location:
+ query = query.where(metadata.c.location.like(location))
+ if starttime:
+ query = query.where(
+ or_(
+ metadata.c.endtime == None,
+ metadata.c.endtime > starttime,
+ )
+ )
+ if endtime:
+ query = query.where(
+ or_(
+ metadata.c.starttime == None,
+ metadata.c.starttime < endtime,
+ )
+ )
+ if created_after:
+ query = query.where(metadata.c.created_time > created_after)
+ if created_before:
+ query = query.where(metadata.c.created_time < created_before)
+ if data_valid is not None:
+ query = query.where(metadata.c.data_valid == data_valid)
+ if metadata_valid is not None:
+ query = query.where(metadata.c.metadata_valid == metadata_valid)
+ if status is not None:
+ query = query.where(metadata.c.status == status)
+ rows = await self.database.fetch_all(query)
+ return [Metadata(**row) for row in rows]
+
+ async def get_metadata_by_id(self, id: int):
+ meta = await self.get_metadata(id=id)
+ if len(meta) != 1:
+ raise ValueError(f"{len(meta)} records found")
+ return meta[0]
+
+ async def get_metadata_history(self, metadata_id: int) -> List[Metadata]:
+ async with self.database.transaction() as transaction:
+ query = metadata_history.select()
+ query = query.where(metadata_history.c.metadata_id == metadata_id).order_by(
+ metadata_history.c.updated_time
+ )
+ rows = await self.database.fetch_all(query)
+ metadata = [Metadata(**row) for row in rows]
+ current_metadata = await self.get_metadata_by_id(id=metadata_id)
+ metadata.append(current_metadata)
+ # return records in order of age(newest first)
+ metadata.reverse()
+ return metadata
+
+ async def update_metadata(self, meta: Metadata, updated_by: str) -> Metadata:
+ async with self.database.transaction() as transaction:
+ # write current record to metadata history table
+ original_metadata = await self.get_metadata_by_id(id=meta.id)
+ original_metadata.metadata_id = original_metadata.id
+ values = original_metadata.datetime_dict(exclude={"id"}, exclude_none=True)
+ query = metadata_history.insert()
+ query = query.values(**values)
+ original_metadata.id = await self.database.execute(query)
+ # update record in metadata table
+ meta.updated_by = updated_by
+ meta.updated_time = UTCDateTime()
+ query = metadata.update().where(metadata.c.id == meta.id)
+ values = meta.datetime_dict(exclude={"id", "metadata_id"})
+ query = query.values(**values)
+ await self.database.execute(query)
+ return await self.get_metadata_by_id(id=meta.id)
diff --git a/geomagio/api/db/__init__.py b/geomagio/api/db/__init__.py
index 50e414c7396b72a2aa4d0b7849ba42b801ab417e..dfb22968ecc34f0a19bcc60648f7efae772bf834 100644
--- a/geomagio/api/db/__init__.py
+++ b/geomagio/api/db/__init__.py
@@ -7,5 +7,10 @@ Modules outside the api should not access the database directly.
"""
from .common import database, sqlalchemy_metadata
+from .MetadataDatabaseFactory import MetadataDatabaseFactory
-__all__ = ["database", "sqlalchemy_metadata"]
+__all__ = [
+ "database",
+ "sqlalchemy_metadata",
+ "MetadataDatabaseFactory",
+]
diff --git a/geomagio/api/db/create.py b/geomagio/api/db/create.py
index 0ccc0a97a0d0ce5161a3850dfc0d8588ada2fdfd..144cf0840f114a2cf3440311c283ea4c655e3b8f 100644
--- a/geomagio/api/db/create.py
+++ b/geomagio/api/db/create.py
@@ -3,6 +3,7 @@ import sqlalchemy
from .common import database, sqlalchemy_metadata
# register models with sqlalchemy_metadata by importing
+from .metadata_history_table import metadata_history
from .metadata_table import metadata
from .session_table import session
diff --git a/geomagio/api/db/metadata_history_table.py b/geomagio/api/db/metadata_history_table.py
new file mode 100644
index 0000000000000000000000000000000000000000..471869fe132d16a249ca31d283c0235a8e2d10d2
--- /dev/null
+++ b/geomagio/api/db/metadata_history_table.py
@@ -0,0 +1,18 @@
+from sqlalchemy import Column, ForeignKey, Integer
+
+from .common import sqlalchemy_metadata
+from .metadata_table import metadata
+
+# create copy of original metadata table and add to sqlalchemy metadata
+metadata_history = metadata.tometadata(
+ metadata=sqlalchemy_metadata, name="metadata_history"
+)
+metadata_history.indexes.clear()
+metadata_history.append_column(
+ Column(
+ "metadata_id",
+ Integer,
+ ForeignKey("metadata.id"),
+ nullable=False,
+ ),
+)
diff --git a/geomagio/api/db/metadata_table.py b/geomagio/api/db/metadata_table.py
index 211a18044c4f675c3f659cad71ae6c2204bfbf33..e55e66878b063583136afaa8a2de152c3a37e859 100644
--- a/geomagio/api/db/metadata_table.py
+++ b/geomagio/api/db/metadata_table.py
@@ -1,10 +1,7 @@
-from datetime import datetime
-
-from sqlalchemy import or_, Boolean, Column, Index, Integer, JSON, String, Table, Text
+from sqlalchemy import Boolean, Column, Index, Integer, JSON, String, Table, Text
import sqlalchemy_utc
-from ...metadata import Metadata, MetadataCategory
-from .common import database, sqlalchemy_metadata
+from .common import sqlalchemy_metadata
"""Metadata database model.
@@ -24,9 +21,9 @@ metadata = Table(
default=sqlalchemy_utc.utcnow(),
index=True,
),
- # reviewer
- Column("reviewed_by", String(length=255), index=True, nullable=True),
- Column("reviewed_time", sqlalchemy_utc.UtcDateTime, index=True, nullable=True),
+ # editor
+ Column("updated_by", String(length=255), index=True, nullable=True),
+ Column("updated_time", sqlalchemy_utc.UtcDateTime, index=True, nullable=True),
# time range
Column("starttime", sqlalchemy_utc.UtcDateTime, index=True, nullable=True),
Column("endtime", sqlalchemy_utc.UtcDateTime, index=True, nullable=True),
@@ -43,6 +40,8 @@ metadata = Table(
Column("data_valid", Boolean, default=True, index=True),
# whether metadata is valid (based on review)
Column("metadata_valid", Boolean, default=True, index=True),
+ # deletion status indicator
+ Column("status", String(length=255), nullable=True),
# metadata json blob
Column("metadata", JSON, nullable=True),
# general comment
@@ -65,6 +64,7 @@ metadata = Table(
# valid
"metadata_valid",
"data_valid",
+ "status",
),
Index(
"index_category_time",
@@ -75,71 +75,3 @@ metadata = Table(
"endtime",
),
)
-
-
-async def create_metadata(meta: Metadata) -> Metadata:
- query = metadata.insert()
- values = meta.datetime_dict(exclude={"id"}, exclude_none=True)
- query = query.values(**values)
- meta.id = await database.execute(query)
- return meta
-
-
-async def delete_metadata(id: int) -> None:
- query = metadata.delete().where(metadata.c.id == id)
- await database.execute(query)
-
-
-async def get_metadata(
- *, # make all params keyword
- id: int = None,
- network: str = None,
- station: str = None,
- channel: str = None,
- location: str = None,
- category: MetadataCategory = None,
- starttime: datetime = None,
- endtime: datetime = None,
- created_after: datetime = None,
- created_before: datetime = None,
- data_valid: bool = None,
- metadata_valid: bool = None,
-):
- query = metadata.select()
- if id:
- query = query.where(metadata.c.id == id)
- if category:
- query = query.where(metadata.c.category == category)
- if network:
- query = query.where(metadata.c.network == network)
- if station:
- query = query.where(metadata.c.station == station)
- if channel:
- query = query.where(metadata.c.channel.like(channel))
- if location:
- query = query.where(metadata.c.location.like(location))
- if starttime:
- query = query.where(
- or_(metadata.c.endtime == None, metadata.c.endtime > starttime)
- )
- if endtime:
- query = query.where(
- or_(metadata.c.starttime == None, metadata.c.starttime < endtime)
- )
- if created_after:
- query = query.where(metadata.c.created_time > created_after)
- if created_before:
- query = query.where(metadata.c.created_time < created_before)
- if data_valid is not None:
- query = query.where(metadata.c.data_valid == data_valid)
- if metadata_valid is not None:
- query = query.where(metadata.c.metadata_valid == metadata_valid)
- rows = await database.fetch_all(query)
- return [Metadata(**row) for row in rows]
-
-
-async def update_metadata(meta: Metadata) -> None:
- query = metadata.update().where(metadata.c.id == meta.id)
- values = meta.datetime_dict(exclude={"id"})
- query = query.values(**values)
- await database.execute(query)
diff --git a/geomagio/api/secure/metadata.py b/geomagio/api/secure/metadata.py
index e619c66362b8b4e960dd92cba5b5d88ff493b6b2..d397faca167ba4caf279b2e90e4a4a36cc791108 100644
--- a/geomagio/api/secure/metadata.py
+++ b/geomagio/api/secure/metadata.py
@@ -21,7 +21,8 @@ from obspy import UTCDateTime
from ...metadata import Metadata, MetadataCategory, MetadataQuery
from ... import pydantic_utcdatetime
-from ..db import metadata_table
+from ..db.common import database
+from ..db import MetadataDatabaseFactory
from .login import require_user, User
# routes for login/logout
@@ -34,17 +35,12 @@ async def create_metadata(
metadata: Metadata,
user: User = Depends(require_user()),
):
- metadata = await metadata_table.create_metadata(metadata)
+ metadata = await MetadataDatabaseFactory(database=database).create_metadata(
+ meta=metadata
+ )
return Response(metadata.json(), status_code=201, media_type="application/json")
-@router.delete("/metadata/{id}")
-async def delete_metadata(
- id: int, user: User = Depends(require_user([os.getenv("ADMIN_GROUP", "admin")]))
-):
- await metadata_table.delete_metadata(id)
-
-
@router.get("/metadata", response_model=List[Metadata])
async def get_metadata(
category: MetadataCategory = None,
@@ -58,6 +54,7 @@ async def get_metadata(
location: str = None,
data_valid: bool = None,
metadata_valid: bool = True,
+ status: str = None,
):
query = MetadataQuery(
category=category,
@@ -71,18 +68,26 @@ async def get_metadata(
location=location,
data_valid=data_valid,
metadata_valid=metadata_valid,
+ status=status,
+ )
+ metas = await MetadataDatabaseFactory(database=database).get_metadata(
+ **query.datetime_dict(exclude={"id", "metadata_id"})
)
- metas = await metadata_table.get_metadata(**query.datetime_dict(exclude={"id"}))
return metas
@router.get("/metadata/{id}", response_model=Metadata)
async def get_metadata_by_id(id: int):
- meta = await metadata_table.get_metadata(id=id)
- if len(meta) != 1:
- return Response(status_code=404)
- else:
- return meta[0]
+ return await MetadataDatabaseFactory(database=database).get_metadata_by_id(id=id)
+
+
+@router.get("/metadata/{metadata_id}/history", response_model=List[Metadata])
+async def get_metadata_history(
+ metadata_id: int,
+):
+ return await MetadataDatabaseFactory(database=database).get_metadata_history(
+ metadata_id=metadata_id,
+ )
@router.put("/metadata/{id}", response_model=Metadata)
@@ -91,6 +96,7 @@ async def update_metadata(
metadata: Metadata = Body(...),
user: User = Depends(require_user([os.getenv("REVIEWER_GROUP", "reviewer")])),
):
- await metadata_table.update_metadata(metadata)
- # should be same, but read from database
- return await get_metadata_by_id(metadata.id)
+ return await MetadataDatabaseFactory(database=database).update_metadata(
+ meta=metadata,
+ updated_by=user.nickname,
+ )
diff --git a/geomagio/api/ws/DataApiQuery.py b/geomagio/api/ws/DataApiQuery.py
index af1c3f73ff3fc880cc75df45cad2bc20240225f8..d955b7882a12988697232f45e72bc33dae66353d 100644
--- a/geomagio/api/ws/DataApiQuery.py
+++ b/geomagio/api/ws/DataApiQuery.py
@@ -1,7 +1,7 @@
import datetime
import enum
import os
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Union
from obspy import UTCDateTime
from pydantic import BaseModel, root_validator, validator
diff --git a/geomagio/api/ws/metadata.py b/geomagio/api/ws/metadata.py
index 655c33e59f0ec84b4329e5e3c6642c6dbe24882c..1dc77ee2c7fbcbee927707629dbf60703207c3f1 100644
--- a/geomagio/api/ws/metadata.py
+++ b/geomagio/api/ws/metadata.py
@@ -4,7 +4,8 @@ from fastapi import APIRouter
from obspy import UTCDateTime
from ...metadata import Metadata, MetadataCategory, MetadataQuery
-from ..db import metadata_table
+from ..db.common import database
+from ..db import MetadataDatabaseFactory
router = APIRouter()
@@ -20,6 +21,7 @@ async def get_metadata(
location: str = None,
data_valid: bool = None,
metadata_valid: bool = True,
+ status: str = None,
):
query = MetadataQuery(
category=category,
@@ -31,6 +33,9 @@ async def get_metadata(
location=location,
data_valid=data_valid,
metadata_valid=metadata_valid,
+ status=status,
+ )
+ metas = await MetadataDatabaseFactory(database=database).get_metadata(
+ **query.datetime_dict(exclude={"id"})
)
- metas = await metadata_table.get_metadata(**query.datetime_dict(exclude={"id"}))
return metas
diff --git a/geomagio/metadata/Metadata.py b/geomagio/metadata/Metadata.py
index ce2ad5211ac07c5023c4821396a8abe1ed1fd4f5..dff97fbeb4d448951f0503950a77ea052ac2f851 100644
--- a/geomagio/metadata/Metadata.py
+++ b/geomagio/metadata/Metadata.py
@@ -48,12 +48,14 @@ class Metadata(BaseModel):
# database id
id: int = None
+ # metadata history id referencing database id
+ metadata_id: int = None
# author
created_by: str = None
created_time: UTCDateTime = None
- # reviewer
- reviewed_by: str = None
- reviewed_time: UTCDateTime = None
+ # editor
+ updated_by: str = None
+ updated_time: UTCDateTime = None
# time range
starttime: UTCDateTime = None
endtime: UTCDateTime = None
@@ -76,10 +78,12 @@ class Metadata(BaseModel):
comment: str = None
# review specific comment
review_comment: str = None
+ # deletion status indicator
+ status: str = None
def datetime_dict(self, **kwargs):
values = self.dict(**kwargs)
- for key in ["created_time", "reviewed_time", "starttime", "endtime"]:
+ for key in ["created_time", "updated_time", "starttime", "endtime"]:
if key in values and values[key] is not None:
values[key] = values[key].datetime.replace(tzinfo=timezone.utc)
return values
diff --git a/geomagio/metadata/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
index 0558bcc20efa7030953ebc4ae6be0c45b700b9d5..e000c887f99b53d8f798b8d2149a1605788ea555 100644
--- a/geomagio/metadata/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -27,15 +27,6 @@ class MetadataFactory(object):
def _get_headers(self):
return {"Authorization": self.token} if self.token else None
- def delete_metadata(self, metadata: Metadata) -> bool:
- response = requests.delete(
- url=f"{self.url}/{metadata.id}",
- headers=self._get_headers(),
- )
- if response.status_code == 200:
- return True
- return False
-
def get_metadata(self, query: MetadataQuery) -> List[Metadata]:
if query.id:
metadata = [self.get_metadata_by_id(id=query.id)]
diff --git a/geomagio/metadata/MetadataQuery.py b/geomagio/metadata/MetadataQuery.py
index 36e6c3ebe3cd20d880c25ff0d683757c84a2d833..5b5214849e1d3bb74821c4389052631f20c868c6 100644
--- a/geomagio/metadata/MetadataQuery.py
+++ b/geomagio/metadata/MetadataQuery.py
@@ -21,6 +21,7 @@ class MetadataQuery(BaseModel):
location: str = None
data_valid: Optional[bool] = None
metadata_valid: Optional[bool] = None
+ status: Optional[str] = None
def datetime_dict(self, **kwargs):
values = self.dict(**kwargs)
diff --git a/geomagio/metadata/main.py b/geomagio/metadata/main.py
index 79cf886baa37a048046498d67b2bb785a9d21473..69e8eeecc3f6e718610547f74df5266cddd86591 100644
--- a/geomagio/metadata/main.py
+++ b/geomagio/metadata/main.py
@@ -1,7 +1,6 @@
import sys
import json
import os
-import textwrap
from typing import Dict, Optional
from obspy import UTCDateTime
@@ -111,24 +110,6 @@ def create(
print(metadata.json())
-@app.command(
- help=f"""
- Delete an existing metadata.
-
- {ENVIRONMENT_VARIABLE_HELP}
- """
-)
-def delete(
- input_file: str,
- url: str = GEOMAG_API_URL,
-):
- metadata_dict = load_metadata(input_file=input_file)
- metadata = Metadata(**metadata_dict)
- deleted = MetadataFactory(url=url).delete_metadata(metadata=metadata)
- if not deleted:
- sys.exit(1)
-
-
@app.command(
help=f"""
Search existing metadata.
@@ -148,6 +129,7 @@ def get(
location: Optional[str] = None,
metadata_valid: Optional[bool] = None,
network: Optional[str] = None,
+ status: Optional[str] = None,
starttime: Optional[str] = None,
station: Optional[str] = None,
url: str = GEOMAG_API_URL,
@@ -165,6 +147,7 @@ def get(
network=network,
starttime=UTCDateTime(starttime) if starttime else None,
station=station,
+ status=status,
)
metadata = MetadataFactory(url=url).get_metadata(query=query)
if getone:
diff --git a/test_metadata.py b/test_metadata.py
index 6f4eab2e8b8277a42774e2bc417c92a1fdaa7170..1a78b1bcfc571d683a27243342301bd0d8bb3558 100644
--- a/test_metadata.py
+++ b/test_metadata.py
@@ -3,7 +3,7 @@ import json
from obspy import UTCDateTime
from geomagio.adjusted import AdjustedMatrix, Metric
-from geomagio.api.db import database, metadata_table
+from geomagio.api.db import database, MetadataDatabaseFactory
from geomagio.api.ws.Observatory import OBSERVATORIES
from geomagio.metadata import Metadata, MetadataCategory
from geomagio.residual import SpreadsheetAbsolutesFactory, WebAbsolutesFactory
@@ -137,7 +137,7 @@ for reading in readings:
category=MetadataCategory.READING,
created_by="test_metadata.py",
network="NT",
- reviewed_by=reviewer,
+ updated_by=reviewer,
starttime=reading.time,
endtime=reading.time,
station=reading.metadata["station"],
@@ -164,14 +164,19 @@ adjusted_matrix = AdjustedMatrix(
)
test_metadata.append(
- Metadata(station="FRD", network="NT", metadata=adjusted_matrix.dict())
+ Metadata(
+ category="adjusted-matrix",
+ station="FRD",
+ network="NT",
+ metadata=adjusted_matrix.dict(),
+ )
)
async def load_test_metadata():
await database.connect()
for meta in test_metadata:
- await metadata_table.create_metadata(meta)
+ await MetadataDatabaseFactory(database=database).create_metadata(meta)
await database.disconnect()