From d2d89ae4220e67b8ffa418c08610d15e1f531653 Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Thu, 8 Apr 2021 12:08:06 -0600
Subject: [PATCH] database for factory __init__, remove reviewed, remove delete
endpoint
---
geomagio/api/db/MetadataDatabaseFactory.py | 90 ++++++++++------------
geomagio/api/db/metadata_table.py | 3 -
geomagio/api/secure/metadata.py | 29 +++----
geomagio/api/ws/metadata.py | 5 +-
geomagio/metadata/Metadata.py | 2 -
geomagio/metadata/MetadataFactory.py | 9 ---
geomagio/metadata/MetadataQuery.py | 1 -
geomagio/metadata/main.py | 20 -----
test_metadata.py | 2 +-
9 files changed, 53 insertions(+), 108 deletions(-)
diff --git a/geomagio/api/db/MetadataDatabaseFactory.py b/geomagio/api/db/MetadataDatabaseFactory.py
index d221c4de4..89475eb33 100644
--- a/geomagio/api/db/MetadataDatabaseFactory.py
+++ b/geomagio/api/db/MetadataDatabaseFactory.py
@@ -1,35 +1,24 @@
from datetime import datetime
from typing import List
+from databases import Database
from obspy import UTCDateTime
-from sqlalchemy import or_, Table
+from sqlalchemy import or_
from ...metadata import Metadata, MetadataCategory
-from .common import database
from .metadata_history_table import metadata_history
from .metadata_table import metadata
class MetadataDatabaseFactory(object):
- def __init__(
- self, table: Table = metadata, history_table: Table = metadata_history
- ):
- self.table = table
- self.history_table = history_table
+ def __init__(self, database: Database):
+ self.database = database
async def create_metadata(self, meta: Metadata) -> Metadata:
query = metadata.insert()
values = meta.datetime_dict(exclude={"id", "metadata_id"}, exclude_none=True)
query = query.values(**values)
- meta.id = await database.execute(query)
- return meta
-
- async def create_metadata_history(self, meta: Metadata) -> Metadata:
- query = self.history_table.insert()
- meta.metadata_id = meta.id
- values = meta.datetime_dict(exclude={"id"}, exclude_none=True)
- query = query.values(**values)
- meta.id = await database.execute(query)
+ meta.id = await self.database.execute(query)
return meta
async def get_metadata(
@@ -47,50 +36,46 @@ class MetadataDatabaseFactory(object):
created_before: datetime = None,
data_valid: bool = None,
metadata_valid: bool = None,
- reviewed: bool = None,
status: str = None,
):
- table = self.table
- query = table.select()
+ query = metadata.select()
if id:
- query = query.where(table.c.id == id)
+ query = query.where(metadata.c.id == id)
if category:
- query = query.where(table.c.category == category)
+ query = query.where(metadata.c.category == category)
if network:
- query = query.where(table.c.network == network)
+ query = query.where(metadata.c.network == network)
if station:
- query = query.where(table.c.station == station)
+ query = query.where(metadata.c.station == station)
if channel:
- query = query.where(table.c.channel.like(channel))
+ query = query.where(metadata.c.channel.like(channel))
if location:
- query = query.where(table.c.location.like(location))
+ query = query.where(metadata.c.location.like(location))
if starttime:
query = query.where(
or_(
- table.c.endtime == None,
- table.c.endtime > starttime,
+ metadata.c.endtime == None,
+ metadata.c.endtime > starttime,
)
)
if endtime:
query = query.where(
or_(
- table.c.starttime == None,
- table.c.starttime < endtime,
+ metadata.c.starttime == None,
+ metadata.c.starttime < endtime,
)
)
if created_after:
- query = query.where(table.c.created_time > created_after)
+ query = query.where(metadata.c.created_time > created_after)
if created_before:
- query = query.where(table.c.created_time < created_before)
+ query = query.where(metadata.c.created_time < created_before)
if data_valid is not None:
- query = query.where(table.c.data_valid == data_valid)
+ query = query.where(metadata.c.data_valid == data_valid)
if metadata_valid is not None:
- query = query.where(table.c.metadata_valid == metadata_valid)
- if reviewed is not None:
- query = query.where(table.c.reviewed == reviewed)
+ query = query.where(metadata.c.metadata_valid == metadata_valid)
if status is not None:
- query = query.where(table.c.status == status)
- rows = await database.fetch_all(query)
+ query = query.where(metadata.c.status == status)
+ rows = await self.database.fetch_all(query)
return [Metadata(**row) for row in rows]
async def get_metadata_by_id(self, id: int):
@@ -100,27 +85,32 @@ class MetadataDatabaseFactory(object):
return meta[0]
async def get_metadata_history(self, metadata_id: int) -> List[Metadata]:
- query = self.history_table.select()
- query = query.where(self.history_table.c.metadata_id == metadata_id)
- rows = await database.fetch_all(query)
+ query = metadata_history.select()
+ query = query.where(metadata_history.c.metadata_id == metadata_id).order_by(
+ metadata_history.c.updated_time
+ )
+ rows = await self.database.fetch_all(query)
metadata = [Metadata(**row) for row in rows]
current_metadata = await self.get_metadata_by_id(id=metadata_id)
metadata.append(current_metadata)
- # return records in order of age
+ # return records in order of age(newest first)
metadata.reverse()
return metadata
- async def update_metadata(
- self, meta: Metadata, username: str, status: str = None
- ) -> Metadata:
- async with database.transaction() as transaction:
+ async def update_metadata(self, meta: Metadata, updated_by: str) -> Metadata:
+ async with self.database.transaction() as transaction:
+ # write current record to metadata history table
original_metadata = await self.get_metadata_by_id(id=meta.id)
- await self.create_metadata_history(meta=original_metadata)
- meta.updated_by = username
+ query = metadata_history.insert()
+ original_metadata.metadata_id = original_metadata.id
+ values = original_metadata.datetime_dict(exclude={"id"}, exclude_none=True)
+ query = query.values(**values)
+ original_metadata.id = await self.database.execute(query)
+ # update record in metadata table
+ meta.updated_by = updated_by
meta.updated_time = UTCDateTime()
- meta.status = status or meta.status
- query = self.table.update().where(self.table.c.id == meta.id)
+ query = metadata.update().where(metadata.c.id == meta.id)
values = meta.datetime_dict(exclude={"id", "metadata_id"})
query = query.values(**values)
- await database.execute(query)
+ await self.database.execute(query)
return await self.get_metadata_by_id(id=meta.id)
diff --git a/geomagio/api/db/metadata_table.py b/geomagio/api/db/metadata_table.py
index c80567828..e55e66878 100644
--- a/geomagio/api/db/metadata_table.py
+++ b/geomagio/api/db/metadata_table.py
@@ -40,8 +40,6 @@ metadata = Table(
Column("data_valid", Boolean, default=True, index=True),
# whether metadata is valid (based on review)
Column("metadata_valid", Boolean, default=True, index=True),
- # whether metadata has been reviewed
- Column("reviewed", Boolean, default=True, index=True),
# deletion status indicator
Column("status", String(length=255), nullable=True),
# metadata json blob
@@ -66,7 +64,6 @@ metadata = Table(
# valid
"metadata_valid",
"data_valid",
- "reviewed",
"status",
),
Index(
diff --git a/geomagio/api/secure/metadata.py b/geomagio/api/secure/metadata.py
index baf4094a0..d397faca1 100644
--- a/geomagio/api/secure/metadata.py
+++ b/geomagio/api/secure/metadata.py
@@ -21,6 +21,7 @@ from obspy import UTCDateTime
from ...metadata import Metadata, MetadataCategory, MetadataQuery
from ... import pydantic_utcdatetime
+from ..db.common import database
from ..db import MetadataDatabaseFactory
from .login import require_user, User
@@ -34,19 +35,10 @@ async def create_metadata(
metadata: Metadata,
user: User = Depends(require_user()),
):
- metadata = await MetadataDatabaseFactory().create_metadata(meta=metadata)
- return Response(metadata.json(), status_code=201, media_type="application/json")
-
-
-@router.delete("/metadata/{id}")
-async def delete_metadata(
- id: int, user: User = Depends(require_user([os.getenv("ADMIN_GROUP", "admin")]))
-):
- database_factory = MetadataDatabaseFactory()
- metadata = await database_factory.get_metadata_by_id(id=id)
- await database_factory.update_metadata(
- meta=metadata, username=user.nickname, status="deleted"
+ metadata = await MetadataDatabaseFactory(database=database).create_metadata(
+ meta=metadata
)
+ return Response(metadata.json(), status_code=201, media_type="application/json")
@router.get("/metadata", response_model=List[Metadata])
@@ -62,7 +54,6 @@ async def get_metadata(
location: str = None,
data_valid: bool = None,
metadata_valid: bool = True,
- reviewed: bool = None,
status: str = None,
):
query = MetadataQuery(
@@ -77,10 +68,9 @@ async def get_metadata(
location=location,
data_valid=data_valid,
metadata_valid=metadata_valid,
- reviewed=reviewed,
status=status,
)
- metas = await MetadataDatabaseFactory().get_metadata(
+ metas = await MetadataDatabaseFactory(database=database).get_metadata(
**query.datetime_dict(exclude={"id", "metadata_id"})
)
return metas
@@ -88,14 +78,14 @@ async def get_metadata(
@router.get("/metadata/{id}", response_model=Metadata)
async def get_metadata_by_id(id: int):
- return await MetadataDatabaseFactory().get_metadata_by_id(id=id)
+ return await MetadataDatabaseFactory(database=database).get_metadata_by_id(id=id)
@router.get("/metadata/{metadata_id}/history", response_model=List[Metadata])
async def get_metadata_history(
metadata_id: int,
):
- return await MetadataDatabaseFactory().get_metadata_history(
+ return await MetadataDatabaseFactory(database=database).get_metadata_history(
metadata_id=metadata_id,
)
@@ -106,6 +96,7 @@ async def update_metadata(
metadata: Metadata = Body(...),
user: User = Depends(require_user([os.getenv("REVIEWER_GROUP", "reviewer")])),
):
- return await MetadataDatabaseFactory().update_metadata(
- meta=metadata, username=user.nickname
+ return await MetadataDatabaseFactory(database=database).update_metadata(
+ meta=metadata,
+ updated_by=user.nickname,
)
diff --git a/geomagio/api/ws/metadata.py b/geomagio/api/ws/metadata.py
index 57b89cb30..1dc77ee2c 100644
--- a/geomagio/api/ws/metadata.py
+++ b/geomagio/api/ws/metadata.py
@@ -4,6 +4,7 @@ from fastapi import APIRouter
from obspy import UTCDateTime
from ...metadata import Metadata, MetadataCategory, MetadataQuery
+from ..db.common import database
from ..db import MetadataDatabaseFactory
router = APIRouter()
@@ -20,7 +21,6 @@ async def get_metadata(
location: str = None,
data_valid: bool = None,
metadata_valid: bool = True,
- reviewed: bool = None,
status: str = None,
):
query = MetadataQuery(
@@ -33,10 +33,9 @@ async def get_metadata(
location=location,
data_valid=data_valid,
metadata_valid=metadata_valid,
- reviewed=reviewed,
status=status,
)
- metas = await MetadataDatabaseFactory().get_metadata(
+ metas = await MetadataDatabaseFactory(database=database).get_metadata(
**query.datetime_dict(exclude={"id"})
)
return metas
diff --git a/geomagio/metadata/Metadata.py b/geomagio/metadata/Metadata.py
index b6bcb6ba0..dff97fbeb 100644
--- a/geomagio/metadata/Metadata.py
+++ b/geomagio/metadata/Metadata.py
@@ -72,8 +72,6 @@ class Metadata(BaseModel):
data_valid: bool = True
# whether metadata is valid (based on review)
metadata_valid: bool = True
- # whether metadata has been reviewed
- reviewed: bool = False
# metadata json blob
metadata: Dict = None
# general comment
diff --git a/geomagio/metadata/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
index 0558bcc20..e000c887f 100644
--- a/geomagio/metadata/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -27,15 +27,6 @@ class MetadataFactory(object):
def _get_headers(self):
return {"Authorization": self.token} if self.token else None
- def delete_metadata(self, metadata: Metadata) -> bool:
- response = requests.delete(
- url=f"{self.url}/{metadata.id}",
- headers=self._get_headers(),
- )
- if response.status_code == 200:
- return True
- return False
-
def get_metadata(self, query: MetadataQuery) -> List[Metadata]:
if query.id:
metadata = [self.get_metadata_by_id(id=query.id)]
diff --git a/geomagio/metadata/MetadataQuery.py b/geomagio/metadata/MetadataQuery.py
index 025c000b7..5b5214849 100644
--- a/geomagio/metadata/MetadataQuery.py
+++ b/geomagio/metadata/MetadataQuery.py
@@ -21,7 +21,6 @@ class MetadataQuery(BaseModel):
location: str = None
data_valid: Optional[bool] = None
metadata_valid: Optional[bool] = None
- reviewed: Optional[bool] = None
status: Optional[str] = None
def datetime_dict(self, **kwargs):
diff --git a/geomagio/metadata/main.py b/geomagio/metadata/main.py
index 1b7c8da81..69e8eeecc 100644
--- a/geomagio/metadata/main.py
+++ b/geomagio/metadata/main.py
@@ -110,24 +110,6 @@ def create(
print(metadata.json())
-@app.command(
- help=f"""
- Delete an existing metadata.
-
- {ENVIRONMENT_VARIABLE_HELP}
- """
-)
-def delete(
- input_file: str,
- url: str = GEOMAG_API_URL,
-):
- metadata_dict = load_metadata(input_file=input_file)
- metadata = Metadata(**metadata_dict)
- deleted = MetadataFactory(url=url).delete_metadata(metadata=metadata)
- if not deleted:
- sys.exit(1)
-
-
@app.command(
help=f"""
Search existing metadata.
@@ -147,7 +129,6 @@ def get(
location: Optional[str] = None,
metadata_valid: Optional[bool] = None,
network: Optional[str] = None,
- reviewed: Optional[bool] = None,
status: Optional[str] = None,
starttime: Optional[str] = None,
station: Optional[str] = None,
@@ -164,7 +145,6 @@ def get(
location=location,
metadata_valid=metadata_valid,
network=network,
- reviewed=reviewed,
starttime=UTCDateTime(starttime) if starttime else None,
station=station,
status=status,
diff --git a/test_metadata.py b/test_metadata.py
index 27403f9b9..1a78b1bcf 100644
--- a/test_metadata.py
+++ b/test_metadata.py
@@ -176,7 +176,7 @@ test_metadata.append(
async def load_test_metadata():
await database.connect()
for meta in test_metadata:
- await MetadataDatabaseFactory().create_metadata(meta)
+ await MetadataDatabaseFactory(database=database).create_metadata(meta)
await database.disconnect()
--
GitLab