Refactor database factory, change delete to update, add status flag

geomagio/api/db/MetadataDatabaseFactory.py

 from datetime import datetime
-from enum import Enum
-from typing import List, Union
+from typing import List
 
-from fastapi import Response
 from obspy import UTCDateTime
 from sqlalchemy import or_, Table
 
@@ -12,11 +10,6 @@ from .metadata_history_table import metadata_history
 from .metadata_table import metadata
 
 
-class TableType(str, Enum):
-    METADATA = "metadata"
-    HISTORY = "history"
-
-
 class MetadataDatabaseFactory(object):
     def __init__(
         self, table: Table = metadata, history_table: Table = metadata_history
@@ -24,38 +17,20 @@ class MetadataDatabaseFactory(object):
         self.table = table
         self.history_table = history_table
 
-    async def create_metadata(
-        self, meta: Metadata, table: TableType = TableType.METADATA
-    ) -> Metadata:
-        exclude = {"id"}
-        if table is TableType.HISTORY:
-            query = self.history_table.insert()
-            meta.metadata_id = meta.id
-        else:
-            query = self.table.insert()
-            exclude.add("metadata_id")
-        values = meta.datetime_dict(exclude=exclude, exclude_none=True)
+    async def create_metadata(self, meta: Metadata) -> Metadata:
+        query = metadata.insert()
+        values = meta.datetime_dict(exclude={"id", "metadata_id"}, exclude_none=True)
         query = query.values(**values)
         meta.id = await database.execute(query)
         return meta
 
-    async def delete_metadata(self, id: int) -> None:
-        query = self.table.delete().where(self.table.c.id == id)
-        await database.execute(query)
-
-    async def get_metadata_by_id(
-        self, id: int, table: TableType = TableType.METADATA
-    ) -> Union[Metadata, List[Metadata]]:
-        if table == TableType.HISTORY:
-            query = self.history_table.select()
-            query = query.where(self.history_table.c.metadata_id == id)
-            rows = await database.fetch_all(query)
-            return [Metadata(**row) for row in rows]
-        meta = await self.get_metadata(id=id)
-        if len(meta) != 1:
-            return Response(status_code=404)
-        else:
-            return meta[0]
+    async def create_metadata_history(self, meta: Metadata) -> Metadata:
+        query = self.history_table.insert()
+        meta.metadata_id = meta.id
+        values = meta.datetime_dict(exclude={"id"}, exclude_none=True)
+        query = query.values(**values)
+        meta.id = await database.execute(query)
+        return meta
 
     async def get_metadata(
         self,
@@ -73,7 +48,8 @@ class MetadataDatabaseFactory(object):
         data_valid: bool = None,
         metadata_valid: bool = None,
         reviewed: bool = None,
-    ) -> List[Metadata]:
+        status: str = None,
+    ):
         table = self.table
         query = table.select()
         if id:
@@ -90,11 +66,17 @@ class MetadataDatabaseFactory(object):
             query = query.where(table.c.location.like(location))
         if starttime:
             query = query.where(
-                or_(table.c.endtime == None, table.c.endtime > starttime)
+                or_(
+                    table.c.endtime == None,
+                    table.c.endtime > starttime,
+                )
             )
         if endtime:
             query = query.where(
-                or_(table.c.starttime == None, table.c.starttime < endtime)
+                or_(
+                    table.c.starttime == None,
+                    table.c.starttime < endtime,
+                )
             )
         if created_after:
             query = query.where(table.c.created_time > created_after)
@@ -106,21 +88,39 @@ class MetadataDatabaseFactory(object):
             query = query.where(table.c.metadata_valid == metadata_valid)
         if reviewed is not None:
             query = query.where(table.c.reviewed == reviewed)
+        if status is not None:
+            query = query.where(table.c.status == status)
         rows = await database.fetch_all(query)
         return [Metadata(**row) for row in rows]
 
+    async def get_metadata_by_id(self, id: int):
+        meta = await self.get_metadata(id=id)
+        if len(meta) != 1:
+            raise ValueError(f"{len(meta)} records found")
+        return meta[0]
+
+    async def get_metadata_history(self, metadata_id: int) -> List[Metadata]:
+        query = self.history_table.select()
+        query = query.where(self.history_table.c.metadata_id == metadata_id)
+        rows = await database.fetch_all(query)
+        metadata = [Metadata(**row) for row in rows]
+        current_metadata = await self.get_metadata_by_id(id=metadata_id)
+        metadata.append(current_metadata)
+        # return records in order of age
+        metadata.reverse()
+        return metadata
+
     async def update_metadata(
-        self,
-        meta: Metadata,
-        username: str,
+        self, meta: Metadata, username: str, status: str = None
     ) -> Metadata:
-        original_metadata = await self.get_metadata_by_id(id=meta.id)
-        await self.create_metadata(meta=original_metadata, table=TableType.HISTORY)
-        meta.updated_by = username
-        meta.updated_time = UTCDateTime()
-        query = self.table.update().where(self.table.c.id == meta.id)
-        values = meta.datetime_dict(exclude={"id", "metadata_id"})
-        query = query.values(**values)
-        await database.execute(query)
-        updated_metadata = await self.get_metadata_by_id(id=meta.id)
-        return updated_metadata
+        async with database.transaction() as transaction:
+            original_metadata = await self.get_metadata_by_id(id=meta.id)
+            await self.create_metadata_history(meta=original_metadata)
+            meta.updated_by = username
+            meta.updated_time = UTCDateTime()
+            meta.status = status or meta.status
+            query = self.table.update().where(self.table.c.id == meta.id)
+            values = meta.datetime_dict(exclude={"id", "metadata_id"})
+            query = query.values(**values)
+            await database.execute(query)
+        return await self.get_metadata_by_id(id=meta.id)

geomagio/api/db/metadata_table.py

@@ -42,6 +42,8 @@ metadata = Table(
     Column("metadata_valid", Boolean, default=True, index=True),
     # whether metadata has been reviewed
     Column("reviewed", Boolean, default=True, index=True),
+    # deletion status indicator
+    Column("status", String(length=255), nullable=True),
     # metadata json blob
     Column("metadata", JSON, nullable=True),
     # general comment
@@ -65,6 +67,7 @@ metadata = Table(
         "metadata_valid",
         "data_valid",
         "reviewed",
+        "status",
     ),
     Index(
         "index_category_time",

geomagio/api/secure/metadata.py

@@ -34,7 +34,7 @@ async def create_metadata(
     metadata: Metadata,
     user: User = Depends(require_user()),
 ):
-    metadata = await MetadataDatabaseFactory().create_metadata(metadata)
+    metadata = await MetadataDatabaseFactory().create_metadata(meta=metadata)
     return Response(metadata.json(), status_code=201, media_type="application/json")
 
 
@@ -42,14 +42,11 @@ async def create_metadata(
 async def delete_metadata(
     id: int, user: User = Depends(require_user([os.getenv("ADMIN_GROUP", "admin")]))
 ):
-    await MetadataDatabaseFactory().delete_metadata(id)
-
-
-@router.get("/metadata/{id}/history", response_model=List[Metadata])
-async def get_metadata_history(
-    id: int,
-):
-    return await MetadataDatabaseFactory().get_metadata_by_id(id=id, table="history")
+    database_factory = MetadataDatabaseFactory()
+    metadata = await database_factory.get_metadata_by_id(id=id)
+    await database_factory.update_metadata(
+        meta=metadata, username=user.nickname, status="deleted"
+    )
 
 
 @router.get("/metadata", response_model=List[Metadata])
@@ -66,6 +63,7 @@ async def get_metadata(
     data_valid: bool = None,
     metadata_valid: bool = True,
     reviewed: bool = None,
+    status: str = None,
 ):
     query = MetadataQuery(
         category=category,
@@ -80,9 +78,10 @@ async def get_metadata(
         data_valid=data_valid,
         metadata_valid=metadata_valid,
         reviewed=reviewed,
+        status=status,
     )
     metas = await MetadataDatabaseFactory().get_metadata(
-        **query.datetime_dict(exclude={"id"})
+        **query.datetime_dict(exclude={"id", "metadata_id"})
     )
     return metas
 
@@ -92,6 +91,15 @@ async def get_metadata_by_id(id: int):
     return await MetadataDatabaseFactory().get_metadata_by_id(id=id)
 
 
+@router.get("/metadata/{metadata_id}/history", response_model=List[Metadata])
+async def get_metadata_history(
+    metadata_id: int,
+):
+    return await MetadataDatabaseFactory().get_metadata_history(
+        metadata_id=metadata_id,
+    )
+
+
 @router.put("/metadata/{id}", response_model=Metadata)
 async def update_metadata(
     id: int,
@@ -99,6 +107,5 @@ async def update_metadata(
     user: User = Depends(require_user([os.getenv("REVIEWER_GROUP", "reviewer")])),
 ):
     return await MetadataDatabaseFactory().update_metadata(
-        meta=metadata,
-        username=user.nickname,
+        meta=metadata, username=user.nickname
     )

geomagio/api/ws/metadata.py

@@ -21,6 +21,7 @@ async def get_metadata(
     data_valid: bool = None,
     metadata_valid: bool = True,
     reviewed: bool = None,
+    status: str = None,
 ):
     query = MetadataQuery(
         category=category,
@@ -33,6 +34,7 @@ async def get_metadata(
         data_valid=data_valid,
         metadata_valid=metadata_valid,
         reviewed=reviewed,
+        status=status,
     )
     metas = await MetadataDatabaseFactory().get_metadata(
         **query.datetime_dict(exclude={"id"})

geomagio/metadata/Metadata.py

@@ -80,6 +80,8 @@ class Metadata(BaseModel):
     comment: str = None
     # review specific comment
     review_comment: str = None
+    # deletion status indicator
+    status: str = None
 
     def datetime_dict(self, **kwargs):
         values = self.dict(**kwargs)

geomagio/metadata/MetadataQuery.py

@@ -22,6 +22,7 @@ class MetadataQuery(BaseModel):
     data_valid: Optional[bool] = None
     metadata_valid: Optional[bool] = None
     reviewed: Optional[bool] = None
+    status: Optional[str] = None
 
     def datetime_dict(self, **kwargs):
         values = self.dict(**kwargs)

geomagio/metadata/main.py

@@ -148,6 +148,7 @@ def get(
     metadata_valid: Optional[bool] = None,
     network: Optional[str] = None,
     reviewed: Optional[bool] = None,
+    status: Optional[str] = None,
     starttime: Optional[str] = None,
     station: Optional[str] = None,
     url: str = GEOMAG_API_URL,
@@ -166,6 +167,7 @@ def get(
         reviewed=reviewed,
         starttime=UTCDateTime(starttime) if starttime else None,
         station=station,
+        status=status,
     )
     metadata = MetadataFactory(url=url).get_metadata(query=query)
     if getone: