diff --git a/geomagio/api/db/MetadataDatabaseFactory.py b/geomagio/api/db/MetadataDatabaseFactory.py
index 83cdde6cffbbbe399b2ba583e17f6b6d83b43929..8782ea4bb679687a95a2d47363301f7014d32ac0 100644
--- a/geomagio/api/db/MetadataDatabaseFactory.py
+++ b/geomagio/api/db/MetadataDatabaseFactory.py
@@ -43,7 +43,6 @@ class MetadataDatabaseFactory(object):
channel,
location,
data_valid,
- metadata_valid,
status,
) = params.datetime_dict().values()
if id:
@@ -78,8 +77,6 @@ class MetadataDatabaseFactory(object):
query = query.where(table.c.created_time < created_before)
if data_valid is not None:
query = query.where(table.c.data_valid == data_valid)
- if metadata_valid is not None:
- query = query.where(table.c.metadata_valid == metadata_valid)
if status is not None:
query = query.where(table.c.status.in_(status))
rows = await self.database.fetch_all(query)
diff --git a/geomagio/api/db/metadata_table.py b/geomagio/api/db/metadata_table.py
index e55e66878b063583136afaa8a2de152c3a37e859..6b5716db50b2e96b9102f0c16f61a021884e6bee 100644
--- a/geomagio/api/db/metadata_table.py
+++ b/geomagio/api/db/metadata_table.py
@@ -38,8 +38,6 @@ metadata = Table(
Column("priority", Integer, default=1),
# whether data is valid (primarily for flags)
Column("data_valid", Boolean, default=True, index=True),
- # whether metadata is valid (based on review)
- Column("metadata_valid", Boolean, default=True, index=True),
# deletion status indicator
Column("status", String(length=255), nullable=True),
# metadata json blob
@@ -62,7 +60,6 @@ metadata = Table(
"starttime",
"endtime",
# valid
- "metadata_valid",
"data_valid",
"status",
),
diff --git a/geomagio/api/secure/metadata.py b/geomagio/api/secure/metadata.py
index ac8113acf98b353f8ef306ae727349527c209665..e5acb0849bc3f08cb00b3e6a8ed092adcfd4b6bd 100644
--- a/geomagio/api/secure/metadata.py
+++ b/geomagio/api/secure/metadata.py
@@ -40,7 +40,6 @@ def get_metadata_query(
channel: str = None,
location: str = None,
data_valid: bool = None,
- metadata_valid: bool = True,
status: List[str] = Query(None),
) -> MetadataQuery:
return MetadataQuery(
@@ -54,7 +53,6 @@ def get_metadata_query(
channel=channel,
location=location,
data_valid=data_valid,
- metadata_valid=metadata_valid,
status=status,
)
diff --git a/geomagio/api/ws/metadata.py b/geomagio/api/ws/metadata.py
index fb5627a0578702cdd2e763a4fe09e586ae1a51b9..1775569a434dda86c04a731494ea0060db1980d0 100644
--- a/geomagio/api/ws/metadata.py
+++ b/geomagio/api/ws/metadata.py
@@ -24,7 +24,6 @@ async def get_metadata(
channel: str = None,
location: str = None,
data_valid: bool = None,
- metadata_valid: bool = True,
status: List[str] = Query(None),
):
query = MetadataQuery(
@@ -36,7 +35,6 @@ async def get_metadata(
channel=channel,
location=location,
data_valid=data_valid,
- metadata_valid=metadata_valid,
status=status,
)
metas = await MetadataDatabaseFactory(database=database).get_metadata(
diff --git a/geomagio/metadata/Metadata.md b/geomagio/metadata/Metadata.md
index 12ff699040f55813bdaea8a5498964176aadc0fa..558d7a5dc1aaded4a5a75f099857cb29a1b5b483 100644
--- a/geomagio/metadata/Metadata.md
+++ b/geomagio/metadata/Metadata.md
@@ -22,8 +22,6 @@
* **reviewed**: after a record is reviewed by an approved user
* **deleted**: archive a record that is invalid or incomplete
-* **metadata_valid**: Whether record is valid and should be used in processing
-
* **review_comment**: Comments from reviewer about why record is valid/invalid
# Details:
diff --git a/geomagio/metadata/Metadata.py b/geomagio/metadata/Metadata.py
index 84f52c5e59477604f9a982a92d0c2cb00494b166..7c073317776baef7d8a08280aaeeaaa5989936f3 100644
--- a/geomagio/metadata/Metadata.py
+++ b/geomagio/metadata/Metadata.py
@@ -70,8 +70,6 @@ class Metadata(BaseModel):
priority: int = 1
# whether data is valid (primarily for flags)
data_valid: bool = True
- # whether metadata is valid (based on review)
- metadata_valid: bool = True
# metadata json blob
metadata: Dict = None
# general comment
diff --git a/geomagio/metadata/MetadataQuery.py b/geomagio/metadata/MetadataQuery.py
index f776f34cb2bd075345980273f66dedb5dbb41c6e..2b873795e05dbc2ce70255e854d34f7886f468e8 100644
--- a/geomagio/metadata/MetadataQuery.py
+++ b/geomagio/metadata/MetadataQuery.py
@@ -20,7 +20,6 @@ class MetadataQuery(BaseModel):
channel: str = None
location: str = None
data_valid: Optional[bool] = None
- metadata_valid: Optional[bool] = None
status: Optional[List[str]] = None
def datetime_dict(self, **kwargs):
diff --git a/geomagio/metadata/main.py b/geomagio/metadata/main.py
index c5546573e17d4e8d020a866ab038be14c326e75a..cf1cd9adca76d198c839739beae0d80bae37d836 100644
--- a/geomagio/metadata/main.py
+++ b/geomagio/metadata/main.py
@@ -80,7 +80,6 @@ def create(
id: int = None,
input_file: str = None,
location: str = None,
- metadata_valid: bool = True,
network: str = None,
starttime: str = None,
station: str = None,
@@ -102,7 +101,6 @@ def create(
id=id,
location=location,
metadata=input_metadata,
- metadata_valid=metadata_valid,
network=network,
starttime=UTCDateTime(starttime) if starttime else None,
station=station,
@@ -129,7 +127,6 @@ def get(
getone: bool = False,
id: Optional[int] = None,
location: Optional[str] = None,
- metadata_valid: Optional[bool] = None,
network: Optional[str] = None,
status: Optional[List[str]] = typer.Argument(None),
starttime: Optional[str] = None,
@@ -145,7 +142,6 @@ def get(
endtime=UTCDateTime(endtime) if endtime else None,
id=id,
location=location,
- metadata_valid=metadata_valid,
network=network,
starttime=UTCDateTime(starttime) if starttime else None,
station=station,
diff --git a/geomagio/processing/affine_matrix.py b/geomagio/processing/affine_matrix.py
index a3e8000d6ccd737cc08d847a5230ddd24a644041..51c8746a5f43c75485ddb0d64570dc6893518766 100644
--- a/geomagio/processing/affine_matrix.py
+++ b/geomagio/processing/affine_matrix.py
@@ -49,7 +49,6 @@ def generate_matrix(
endtime=readings_endtime,
category=MetadataCategory.READING,
data_valid=True,
- metadata_valid=True,
)
)
readings = [Reading(**m.metadata) for m in metadata]
diff --git a/migrations/versions/4324b87e0b3c_drop_metadata_valid_column.py b/migrations/versions/4324b87e0b3c_drop_metadata_valid_column.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bf200c01657d8352aa574f9ef8d52ea9434c4ee
--- /dev/null
+++ b/migrations/versions/4324b87e0b3c_drop_metadata_valid_column.py
@@ -0,0 +1,85 @@
+"""drop metadata_valid column
+
+Revision ID: 4324b87e0b3c
+Revises: 2280fe551e60
+Create Date: 2021-07-14 15:48:32.154022
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "4324b87e0b3c"
+down_revision = "2280fe551e60"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_index("ix_metadata_metadata_valid", table_name="metadata")
+ op.drop_index("index_station_metadata", table_name="metadata")
+ op.create_index(
+ "index_station_metadata",
+ "metadata",
+ [
+ "network",
+ "station",
+ "channel",
+ "location",
+ "category",
+ "starttime",
+ "endtime",
+ "data_valid",
+ "status",
+ ],
+ unique=False,
+ )
+ op.drop_column("metadata", "metadata_valid")
+ op.drop_column("metadata_history", "metadata_valid")
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column(
+ "metadata_history",
+ sa.Column(
+ "metadata_valid",
+ mysql.TINYINT(display_width=1),
+ autoincrement=False,
+ nullable=True,
+ ),
+ )
+ op.add_column(
+ "metadata",
+ sa.Column(
+ "metadata_valid",
+ mysql.TINYINT(display_width=1),
+ autoincrement=False,
+ nullable=True,
+ ),
+ )
+ op.drop_index("index_station_metadata", table_name="metadata")
+ op.create_index(
+ "index_station_metadata",
+ "metadata",
+ [
+ "network",
+ "station",
+ "channel",
+ "location",
+ "category",
+ "starttime",
+ "endtime",
+ "metadata_valid",
+ "data_valid",
+ "status",
+ ],
+ unique=False,
+ )
+ op.create_index(
+ "ix_metadata_metadata_valid", "metadata", ["metadata_valid"], unique=False
+ )
+ # ### end Alembic commands ###
diff --git a/test_metadata.py b/test_metadata.py
index 1a78b1bcfc571d683a27243342301bd0d8bb3558..2a975f896f4469d831bf33588eac9539d0f97dda 100644
--- a/test_metadata.py
+++ b/test_metadata.py
@@ -142,7 +142,6 @@ for reading in readings:
endtime=reading.time,
station=reading.metadata["station"],
metadata=reading_dict,
- metadata_valid=reading.valid,
)
)