From 448708a8a8cdd3269e81bf72dce29da392e9a553 Mon Sep 17 00:00:00 2001
From: spencer <swilbur@usgs.gov>
Date: Tue, 24 Sep 2024 15:06:44 -0600
Subject: [PATCH 1/3] Adding a few changes to FDSNFActory regarding the
FDSNNoDataException. I also created a private function to contain the logic
that removes sensitivity/response info for traces. I added all avaiable
stations that transmit geomgagnetic data for networks, IU, N4, and US to the
VariometerMetadata.py module.
---
geomagio/VariometerMetadata.py | 202 +++++++++++++++++++++++++++++----
geomagio/edge/FDSNFactory.py | 80 +++++++------
2 files changed, 221 insertions(+), 61 deletions(-)
diff --git a/geomagio/VariometerMetadata.py b/geomagio/VariometerMetadata.py
index cdffb271..3dfab0e6 100644
--- a/geomagio/VariometerMetadata.py
+++ b/geomagio/VariometerMetadata.py
@@ -30,8 +30,8 @@ DEFAULT_ASL_METADATA = {
"ANMO": {
"metadata": {
"elevation": 1820,
- "geodetic_latitude": 34.946,
- "geodetic_longitude": -106.457,
+ "geodetic_latitude": 34.94591,
+ "geodetic_longitude": -106.4572,
"station_name": "Albuquerque",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
@@ -41,8 +41,8 @@ DEFAULT_ASL_METADATA = {
"CASY": {
"metadata": {
"elevation": 10,
- "geodetic_latitude": -66.279,
- "geodetic_longitude": 110.535,
+ "geodetic_latitude": -66.2792,
+ "geodetic_longitude": 110.5354,
"station_name": "Casey",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
@@ -52,8 +52,8 @@ DEFAULT_ASL_METADATA = {
"COLA": {
"metadata": {
"elevation": 200,
- "geodetic_latitude": 64.874,
- "geodetic_longitude": -147.862,
+ "geodetic_latitude": 64.8736,
+ "geodetic_longitude": -147.8616,
"station_name": "College Outpost",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
@@ -63,20 +63,31 @@ DEFAULT_ASL_METADATA = {
"COR": {
"metadata": {
"elevation": 110,
- "geodetic_latitude": 44.586,
- "geodetic_longitude": -123.305,
+ "geodetic_latitude": 44.5855,
+ "geodetic_longitude": -123.3046,
"station_name": "Corvallis",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
},
"interval_specific": DEFAULT_INTERVAL_SPECIFIC,
},
- "QSPA": {
+ "KBS": {
"metadata": {
- "elevation": 2850,
- "geodetic_latitude": -89.929,
- "geodetic_longitude": 144.438,
- "station_name": "South Pole Remote Earth Science Observatory",
+ "elevation": 90,
+ "geodetic_latitude": 78.9154,
+ "geodetic_longitude": 11.9385,
+ "station_name": "Ny-Alesund",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "KONO": {
+ "metadata": {
+ "elevation": 558,
+ "geodetic_latitude": 59.6521,
+ "geodetic_longitude": 9.5946,
+ "station_name": "Kongsberg",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
},
@@ -85,8 +96,8 @@ DEFAULT_ASL_METADATA = {
"RSSD": {
"metadata": {
"elevation": 2090,
- "geodetic_latitude": 44.121,
- "geodetic_longitude": -104.036,
+ "geodetic_latitude": 44.1212,
+ "geodetic_longitude": -104.0359,
"station_name": "Black Hills",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
@@ -96,8 +107,8 @@ DEFAULT_ASL_METADATA = {
"SBA": {
"metadata": {
"elevation": 50,
- "geodetic_latitude": -77.849,
- "geodetic_longitude": 166.757,
+ "geodetic_latitude": -77.8492,
+ "geodetic_longitude": 166.7572,
"station_name": "Scott Base",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
@@ -107,20 +118,163 @@ DEFAULT_ASL_METADATA = {
"SFJD": {
"metadata": {
"elevation": 330,
- "geodetic_latitude": 66.996,
- "geodetic_longitude": -50.621,
+ "geodetic_latitude": 66.9961,
+ "geodetic_longitude": -50.6207,
"station_name": "Sondre Stromfjord",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
},
"interval_specific": DEFAULT_INTERVAL_SPECIFIC,
},
- "SPA": {
+ "SSPA": {
+ "metadata": {
+ "elevation": 270,
+ "geodetic_latitude": 40.6358,
+ "geodetic_longitude": -77.8876,
+ "station_name": "Standing Stone, Pennsylvania",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "QSPA": {
+ "metadata": {
+ "elevation": 2850,
+ "geodetic_latitude": -89.9289,
+ "geodetic_longitude": 144.4382,
+ "station_name": "South Pole Remote Earth Science Observatory",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "E62A": {
+ "metadata": {
+ "elevation": 365,
+ "geodetic_latitude": 46.6201,
+ "geodetic_longitude": -69.5227,
+ "station_name": "Eastport",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "E46A": {
+ "metadata": {
+ "elevation": 269,
+ "geodetic_latitude": 46.3665,
+ "geodetic_longitude": -84.3062,
+ "station_name": "Lake Superior",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "PAB": {
+ "metadata": {
+ "elevation": 950,
+ "geodetic_latitude": 39.5446,
+ "geodetic_longitude": -4.3499,
+ "station_name": "San Pablo, Spain",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "WCI": {
+ "metadata": {
+ "elevation": 210,
+ "geodetic_latitude": 38.2289,
+ "geodetic_longitude": -86.2939,
+ "station_name": "Wyandotte Cave, Indiana, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "KSU1": {
+ "metadata": {
+ "elevation": 317,
+ "geodetic_latitude": 39.1009,
+ "geodetic_longitude": -96.6094,
+ "station_name": "Kansas State University",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "EYMN": {
+ "metadata": {
+ "elevation": 475,
+ "geodetic_latitude": 47.9462,
+ "geodetic_longitude": -91.4953,
+ "station_name": "Ely, Minnesota, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "GOGA": {
+ "metadata": {
+ "elevation": 150,
+ "geodetic_latitude": 33.41476,
+ "geodetic_longitude": -83.47327,
+ "station_name": "Godfrey, GA, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "DGMT": {
+ "metadata": {
+ "elevation": 646,
+ "geodetic_latitude": 48.4702,
+ "geodetic_longitude": -104.1959,
+ "station_name": "Dagmar Montana, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "DWPF": {
+ "metadata": {
+ "elevation": 30,
+ "geodetic_latitude": 28.1103,
+ "geodetic_longitude": -81.4327,
+ "station_name": "Disney Wilderness Preserve, Florida, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "ECSD": {
+ "metadata": {
+ "elevation": 478,
+ "geodetic_latitude": 43.7337,
+ "geodetic_longitude": -96.6141,
+ "station_name": "EROS Data Center, Sioux Falls, South Dakota, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "HRV": {
+ "metadata": {
+ "elevation": 200,
+ "geodetic_latitude": 42.5064,
+ "geodetic_longitude": -71.5583,
+ "station_name": "Adam Dziewonski Observatory (Oak Ridge), Massachusetts, USA",
+ "agency_name": "United States Geological Survey (USGS)",
+ "declination_base": None,
+ },
+ "interval_specific": DEFAULT_INTERVAL_SPECIFIC,
+ },
+ "O20A": {
"metadata": {
- "elevation": 2927,
- "geodetic_latitude": -90.0,
- "geodetic_longitude": 0.0,
- "station_name": "South Pole",
+ "elevation": 1915,
+ "geodetic_latitude": 40.1348,
+ "geodetic_longitude": -108.2416,
+ "station_name": "White River City, CO, USA",
"agency_name": "United States Geological Survey (USGS)",
"declination_base": None,
},
diff --git a/geomagio/edge/FDSNFactory.py b/geomagio/edge/FDSNFactory.py
index afd9de17..b4109d4d 100644
--- a/geomagio/edge/FDSNFactory.py
+++ b/geomagio/edge/FDSNFactory.py
@@ -256,6 +256,13 @@ class FDSNFactory(TimeseriesFactory):
endtime=endtime + half_delta,
attach_response=True,
)
+ # Perfrom calibrations to properly remove response/sensitivity
+ data = self._calibrate_sensitivty_and_response(data)
+
+ if channel in channel_rotations:
+ data = self._rotate_and_return_requested_channel(
+ data, sncl, starttime, endtime, channel
+ )
except FDSNNoDataException:
data = Stream()
@@ -282,43 +289,8 @@ class FDSNFactory(TimeseriesFactory):
TimeseriesUtility.pad_and_trim_trace(
trace=data[0], starttime=starttime, endtime=endtime
)
- if channel in channel_rotations:
- data = self._rotate_and_return_requested_channel(
- data, sncl, starttime, endtime, channel
- )
-
- for trace in data:
- if hasattr(trace.stats, "response") and trace.stats.response is not None:
- response = trace.stats.response
- if self.remove_sensitivity is not None:
- if self.remove_sensitivity_flag == "remove_sensitivity":
- trace.remove_sensitivity()
- elif self.remove_sensitivity_flag == "remove_response":
- trace.remove_response()
- else:
- raise ValueError(
- f"Warning: Unrecognized remove_sensitivity value '{self.remove_sensitivity}'. No sensitivity or response removal applied."
- )
- else:
- if response.instrument_sensitivity:
- if response.instrument_sensitivity.input_units == "T":
- # some ASL stations produce data in Teslas, not nanoteslas
- response.instrument_sensitivity.value /= 1e9
- response.instrument_sensitivity.input_units = "nT"
- # apply total gain/sensitivity, ignore any frequency response
- trace.remove_sensitivity()
- else:
- # according to fdsn docs, if no instrument_sensitivity,
- # the response must be an instrument_polynomial
- # https://docs.fdsn.org/projects/stationxml/en/latest/reference.html#instrumentsensitivity
-
- # Geomag Program observatories use 2nd order instrument_polynomial
- # to capture both offset and scale, but no frequency response
- data.remove_response()
-
- else:
- self._set_metadata(data, observatory, channel, type, interval)
+ self._set_metadata(data, observatory, channel, type, interval)
return data
@@ -403,7 +375,7 @@ class FDSNFactory(TimeseriesFactory):
# Initialize FDSN client and get the necessary data
FDSN = self.Client
- # Determine if the requested channel is X, Y, or Z
+ # Determine if the requested channel is X, Y, or
inv = FDSN.get_stations(
network=sncl.network,
@@ -436,3 +408,37 @@ class FDSNFactory(TimeseriesFactory):
return Stream()
return rotated_stream
+
+ def _calibrate_sensitivty_and_response(self, data: Stream) -> Stream:
+
+ for trace in data:
+ if hasattr(trace.stats, "response") and trace.stats.response is not None:
+ response = trace.stats.response
+ if self.remove_sensitivity is not None:
+ if self.remove_sensitivity_flag == "remove_sensitivity":
+ trace.remove_sensitivity()
+
+ elif self.remove_sensitivity_flag == "remove_response":
+ trace.remove_response()
+ else:
+ raise ValueError(
+ f"Warning: Unrecognized remove_sensitivity value '{self.remove_sensitivity}'. No sensitivity or response removal applied."
+ )
+ else:
+ if response.instrument_sensitivity:
+ if response.instrument_sensitivity.input_units == "T":
+ # some ASL stations produce data in Teslas, not nanoteslas
+ response.instrument_sensitivity.value /= 1e9
+ response.instrument_sensitivity.input_units = "nT"
+ # apply total gain/sensitivity, ignore any frequency response
+ trace.remove_sensitivity()
+
+ else:
+ # according to fdsn docs, if no instrument_sensitivity,
+ # the response must be an instrument_polynomial
+ # https://docs.fdsn.org/projects/stationxml/en/latest/reference.html#instrumentsensitivity
+
+ # Geomag Program observatories use 2nd order instrument_polynomial
+ # to capture both offset and scale, but no frequency response
+ data.remove_response()
+ return data
--
GitLab
From f98589290b27c09df9a760ec995568ea25c6ae4c Mon Sep 17 00:00:00 2001
From: spencer <swilbur@usgs.gov>
Date: Thu, 26 Sep 2024 09:02:57 -0600
Subject: [PATCH 2/3] I removed an unecessary call to FilterApiQuery from
within Data.py. I also added validation to the FilterApiQuery.py. Rather than
using super I opted to add the validation to this file beacuse
sampling_period was ignored from the base class i.e. DataApiQuery and this is
why the REQUEST_LIMIT was being ignored.
---
geomagio/api/ws/DataApiQuery.py | 12 +-----------
geomagio/api/ws/FilterApiQuery.py | 26 +++++++++++++++++++++++++-
geomagio/api/ws/data.py | 1 -
3 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/geomagio/api/ws/DataApiQuery.py b/geomagio/api/ws/DataApiQuery.py
index a1716691..6cba83df 100644
--- a/geomagio/api/ws/DataApiQuery.py
+++ b/geomagio/api/ws/DataApiQuery.py
@@ -12,7 +12,7 @@ from .Observatory import OBSERVATORY_INDEX, ASL_OBSERVATORY_INDEX
DEFAULT_ELEMENTS = ["X", "Y", "Z", "F"]
-REQUEST_LIMIT = 345600
+REQUEST_LIMIT = 3456000 # Increased the request limit by 10x what was decided by Jeremy
VALID_ELEMENTS = [e.id for e in ELEMENTS]
@@ -149,13 +149,3 @@ class DataApiQuery(BaseModel):
raise ValueError(f"Request exceeds limit ({samples} > {REQUEST_LIMIT})")
# otherwise okay
return values
-
-
-# The new class inheriting everything except input/output_sampling period from DataApiQuery
-class FilterDataApiQuery(DataApiQuery):
- input_sampling_period: SamplingPeriod = SamplingPeriod.SECOND
- output_sampling_period: SamplingPeriod = SamplingPeriod.MINUTE
-
- # Remove inherited fields that we don't need
- class Config:
- fields = {"sampling_period": {"exclude": True}, "data_host": {"exclude": True}}
diff --git a/geomagio/api/ws/FilterApiQuery.py b/geomagio/api/ws/FilterApiQuery.py
index 2eb960ac..dcae69ac 100644
--- a/geomagio/api/ws/FilterApiQuery.py
+++ b/geomagio/api/ws/FilterApiQuery.py
@@ -1,4 +1,5 @@
-from .DataApiQuery import DataApiQuery, SamplingPeriod
+from .DataApiQuery import DataApiQuery, SamplingPeriod, REQUEST_LIMIT
+from pydantic import root_validator
"""This script contains the class inheriting everything except input/output_sampling
period from the DataApiQuery class. This is where more specific functionailty
@@ -12,3 +13,26 @@ class FilterApiQuery(DataApiQuery):
# Remove inherited fields that we don't need for this specific endpoint
class Config:
fields = {"sampling_period": {"exclude": True}}
+
+ @root_validator
+ def validate_combinations(cls, values):
+ starttime, endtime, elements, format, input_sampling_period = (
+ values.get("starttime"),
+ values.get("endtime"),
+ values.get("elements"),
+ values.get("format"),
+ values.get("input_sampling_period"),
+ )
+ if len(elements) > 4 and format == "iaga2002":
+ raise ValueError("No more than four elements allowed for iaga2002 format.")
+ if starttime > endtime:
+ raise ValueError("Starttime must be before endtime.")
+
+ # Calculate the number of samples based on the input sampling period
+ samples = int(len(elements) * (endtime - starttime) / input_sampling_period)
+
+ # Validate the request size
+ if samples > REQUEST_LIMIT:
+ raise ValueError(f"Request exceeds limit ({samples} > {REQUEST_LIMIT})")
+
+ return values
diff --git a/geomagio/api/ws/data.py b/geomagio/api/ws/data.py
index 3cde0746..40a09da6 100644
--- a/geomagio/api/ws/data.py
+++ b/geomagio/api/ws/data.py
@@ -13,7 +13,6 @@ from .Observatory import ASL_OBSERVATORY_INDEX
from .DataApiQuery import (
DEFAULT_ELEMENTS,
DataApiQuery,
- FilterDataApiQuery,
DataType,
OutputFormat,
SamplingPeriod,
--
GitLab
From 7ed080e4c0b3ebf94994dd8aecbc140a2f545fae Mon Sep 17 00:00:00 2001
From: spencer <swilbur@usgs.gov>
Date: Thu, 26 Sep 2024 09:24:54 -0600
Subject: [PATCH 3/3] Updated data_test.py to not retrieve FilterApiQuery from
DataApiQuery. Honestly not sure how these are just now causing issues when
all my previous commits have contained this code.
---
test/api_test/ws_test/data_test.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/test/api_test/ws_test/data_test.py b/test/api_test/ws_test/data_test.py
index f56f7429..e0b731ef 100644
--- a/test/api_test/ws_test/data_test.py
+++ b/test/api_test/ws_test/data_test.py
@@ -10,7 +10,6 @@ from geomagio.api.ws.DataApiQuery import (
DataApiQuery,
OutputFormat,
SamplingPeriod,
- FilterDataApiQuery,
)
--
GitLab