From 5bbe3aa84f6705faf7f58e2abb06eefbc5864c35 Mon Sep 17 00:00:00 2001
From: bgeels <bgeels@usgs.gov>
Date: Wed, 7 Aug 2024 10:28:35 -0600
Subject: [PATCH 1/5] Add metadatafactory docstrings
---
geomagio/metadata/MetadataFactory.py | 76 ++++++++++++++++++++++++++--
1 file changed, 71 insertions(+), 5 deletions(-)
diff --git a/geomagio/metadata/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
index 92b44975..9da66684 100644
--- a/geomagio/metadata/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -9,7 +9,7 @@ from pydantic import parse_obj_as
from .Metadata import Metadata
from .MetadataQuery import MetadataQuery
-
+# Set the API host and URL for the Geomagnetic Metadata service
GEOMAG_API_HOST = os.getenv("GEOMAG_API_HOST", "geomag.usgs.gov")
GEOMAG_API_URL = f"https://{GEOMAG_API_HOST}/ws/secure/metadata"
if "127.0.0.1" in GEOMAG_API_URL:
@@ -17,15 +17,36 @@ if "127.0.0.1" in GEOMAG_API_URL:
class MetadataFactory(object):
+ """
+ A factory class for interacting with metadata servers.
+
+ Attributes:
+ url (str): The base URL for the metadata server.
+ token (str): The authorization token for accessing the API.
+ """
+
def __init__(
self,
url: str = GEOMAG_API_URL,
token: str = os.getenv("GITLAB_API_TOKEN"),
):
+ """
+ Initializes the MetadataFactory with the specified URL and token.
+
+ Args:
+ url (str): The base URL for the metadata server (default is GEOMAG_API_URL).
+ token (str): The authorization token (default is fetched from environment).
+ """
self.url = url
self.token = token
def _get_headers(self):
+ """
+ Constructs the headers for the API request.
+
+ Returns:
+ dict: A dictionary containing the authorization token and content type.
+ """
return (
{"Authorization": self.token, "content-type": "application/json"}
if self.token
@@ -33,6 +54,15 @@ class MetadataFactory(object):
)
def get_metadata(self, query: MetadataQuery) -> List[Metadata]:
+ """
+ Retrieves metadata based on the provided query.
+
+ Args:
+ query (MetadataQuery): The query object containing search parameters.
+
+ Returns:
+ List[Metadata]: A list of Metadata objects matching the query.
+ """
if query.id:
metadata = [self.get_metadata_by_id(id=query.id)]
else:
@@ -43,7 +73,7 @@ class MetadataFactory(object):
)
metadata = parse_obj_as(List[Metadata], response.json())
- # if metadata dict() provided in query, only include Metadata objects with matching key-value pairs
+ # Filter metadata based on additional criteria if provided
if query.metadata is not None:
query_metadata_items = query.metadata.items()
metadata_list = metadata
@@ -60,6 +90,15 @@ class MetadataFactory(object):
return metadata
def get_metadata_by_id(self, id: int) -> Metadata:
+ """
+ Retrieves metadata for a specific ID.
+
+ Args:
+ id (int): The ID of the metadata to retrieve.
+
+ Returns:
+ Metadata: The Metadata object corresponding to the given ID.
+ """
response = requests.get(
url=f"{self.url}/{id}",
headers=self._get_headers(),
@@ -67,6 +106,15 @@ class MetadataFactory(object):
return Metadata(**response.json())
def create_metadata(self, metadata: Metadata) -> Metadata:
+ """
+ Creates new metadata objects in the database.
+
+ Args:
+ metadata (Metadata): The Metadata object to create.
+
+ Returns:
+ Metadata: The created Metadata object.
+ """
response = requests.post(
url=self.url,
data=metadata.json(),
@@ -75,6 +123,15 @@ class MetadataFactory(object):
return Metadata(**response.json())
def update_metadata(self, metadata: Metadata) -> Metadata:
+ """
+ Updates existing metadata in the system.
+
+ Args:
+ metadata (Metadata): The Metadata object to update.
+
+ Returns:
+ Metadata: The updated Metadata object.
+ """
response = requests.put(
url=f"{self.url}/{metadata.id}",
data=metadata.json(),
@@ -84,17 +141,26 @@ class MetadataFactory(object):
def parse_params(query: MetadataQuery) -> str:
+ """
+ Parses the query object into a dictionary of parameters for the API request.
+
+ Args:
+ query (MetadataQuery): The query object containing search parameters.
+
+ Returns:
+ dict: A dictionary of parameters formatted for the API request.
+ """
query = query.dict(exclude_none=True)
args = {}
for key in query.keys():
element = query[key]
- # convert times to strings
+ # Convert times to ISO format strings
if isinstance(element, UTCDateTime):
element = element.isoformat()
- # serialize the metadata dictionary to a JSON string
+ # Serialize the metadata dictionary to a JSON string
elif key == "metadata" and isinstance(element, dict):
args[key] = json.dumps(element)
- # get string value of metadata category
+ # Get string value of metadata category
if key == "category":
element = element.value
args[key] = element
--
GitLab
From 903f0da98283eb388b32903fc57c418e6e84fbfe Mon Sep 17 00:00:00 2001
From: bgeels <bgeels@usgs.gov>
Date: Wed, 7 Aug 2024 10:29:02 -0600
Subject: [PATCH 2/5] Add timeouts to metadata factory
---
geomagio/metadata/MetadataFactory.py | 106 ++++++++++++++++++++-------
1 file changed, 78 insertions(+), 28 deletions(-)
diff --git a/geomagio/metadata/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
index 9da66684..2f718fe3 100644
--- a/geomagio/metadata/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -21,8 +21,11 @@ class MetadataFactory(object):
A factory class for interacting with metadata servers.
Attributes:
- url (str): The base URL for the metadata server.
- token (str): The authorization token for accessing the API.
+ ----------
+ url: str
+ The base URL for the metadata server.
+ token: str
+ The authorization token for accessing the API.
"""
def __init__(
@@ -33,9 +36,12 @@ class MetadataFactory(object):
"""
Initializes the MetadataFactory with the specified URL and token.
- Args:
- url (str): The base URL for the metadata server (default is GEOMAG_API_URL).
- token (str): The authorization token (default is fetched from environment).
+ Parameters:
+ -----------
+ url: str
+ The base URL for the metadata server (default is GEOMAG_API_URL).
+ token: str
+ The authorization token (default is fetched from environment).
"""
self.url = url
self.token = token
@@ -45,7 +51,9 @@ class MetadataFactory(object):
Constructs the headers for the API request.
Returns:
- dict: A dictionary containing the authorization token and content type.
+ --------
+ dict:
+ A dictionary containing the authorization token and content type.
"""
return (
{"Authorization": self.token, "content-type": "application/json"}
@@ -53,15 +61,25 @@ class MetadataFactory(object):
else None
)
- def get_metadata(self, query: MetadataQuery) -> List[Metadata]:
+ def get_metadata(
+ self, query: MetadataQuery, timeout: float | tuple = None
+ ) -> List[Metadata]:
"""
Retrieves metadata based on the provided query.
- Args:
- query (MetadataQuery): The query object containing search parameters.
+ Parameters:
+ -----------
+ query: MetadataQuery
+ The query object containing search parameters.
+ timeout: float or tuple, optional
+ The timeout for the API request.
+ Can be a single number (for both connection and read timeouts)
+ or a tuple (connection timeout, read timeout). Defaults to None.
Returns:
- List[Metadata]: A list of Metadata objects matching the query.
+ --------
+ metadata: List[Metadata]
+ A list of Metadata objects matching the query.
"""
if query.id:
metadata = [self.get_metadata_by_id(id=query.id)]
@@ -70,6 +88,7 @@ class MetadataFactory(object):
url=self.url,
headers=self._get_headers(),
params=parse_params(query=query),
+ timeout=timeout,
)
metadata = parse_obj_as(List[Metadata], response.json())
@@ -89,15 +108,23 @@ class MetadataFactory(object):
return metadata
- def get_metadata_by_id(self, id: int) -> Metadata:
+ def get_metadata_by_id(self, id: int, timeout: float | tuple = None) -> Metadata:
"""
Retrieves metadata for a specific ID.
- Args:
- id (int): The ID of the metadata to retrieve.
+ Parameters:
+ -----------
+ id: int
+ The ID of the metadata to retrieve.
+ timeout: float or tuple, optional
+ The timeout for the API request.
+ Can be a single number (for both connection and read timeouts)
+ or a tuple (connection timeout, read timeout). Defaults to None.
Returns:
- Metadata: The Metadata object corresponding to the given ID.
+ --------
+ Metadata: object
+ The Metadata object corresponding to the given ID.
"""
response = requests.get(
url=f"{self.url}/{id}",
@@ -105,14 +132,23 @@ class MetadataFactory(object):
)
return Metadata(**response.json())
- def create_metadata(self, metadata: Metadata) -> Metadata:
+ def create_metadata(
+ self, metadata: Metadata, timeout: float | tuple = None
+ ) -> Metadata:
"""
Creates new metadata objects in the database.
- Args:
- metadata (Metadata): The Metadata object to create.
+ Parameters:
+ -----------
+ metadata: Metadata
+ The Metadata object to create.
+ timeout: float or tuple, optional
+ The timeout for the API request.
+ Can be a floating-point number (for both connection and read timeouts)
+ or a tuple (connection timeout, read timeout). Defaults to None.
Returns:
+ --------
Metadata: The created Metadata object.
"""
response = requests.post(
@@ -122,15 +158,25 @@ class MetadataFactory(object):
)
return Metadata(**response.json())
- def update_metadata(self, metadata: Metadata) -> Metadata:
+ def update_metadata(
+ self, metadata: Metadata, timeout: float | tuple = None
+ ) -> Metadata:
"""
Updates existing metadata in the system.
- Args:
- metadata (Metadata): The Metadata object to update.
+ Parameters:
+ -----------
+ metadata: Metadata
+ The Metadata object to update.
+ timeout: float or tuple, optional
+ The timeout for the API request.
+ Can be a floating-point number (for both connection and read timeouts)
+ or a tuple (connection timeout, read timeout). Defaults to None.
Returns:
- Metadata: The updated Metadata object.
+ -----------
+ Metadata:
+ The updated Metadata object.
"""
response = requests.put(
url=f"{self.url}/{metadata.id}",
@@ -144,14 +190,18 @@ def parse_params(query: MetadataQuery) -> str:
"""
Parses the query object into a dictionary of parameters for the API request.
- Args:
- query (MetadataQuery): The query object containing search parameters.
+ Parameters:
+ -----------
+ query: MetadataQuery
+ The query object containing search parameters.
Returns:
- dict: A dictionary of parameters formatted for the API request.
+ --------
+ dict:
+ A dictionary of parameters formatted for the API request.
"""
query = query.dict(exclude_none=True)
- args = {}
+ Parameters = {}
for key in query.keys():
element = query[key]
# Convert times to ISO format strings
@@ -159,9 +209,9 @@ def parse_params(query: MetadataQuery) -> str:
element = element.isoformat()
# Serialize the metadata dictionary to a JSON string
elif key == "metadata" and isinstance(element, dict):
- args[key] = json.dumps(element)
+ Parameters[key] = json.dumps(element)
# Get string value of metadata category
if key == "category":
element = element.value
- args[key] = element
- return args
+ Parameters[key] = element
+ return Parameters
--
GitLab
From fe5946724a3c0ac7820d933088615499b51c00c6 Mon Sep 17 00:00:00 2001
From: bgeels <bgeels@usgs.gov>
Date: Wed, 7 Aug 2024 10:36:28 -0600
Subject: [PATCH 3/5] Add timeout to InstrumentCalibrations & fix docstrings
---
.../instrument/InstrumentCalibrations.py | 141 +++++++++---------
1 file changed, 72 insertions(+), 69 deletions(-)
diff --git a/geomagio/metadata/instrument/InstrumentCalibrations.py b/geomagio/metadata/instrument/InstrumentCalibrations.py
index 1bdc33d7..8dfcef48 100644
--- a/geomagio/metadata/instrument/InstrumentCalibrations.py
+++ b/geomagio/metadata/instrument/InstrumentCalibrations.py
@@ -14,33 +14,33 @@ class InstrumentCalibrations:
Attributes
----------
- metadata_list : list
- a list of metadata objects to be processed
- previous_calibration : dict
- a dictionary to store the previous calibration values for each axis and key
- station : str
- the station code, checked for consistency across all metadata objects
- location : str
- the location code, checked for consistency across all metadata objects
- network : str
- the network code, checked for consistency across all metadata objects
+ metadata_list : list
+ a list of metadata objects to be processed
+ previous_calibration : dict
+ a dictionary to store the previous calibration values for each axis and key
+ station : str
+ the station code, checked for consistency across all metadata objects
+ location : str
+ the location code, checked for consistency across all metadata objects
+ network : str
+ the network code, checked for consistency across all metadata objects
Methods
-------
- get_calibrations():
- Main method to gather applicable calibrations from the metadata list.
- create_overlap_interval(current_metadata):
- Creates an overlap interval from the current metadata.
- update_overlap_interval(overlap_interval, current_metadata):
- Updates the overlap interval with the current metadata.
- update_overlap_interval_with_same_starttime(i, sorted_metadata_list, overlap_interval):
- Updates the overlap interval with metadata that have the same starttime.
- set_endtime(i, sorted_metadata_list, overlap_interval, current_metadata):
- Sets the endtime for the overlap interval.
- convert_to_calibration(overlap_metadata):
- Converts overlapping metadata information to an applicable calibration.
- get_channels(overlap_metadata):
- Gets the channels for the applicable calibration.
+ get_calibrations():
+ Main method to gather applicable calibrations from the metadata list.
+ create_overlap_interval(current_metadata):
+ Creates an overlap interval from the current metadata.
+ update_overlap_interval(overlap_interval, current_metadata):
+ Updates the overlap interval with the current metadata.
+ update_overlap_interval_with_same_starttime(i, sorted_metadata_list, overlap_interval):
+ Updates the overlap interval with metadata that have the same starttime.
+ set_endtime(i, sorted_metadata_list, overlap_interval, current_metadata):
+ Sets the endtime for the overlap interval.
+ convert_to_calibration(overlap_metadata):
+ Converts overlapping metadata information to an applicable calibration.
+ get_channels(overlap_metadata):
+ Gets the channels for the applicable calibration.
"""
def __init__(self, metadata_list):
@@ -81,14 +81,14 @@ class InstrumentCalibrations:
Main method to compile applicable calibrations from the metadata list.
Parameters
- ----------
- query_starttime : UTCDateTime
- the query starttime
+ ----------
+ query_starttime : UTCDateTime
+ the query starttime
Returns
-------
- list
- a list of dictionaries, each representing an applicable calibration
+ list
+ a list of dictionaries, each representing an applicable calibration
"""
sorted_metadata_list = sorted(
self.metadata_list,
@@ -120,13 +120,13 @@ class InstrumentCalibrations:
Parameters
----------
- current_metadata : Metadata
- the current metadata object
+ current_metadata : Metadata
+ the current metadata object
Returns
-------
- dict
- a dictionary representing an overlap interval
+ dict
+ a dictionary representing an overlap interval
"""
overlap_interval = {"starttime": current_metadata.starttime}
self.update_overlap_interval(overlap_interval, current_metadata)
@@ -138,10 +138,10 @@ class InstrumentCalibrations:
Parameters
----------
- overlap_interval : dict
- the overlap interval to be updated
- current_metadata : Metadata
- the current metadata object
+ overlap_interval : dict
+ the overlap interval to be updated
+ current_metadata : Metadata
+ the current metadata object
"""
for axis in ["u", "v", "w"]:
for key in ["constant", "bin", "offset"]:
@@ -172,17 +172,17 @@ class InstrumentCalibrations:
Parameters
----------
- i : int
- the current index in the sorted metadata list
- sorted_metadata_list : list
- the sorted list of metadata objects
- overlap_interval : dict
- the overlap interval to be updated
+ i : int
+ the current index in the sorted metadata list
+ sorted_metadata_list : list
+ the sorted list of metadata objects
+ overlap_interval : dict
+ the overlap interval to be updated
Returns
-------
- int
- the updated index in the sorted metadata list
+ int
+ the updated index in the sorted metadata list
"""
while (
i + 1 < len(sorted_metadata_list)
@@ -200,14 +200,14 @@ class InstrumentCalibrations:
Parameters
----------
- i : int
- the current index in the sorted metadata list
- sorted_metadata_list : list
- the sorted list of metadata objects
- overlap_interval : dict
- the overlap interval to be updated
- current_metadata : Metadata
- the current metadata object
+ i : int
+ the current index in the sorted metadata list
+ sorted_metadata_list : list
+ the sorted list of metadata objects
+ overlap_interval : dict
+ the overlap interval to be updated
+ current_metadata : Metadata
+ the current metadata object
"""
if (
i + 1 < len(sorted_metadata_list)
@@ -223,13 +223,13 @@ class InstrumentCalibrations:
Parameters
----------
- overlap_metadata : dict
- the metadata overlap data to be converted
+ overlap_metadata : dict
+ the metadata overlap data to be converted
Returns
-------
- dict
- a dictionary representing an applicable calibration
+ dict
+ a dictionary representing an applicable calibration
"""
calibration = {
"network": self.network,
@@ -250,13 +250,13 @@ class InstrumentCalibrations:
Parameters
----------
- overlap_metadata : dict
- the metadata overlap data from which to get the channels
+ overlap_metadata : dict
+ the metadata overlap data from which to get the channels
Returns
-------
- dict
- a dictionary representing the channels for the applicable calibration
+ dict
+ a dictionary representing the channels for the applicable calibration
"""
channels = {}
for axis in ["U", "V", "W"]:
@@ -304,14 +304,17 @@ def get_instrument_calibrations(
):
"""Get instrument metadata
- Args:
- observatory: observatory code
- start_time: start time to match, or None to match any.
- end_time: end time to match, or None to match any.
- calibrations: use custom list, defaults to pulling and converting instrument metadata
- metadata_url: metadata database url
+ Parameters:
+ -----------
+ observatory: observatory code
+ start_time: start time to match, or None to match any.
+ end_time: end time to match, or None to match any.
+ calibrations: use custom list, defaults to pulling and converting instrument metadata
+ metadata_url: metadata database url
+
Returns:
- list of applicable instrument calibrations
+ --------
+ list of applicable instrument calibrations
"""
if not calibrations:
@@ -329,7 +332,7 @@ def get_instrument_calibrations(
data_valid=True,
)
try:
- metadata = factory.get_metadata(query=query)
+ metadata = factory.get_metadata(query=query, timeout=28)
except:
print(
"Warning: An error occurred while trying to pull metadata from the metadata server!"
--
GitLab
From 6e331efba3ccaa86f01778fb52f3c7b4fbaf51b7 Mon Sep 17 00:00:00 2001
From: bgeels <bgeels@usgs.gov>
Date: Wed, 7 Aug 2024 10:49:45 -0600
Subject: [PATCH 4/5] It might be a good idea to actually use the timeout
params...
---
geomagio/metadata/MetadataFactory.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/geomagio/metadata/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
index 2f718fe3..3212f6e7 100644
--- a/geomagio/metadata/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -127,8 +127,7 @@ class MetadataFactory(object):
The Metadata object corresponding to the given ID.
"""
response = requests.get(
- url=f"{self.url}/{id}",
- headers=self._get_headers(),
+ url=f"{self.url}/{id}", headers=self._get_headers(), timeout=timeout
)
return Metadata(**response.json())
@@ -155,6 +154,7 @@ class MetadataFactory(object):
url=self.url,
data=metadata.json(),
headers=self._get_headers(),
+ timeout=timeout,
)
return Metadata(**response.json())
@@ -182,6 +182,7 @@ class MetadataFactory(object):
url=f"{self.url}/{metadata.id}",
data=metadata.json(),
headers=self._get_headers(),
+ timeout=timeout,
)
return Metadata(**response.json())
--
GitLab
From 801fb843fddd499b387a5351d60d88abf349307c Mon Sep 17 00:00:00 2001
From: bgeels <bgeels@usgs.gov>
Date: Wed, 7 Aug 2024 10:53:32 -0600
Subject: [PATCH 5/5] Undo overzealous 'replace-all'
---
geomagio/metadata/MetadataFactory.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/geomagio/metadata/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
index 3212f6e7..fdd134e6 100644
--- a/geomagio/metadata/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -202,7 +202,7 @@ def parse_params(query: MetadataQuery) -> str:
A dictionary of parameters formatted for the API request.
"""
query = query.dict(exclude_none=True)
- Parameters = {}
+ args = {}
for key in query.keys():
element = query[key]
# Convert times to ISO format strings
@@ -210,9 +210,9 @@ def parse_params(query: MetadataQuery) -> str:
element = element.isoformat()
# Serialize the metadata dictionary to a JSON string
elif key == "metadata" and isinstance(element, dict):
- Parameters[key] = json.dumps(element)
+ args[key] = json.dumps(element)
# Get string value of metadata category
if key == "category":
element = element.value
- Parameters[key] = element
- return Parameters
+ args[key] = element
+ return args
--
GitLab