Add cache file utils & err handling

geomagio/Util.py

@@ -2,6 +2,8 @@ import numpy
 import os
 from obspy.core import Stats, Trace
 from io import BytesIO
+import json
+import fcntl
 
 
 class ObjectView(object):
@@ -199,3 +201,97 @@ def create_empty_trace(trace, channel):
     count = len(trace.data)
     numpy_data = numpy.full((count), numpy.nan)
     return Trace(numpy_data, stats)
+
+
+def write_cache_file(filename, data, directory=None):
+    """
+    Writes data to a cache file in a thread-safe manner.
+
+    Parameters:
+    -----------
+    filename: String
+        The name of the file to write to.
+    data:
+        The data to write to the file. This should be a Python object that can be serialized with json.
+    directory: String
+        The directory to write the file to. If not provided, the file will be written to the .cache directory in the current user's home directory.
+
+    Returns:
+    --------
+    None
+
+    Raises:
+    -------
+    IOError: If an I/O error occurs.
+    TypeError: If the data cannot be serialized to JSON.
+    """
+    if directory is None:
+        directory = os.path.join(os.path.expanduser("~"), ".cache", "geomag-algorithms")
+
+    # Create the directory if it doesn't exist
+    try:
+        os.makedirs(directory, exist_ok=True)
+    except OSError as e:
+        print(f"Error creating directory: {e}")
+        raise
+
+    filepath = os.path.join(directory, filename)
+
+    try:
+        with open(filepath, "w") as f:
+            try:
+                fcntl.flock(f, fcntl.LOCK_EX)
+                json.dump(data, f)
+                fcntl.flock(f, fcntl.LOCK_UN)
+            except IOError as e:
+                print(f"Error locking or writing to file: {e}")
+                raise
+            except TypeError as e:
+                print(f"Error serializing data to JSON: {e}")
+                raise
+    except IOError as e:
+        print(f"Error opening file: {e}")
+        raise
+
+
+def read_cache_file(filename, directory=None):
+    """
+    Reads data from a cache file in a thread-safe manner.
+
+    Parameters:
+    filename: String
+        The name of the file to read from.
+    directory: String
+        The directory to read the file from. If not provided, the file will be read from the .cache directory in the current user's home directory.
+
+    Returns:
+    --------
+    data: Object
+        Python object that was deserialized from the json cache file.
+
+    Raises:
+    -------
+    IOError: If an I/O error occurs.
+    json.JSONDecodeError: If the data cannot be deserialized from JSON.
+    """
+    if directory is None:
+        directory = os.path.join(os.path.expanduser("~"), ".cache", "geomag-algorithms")
+
+    filepath = os.path.join(directory, filename)
+
+    try:
+        with open(filepath, "r") as f:
+            try:
+                fcntl.flock(f, fcntl.LOCK_SH)
+                data = json.load(f)
+                fcntl.flock(f, fcntl.LOCK_UN)
+                return data
+            except IOError as e:
+                print(f"Error locking or reading from file: {e}")
+                raise
+            except json.JSONDecodeError as e:
+                print(f"Error deserializing data from JSON: {e}")
+                raise
+    except IOError as e:
+        print(f"Error opening file: {e}")
+        raise

geomagio/metadata/instrument/InstrumentCalibrations.py

 from obspy import UTCDateTime
 
-from geomagio.metadata import Metadata, MetadataFactory, MetadataCategory
+from requests.exceptions import JSONDecodeError, ConnectTimeout
 
-# REMOVE ANY INSTRUMENT METADATA ELEMENTS W/ "DATA_VALID=FALSE"
+from geomagio.metadata import Metadata, MetadataFactory, MetadataCategory
+from geomagio.Util import write_cache_file, read_cache_file
 
 
 class InstrumentCalibrations:
@@ -314,6 +315,9 @@ def get_instrument_calibrations(
     """
 
     if not calibrations:
+        cache_filename = f"{observatory}_instrument_cals.json"
+        metadata = []
+
         factory = MetadataFactory(
             url=metadata_url or "https://staging-geomag.cr.usgs.gov/ws/secure/metadata",
         )
@@ -323,38 +327,43 @@ def get_instrument_calibrations(
             endtime=end_time,
             station=observatory,
         )
-        metadata = factory.get_metadata(query=query)
-
-        # remove invalid metadata entries
-        for i, m in enumerate(metadata):
-            if not m.data_valid or "instrument_category" not in m.metadata:
-                metadata.remove(m)
+        try:
+            metadata = factory.get_metadata(query=query)
+
+            # remove invalid metadata entries
+            for i, m in enumerate(metadata):
+                if not m.data_valid or "instrument_category" not in m.metadata:
+                    metadata.remove(m)
+        except:
+            print(
+                "Warning: An error occurred while trying to pull metadata from the metadata server!"
+            )
 
         if not metadata or metadata is None:
-            raise ValueError(
-                f"No valid metadata returned for {observatory} for time interval: {start_time} - {end_time}"
+            print(
+                f"Warning: No valid metadata returned for {observatory} for time interval: {start_time} - {end_time}"
             )
         else:
-            instrumentCalibrations = InstrumentCalibrations(metadata)
-            calibrations = instrumentCalibrations.get_calibrations()
-
-    if not calibrations or calibrations is None:
-        raise ValueError(
-            f"No valid calibrations returned for {observatory} for time interval: {start_time} - {end_time}"
-        )
-    else:
-        return [
-            c
-            for c in calibrations
-            if c["station"] == observatory
-            and (
-                end_time is None
-                or c["start_time"] is None
-                or c["start_time"] < end_time
-            )
-            and (
-                start_time is None
-                or c["end_time"] is None
-                or c["end_time"] > start_time
+            try:
+                instrumentCalibrations = InstrumentCalibrations(metadata)
+                calibrations = instrumentCalibrations.get_calibrations()
+            except ValueError as e:
+                print(e)
+
+        if not calibrations or calibrations is None:
+            print(
+                f"Warning: No valid calibrations generated for {observatory} for time interval: {start_time} - {end_time}"
             )
-        ]
+            print(f"Pulling calibrations for {observatory} from cache file.")
+
+            calibrations = read_cache_file(filename=cache_filename)
+        else:
+            write_cache_file(cache_filename, calibrations)
+
+    return [
+        c
+        for c in calibrations
+        if c["station"] == observatory
+        and (end_time is None or c["start_time"] is None or c["start_time"] < end_time)
+        and (start_time is None or c["end_time"] is None or c["end_time"] > start_time)
+    ]