Updated the filter endpoint to not allow for the output sampling period to be...

Updated the filter endpoint to not allow for the output sampling period to be none. If it is left empty an error code wil prompt you to provide an appropriate sampling period. This involved adding some logic to a few of the test and a couple logger warming that would skip trying to determine wehter the number of smaple reqeusted ignores the sample request limit.

geomagio/api/ws/DataApiQuery.py

@@ -10,8 +10,12 @@ from pydantic import ConfigDict, field_validator, model_validator, Field, BaseMo
 from .Element import ELEMENTS
 from .Observatory import OBSERVATORY_INDEX, ASL_OBSERVATORY_INDEX
 from ...pydantic_utcdatetime import CustomUTCDateTimeType
+import logging
 
-
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
 DEFAULT_ELEMENTS = ["X", "Y", "Z", "F"]
 REQUEST_LIMIT = 3456000  # Increased the request limit by 10x what was decided by Jeremy
 VALID_ELEMENTS = [e.id for e in ELEMENTS]
@@ -69,7 +73,7 @@ class DataApiQuery(BaseModel):
     # endtime default is dependent on start time, so it's handled after validation in the model_validator
     endtime: Optional[CustomUTCDateTimeType] = None
     elements: List[str] = DEFAULT_ELEMENTS
-    sampling_period: SamplingPeriod = SamplingPeriod.MINUTE
+    sampling_period: Optional[SamplingPeriod] = None
     data_type: Union[DataType, str] = DataType.VARIATION
     format: Union[OutputFormat, str] = OutputFormat.IAGA2002
     data_host: Union[DataHost, str] = DataHost.DEFAULT
@@ -124,11 +128,21 @@ class DataApiQuery(BaseModel):
             self.endtime = self.starttime + (86400 - 0.001)
         if self.starttime > self.endtime:
             raise ValueError("Starttime must be before endtime.")
-        # check data volume
-        samples = int(
-            len(self.elements) * (self.endtime - self.starttime) / self.sampling_period
-        )
-        if samples > REQUEST_LIMIT:
-            raise ValueError(f"Request exceeds limit ({samples} > {REQUEST_LIMIT})")
-        # otherwise okay
+
+        # check data volume and if SamplingPeriod is assigned as None
+        if self.sampling_period is None:
+            logging.warning(
+                "Sampling period is None. Default value or further processing needed."
+            )
+            # self.sampling_period = 0.0
+
+        else:
+            samples = int(
+                len(self.elements)
+                * (self.endtime - self.starttime)
+                / self.sampling_period
+            )
+            if samples > REQUEST_LIMIT:
+                raise ValueError(f"Request exceeds limit ({samples} > {REQUEST_LIMIT})")
+            # otherwise okay
         return self

geomagio/api/ws/FilterApiQuery.py

@@ -5,6 +5,13 @@ from .DataApiQuery import (
 )
 from pydantic import ConfigDict, model_validator, field_validator, ValidationError
 from typing import Optional
+import logging
+from fastapi import HTTPException
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
 
 """This class inherits all the fields and validation on DataApiQuery and adds
 the fields input_sampling_period and output_sampling_period."""
@@ -17,13 +24,22 @@ class FilterApiQuery(DataApiQuery):
 
     @model_validator(mode="after")
     def validate_sample_size(self):
-        # Calculate the number of samples based on the input sampling period
-        samples = int(
-            len(self.elements) * (self.endtime - self.starttime) / self.sampling_period
-        )
-
-        # Validate the request size
-        if samples > REQUEST_LIMIT:
-            raise ValueError(f"Request exceeds limit ({samples} > {REQUEST_LIMIT})")
+        if self.sampling_period is None:
+            # Log a warning indicating that the sampling period is missing
+            logging.warning(
+                "Sampling period is None. Please provide a valid Sampling Period."
+            )
+
+        else:
+            # Calculate the number of samples based on the input sampling period
+            samples = int(
+                len(self.elements)
+                * (self.endtime - self.starttime)
+                / self.sampling_period
+            )
+
+            # Validate the request size
+            if samples > REQUEST_LIMIT:
+                raise ValueError(f"Request exceeds limit ({samples} > {REQUEST_LIMIT})")
 
         return self

geomagio/api/ws/algorithms.py

@@ -14,7 +14,6 @@ from .FilterApiQuery import FilterApiQuery
 from .data import format_timeseries, get_data_factory, get_data_query, get_timeseries
 from .filter import get_filter_data_query
 from . import filter
-from ...algorithm.FilterAlgorithm import STEPS
 import logging
 
 logger = logging.getLogger(__name__)

geomagio/api/ws/data.py

@@ -74,7 +74,7 @@ def get_data_query(
         " NOTE: when using 'iaga2002' output format, a maximum of 4 elements is allowed",
     ),
     sampling_period: Union[SamplingPeriod, float] = Query(
-        SamplingPeriod.MINUTE,
+        None,
         title="data rate",
         description="Interval in seconds between values.",
     ),

geomagio/api/ws/filter.py

@@ -35,11 +35,11 @@ def get_filter_data_query(
     ),
     input_sampling_period: Optional[SamplingPeriod] = Query(
         None,
-        title="Initial Sampling Period",
+        title="Input Sampling Period",
         description="`--` dynamically determines a necessary sampling period.",
     ),
-    sampling_period: SamplingPeriod = Query(
-        SamplingPeriod.SECOND,
+    sampling_period: Optional[SamplingPeriod] = Query(
+        None,
         alias="output_sampling_period",
         title="Output sampling period",
     ),
@@ -66,7 +66,6 @@ def get_timeseries(query: FilterApiQuery) -> Stream:
     data_factory = get_data_factory(query=query)
 
     # Determine input sampling period if not provided
-    # if query.input_sampling_period == SamplingPeriodWithAuto.AUTO or query.input_sampling_period is None:
     if query.input_sampling_period is None:
         # Dynamically determine the input sampling period
         input_sampling_period, data = determine_available_period(
@@ -79,7 +78,7 @@ def get_timeseries(query: FilterApiQuery) -> Stream:
         input_sample_period=input_sampling_period,
         output_sample_period=query.sampling_period,
     )
-    # Fetch data
+    # Fetch filtered data
     starttime, endtime = filt.get_input_interval(query.starttime, query.endtime)
 
     data = data_factory.get_timeseries(
@@ -91,7 +90,8 @@ def get_timeseries(query: FilterApiQuery) -> Stream:
         interval=TimeseriesUtility.get_interval_from_delta(filt.input_sample_period),
     )
 
-    # Apply filtering
+    # Apply filtering if needed
+
     filtered_timeseries = filt.process(data)
     return filtered_timeseries
 
@@ -106,9 +106,13 @@ def determine_available_period(output_sampling_period: float, query, data_factor
     sorted_periods: List[SamplingPeriod] = sorted(
         SamplingPeriod, key=lambda p: p.value, reverse=True
     )
-    valid_sampling_periods = [
-        p for p in sorted_periods if p.value <= output_sampling_period
-    ]
+    if output_sampling_period is None:
+        raise ValueError("Output sampling period cannot be None.")
+    else:
+
+        valid_sampling_periods = [
+            p for p in sorted_periods if p.value <= output_sampling_period
+        ]
 
     for period in valid_sampling_periods:
         if period <= output_sampling_period:

pytest.ini

@@ -3,3 +3,9 @@ norecursedirs = */site-packages
 testpaths = test
 asyncio_mode=auto
 asyncio_default_fixture_loop_scope="function"
+# Suppress warnings of level WARNING and below
+log_level = WARNING
+
+# Optionally, you can filter out UserWarnings generated by logging
+filterwarnings =
+    ignore::UserWarning
\ No newline at end of file

test/DataApiQuery_test.py

@@ -23,7 +23,7 @@ def test_DataApiQuery_defaults():
     assert_equal(query.starttime, expected_start_time)
     assert_equal(query.endtime, expected_endtime)
     assert_equal(query.elements, ["X", "Y", "Z", "F"])
-    assert_equal(query.sampling_period, SamplingPeriod.MINUTE)
+    assert_equal(query.sampling_period, None)
     assert_equal(query.data_type, DataType.VARIATION)
     assert_equal(query.format, OutputFormat.IAGA2002)
     # assumes the env var DATA_HOST is not set
@@ -41,7 +41,7 @@ def test_DataApiQuery_starttime_is_none():
     assert_equal(query.starttime, expected_start_time)
     assert_equal(query.endtime, expected_endtime)
     assert_equal(query.elements, ["X", "Y", "Z", "F"])
-    assert_equal(query.sampling_period, SamplingPeriod.MINUTE)
+    assert_equal(query.sampling_period, None)
     assert_equal(query.data_type, DataType.VARIATION)
     assert_equal(query.format, OutputFormat.IAGA2002)
     # assumes the env var DATA_HOST is not set
@@ -100,7 +100,7 @@ def test_DataApiQuery_default_endtime():
     # endtime is 1 day after start time
     assert_equal(query.endtime, UTCDateTime("2024-11-02T00:00:00.999"))
     assert_equal(query.elements, ["X", "Y", "Z", "F"])
-    assert_equal(query.sampling_period, SamplingPeriod.MINUTE)
+    assert_equal(query.sampling_period, None)
     assert_equal(query.data_type, DataType.VARIATION)
     assert_equal(query.format, OutputFormat.IAGA2002)
     assert_equal(query.data_host, DataHost.DEFAULT)
@@ -122,7 +122,7 @@ def test_DataApiQuery_default_only_endtime():
     assert_equal(query.endtime, hour_later)
 
     assert_equal(query.elements, ["X", "Y", "Z", "F"])
-    assert_equal(query.sampling_period, SamplingPeriod.MINUTE)
+    assert_equal(query.sampling_period, None)
     assert_equal(query.data_type, DataType.VARIATION)
     assert_equal(query.format, OutputFormat.IAGA2002)
     assert_equal(query.data_host, DataHost.DEFAULT)

test/FilterApiQuery_test.py

@@ -23,7 +23,7 @@ def test_FilterApiQuery_defaults():
     assert_equal(query.endtime, expected_endtime)
     assert_equal(query.elements, ["X", "Y", "Z", "F"])
     assert_equal(query.input_sampling_period, None)
-    assert_equal(query.sampling_period, SamplingPeriod.MINUTE)
+    assert_equal(query.sampling_period, None)
     assert_equal(query.data_type, DataType.VARIATION)
     assert_equal(query.format, OutputFormat.IAGA2002)
     assert_equal(query.data_host, DataHost.DEFAULT)
@@ -83,7 +83,7 @@ def test_FilterApiQuery_default_endtime():
     # endtime is 1 day after start time
     assert_equal(query.endtime, UTCDateTime("2024-11-02T00:00:00.999"))
     assert_equal(query.elements, ["X", "Y", "Z", "F"])
-    assert_equal(query.sampling_period, SamplingPeriod.MINUTE)
+    assert_equal(query.sampling_period, None)
     assert_equal(query.data_type, DataType.VARIATION)
     assert_equal(query.format, OutputFormat.IAGA2002)
     assert_equal(query.data_host, DataHost.DEFAULT)
@@ -190,3 +190,13 @@ def test_FilterApiQuery_extra_fields():
         assert "Extra inputs are not permitted" == err[0]["msg"]
 
     assert_equal(query, None)
+
+
+def test_FilterApiQuery_no_output_sampling_period():
+    query = None
+    try:
+        query = FilterApiQuery(id="ANMO", sampling_period=None)
+    except Exception as e:
+        err = e.errors()
+        assert "Output sampling period cannot be None." == err[0]["msg"]
+        assert_equal(query, None)

test/api_test/ws_test/filter_test.py

@@ -34,7 +34,7 @@ def test_get_filter_data_query(test_client):
     assert_equal(query.format, "iaga2002")
     assert_equal(query.data_type, "variation")
     assert_equal(query.input_sampling_period, SamplingPeriod.MINUTE)
-    assert_equal(query.output_sampling_period, SamplingPeriod.HOUR)
+    assert_equal(query.sampling_period, SamplingPeriod.HOUR)
 
 
 def test_get_filter_data_query_no_starttime(test_client):