From fd912623d11807c6a739cd3fda6048471c7005d9 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 10:16:46 -0600
Subject: [PATCH 01/15] Test for location code as data type in SNCL tests
---
test/edge_test/LegacySNCL_test.py | 10 +++++++++
test/edge_test/SNCL_test.py | 34 +++++++++++++++++++++++++++++++
2 files changed, 44 insertions(+)
diff --git a/test/edge_test/LegacySNCL_test.py b/test/edge_test/LegacySNCL_test.py
index adbcc8a83..190dc09a9 100644
--- a/test/edge_test/LegacySNCL_test.py
+++ b/test/edge_test/LegacySNCL_test.py
@@ -133,6 +133,13 @@ def test_get_location():
assert get_location(element="D", data_type="quasi-definitive") == "Q0"
assert get_location(element="D", data_type="definitive") == "D0"
assert get_location(element="D_Sat", data_type="variation") == "R1"
+ assert get_location(element="D_Sat", data_type="adjusted") == "A1"
+ assert get_location(element="D", data_type="R0") == "R0"
+ assert get_location(element="D", data_type="A0") == "A0"
+ assert get_location(element="D", data_type="Q0") == "Q0"
+ assert get_location(element="D", data_type="D0") == "D0"
+ assert get_location(element="D", data_type="R1") == "R1"
+ assert get_location(element="D", data_type="A1") == "A1"
def test_get_sncl():
@@ -140,6 +147,9 @@ def test_get_sncl():
assert LegacySNCL.get_sncl(
station="BOU", data_type="variation", interval="second", element="H"
) == LegacySNCL(station="BOU", network="NT", channel="SVH", location="R0")
+ assert LegacySNCL.get_sncl(
+ station="BOU", data_type="R0", interval="second", element="H"
+ ) == LegacySNCL(station="BOU", network="NT", channel="SVH", location="R0")
def test_interval():
diff --git a/test/edge_test/SNCL_test.py b/test/edge_test/SNCL_test.py
index 51d5b73c3..1f963526b 100644
--- a/test/edge_test/SNCL_test.py
+++ b/test/edge_test/SNCL_test.py
@@ -115,16 +115,22 @@ def test_get_channel():
== "BYU"
)
assert get_channel(element="D", interval="second", data_type="variation") == "LFD"
+ assert get_channel(element="D", interval="second", data_type="R0") == "LFD"
assert get_channel(element="F", interval="minute", data_type="variation") == "UFF"
assert get_channel(element="U", interval="hour", data_type="variation") == "RFU"
assert get_channel(element="V", interval="hour", data_type="variation") == "RFV"
assert get_channel(element="W", interval="hour", data_type="variation") == "RFW"
assert get_channel(element="H", interval="hour", data_type="variation") == "RFU"
+ assert get_channel(element="H", interval="hour", data_type="R0") == "RFU"
assert get_channel(element="E", interval="hour", data_type="variation") == "RFV"
+ assert get_channel(element="E", interval="hour", data_type="R0") == "RFV"
assert get_channel(element="Z", interval="hour", data_type="variation") == "RFW"
+ assert get_channel(element="Z", interval="hour", data_type="R0") == "RFW"
# not variation data_type, test that H,Z is not converted to U,V
assert get_channel(element="H", interval="hour", data_type="adjusted") == "RFH"
+ assert get_channel(element="H", interval="hour", data_type="A0") == "RFH"
assert get_channel(element="Z", interval="hour", data_type="adjusted") == "RFZ"
+ assert get_channel(element="Z", interval="hour", data_type="A0") == "RFZ"
assert get_channel(element="Dst4", interval="day", data_type="variation") == "PX4"
assert (
get_channel(element="Dst3", interval="minute", data_type="variation") == "UX3"
@@ -135,12 +141,23 @@ def test_get_channel():
assert (
get_channel(element="U_Dist", interval="minute", data_type="variation") == "UFU"
)
+ assert get_channel(element="U", interval="minute", data_type="RD") == "UFU"
assert (
get_channel(element="U_SQ", interval="minute", data_type="variation") == "UFU"
)
+ assert get_channel(element="U", interval="minute", data_type="RQ") == "UFU"
assert (
get_channel(element="U_SV", interval="minute", data_type="variation") == "UFU"
)
+ assert get_channel(element="U", interval="minute", data_type="RV") == "UFU"
+ assert (
+ get_channel(element="U_Dist", interval="minute", data_type="adjusted") == "UFU"
+ )
+ assert get_channel(element="U", interval="minute", data_type="AD") == "UFU"
+ assert get_channel(element="U_SQ", interval="minute", data_type="adjusted") == "UFU"
+ assert get_channel(element="U", interval="minute", data_type="AQ") == "UFU"
+ assert get_channel(element="U_SV", interval="minute", data_type="adjusted") == "UFU"
+ assert get_channel(element="U", interval="minute", data_type="AV") == "UFU"
assert (
get_channel(element="UK1.R0", interval="minute", data_type="variation") == "UK1"
)
@@ -149,13 +166,21 @@ def test_get_channel():
def test_get_location():
"""edge_test.SNCL_test.test_get_location()"""
assert get_location(element="D", data_type="variation") == "R0"
+ assert get_location(element="D", data_type="R0") == "R0"
assert get_location(element="D", data_type="adjusted") == "A0"
+ assert get_location(element="D", data_type="A0") == "A0"
assert get_location(element="D", data_type="quasi-definitive") == "Q0"
+ assert get_location(element="D", data_type="Q0") == "Q0"
assert get_location(element="D", data_type="definitive") == "D0"
+ assert get_location(element="D", data_type="D0") == "D0"
assert get_location(element="D_Sat", data_type="variation") == "R1"
+ assert get_location(element="D", data_type="R1") == "R1"
assert get_location(element="D_Dist", data_type="variation") == "RD"
+ assert get_location(element="D", data_type="RD") == "RD"
assert get_location(element="D_SQ", data_type="variation") == "RQ"
+ assert get_location(element="D", data_type="RQ") == "RQ"
assert get_location(element="D_SV", data_type="variation") == "RV"
+ assert get_location(element="D", data_type="RV") == "RV"
def test_get_sncl():
@@ -163,6 +188,15 @@ def test_get_sncl():
assert SNCL.get_sncl(
station="BOU", data_type="variation", interval="second", element="U"
) == SNCL(station="BOU", network="NT", channel="LFU", location="R0")
+ assert SNCL.get_sncl(
+ station="BOU", data_type="variation", interval="second", element="H"
+ ) == SNCL(station="BOU", network="NT", channel="LFU", location="R0")
+ assert SNCL.get_sncl(
+ station="BOU", data_type="R0", interval="second", element="H"
+ ) == SNCL(station="BOU", network="NT", channel="LFU", location="R0")
+ assert SNCL.get_sncl(
+ station="BOU", data_type="A0", interval="second", element="H"
+ ) == SNCL(station="BOU", network="NT", channel="LFH", location="A0")
def test_interval():
--
GitLab
From 5bf9572025b3c64c9e32abb9595904f6446830da Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 10:18:47 -0600
Subject: [PATCH 02/15] mseed/edge factories should set type to SNCL type
---
test/edge_test/EdgeFactory_test.py | 31 +++++++++++++++++++-
test/edge_test/MiniSeedFactory_test.py | 39 +++++++++++++++++++++++---
2 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/test/edge_test/EdgeFactory_test.py b/test/edge_test/EdgeFactory_test.py
index 55e94f126..7f92b7ecc 100644
--- a/test/edge_test/EdgeFactory_test.py
+++ b/test/edge_test/EdgeFactory_test.py
@@ -5,7 +5,7 @@ from geomagio.edge import EdgeFactory
from numpy.testing import assert_equal
-def dont_get_timeseries():
+def test_get_timeseries():
"""edge_test.EdgeFactory_test.test_get_timeseries()"""
# Call get_timeseries, and test stats for comfirmation that it came back.
# TODO, need to pass in host and port from a config file, or manually
@@ -29,6 +29,35 @@ def dont_get_timeseries():
"H",
"Expect timeseries stats channel to be equal to H",
)
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.data_type,
+ "variation",
+ "Expect timeseries stats data_type to be equal to variation",
+ )
+ # timeseries requested with location code as type
+ timeseries = edge_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
+ "BOU",
+ ("H"),
+ "R0",
+ "minute",
+ )
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.station,
+ "BOU",
+ "Expect timeseries to have stats",
+ )
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.channel,
+ "H",
+ "Expect timeseries stats channel to be equal to H",
+ )
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.data_type,
+ "variation",
+ "Expect timeseries stats data_type to be equal to variation",
+ )
def test_add_empty_channels():
diff --git a/test/edge_test/MiniSeedFactory_test.py b/test/edge_test/MiniSeedFactory_test.py
index 5689d69c5..22475d60b 100644
--- a/test/edge_test/MiniSeedFactory_test.py
+++ b/test/edge_test/MiniSeedFactory_test.py
@@ -1,9 +1,11 @@
"""Tests for MiniSeedFactory.py"""
import io
+from test.edge_test.conftest import MockMiniSeedClient
import numpy
from numpy.testing import assert_equal
from obspy.core import read, Stats, Stream, Trace, UTCDateTime
+import pytest
from geomagio import TimeseriesUtility
from geomagio.edge import MiniSeedFactory, MiniSeedInputClient
@@ -85,14 +87,14 @@ def test__set_metadata():
assert_equal(stream[1].stats["channel"], "H")
-# def test_get_timeseries():
-def dont_get_timeseries():
+def test_get_timeseries(MockMiniSeedClient):
"""edge_test.MiniSeedFactory_test.test_get_timeseries()"""
# Call get_timeseries, and test stats for comfirmation that it came back.
# TODO, need to pass in host and port from a config file, or manually
# change for a single test.
- edge_factory = MiniSeedFactory(host="TODO", port="TODO")
- timeseries = edge_factory.get_timeseries(
+ miniseed_factory = MiniSeedFactory()
+ miniseed_factory.client = MockMiniSeedClient()
+ timeseries = miniseed_factory.get_timeseries(
UTCDateTime(2015, 3, 1, 0, 0, 0),
UTCDateTime(2015, 3, 1, 1, 0, 0),
"BOU",
@@ -110,6 +112,35 @@ def dont_get_timeseries():
"H",
"Expect timeseries stats channel to be equal to H",
)
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.data_type,
+ "variation",
+ "Expect timeseries stats data_type to be equal to variation",
+ )
+ # timeseries requested with location code as type
+ timeseries = miniseed_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
+ "BOU",
+ ("H"),
+ "R0",
+ "minute",
+ )
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.station,
+ "BOU",
+ "Expect timeseries to have stats",
+ )
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.channel,
+ "H",
+ "Expect timeseries stats channel to be equal to H",
+ )
+ assert_equal(
+ timeseries.select(channel="H")[0].stats.data_type,
+ "variation",
+ "Expect timeseries stats data_type to be equal to variation",
+ )
def __create_trace(
--
GitLab
From f015314f3637258a3e9323533e1d12ee38d3e92d Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 10:19:17 -0600
Subject: [PATCH 03/15] Add mock client fixture for mseed test
---
test/edge_test/conftest.py | 65 ++++++++++++++++++++++++++++++++++++++
1 file changed, 65 insertions(+)
create mode 100644 test/edge_test/conftest.py
diff --git a/test/edge_test/conftest.py b/test/edge_test/conftest.py
new file mode 100644
index 000000000..4801e6ab8
--- /dev/null
+++ b/test/edge_test/conftest.py
@@ -0,0 +1,65 @@
+import numpy
+from obspy import Stream, UTCDateTime
+from obspy.clients.neic.client import Client
+import pytest
+
+from geomagio import TimeseriesUtility
+
+
+@pytest.fixture(scope="class")
+def MockMiniSeedClient() -> Client:
+ """replaces default obspy miniseed client's get_waveforms method to return trace of ones"""
+
+ class MockMiniSeedClient(Client):
+ def get_waveforms(
+ self,
+ network: str,
+ station: str,
+ location: str,
+ channel: str,
+ starttime: UTCDateTime,
+ endtime: UTCDateTime,
+ ):
+ trace = TimeseriesUtility.create_empty_trace(
+ starttime=starttime,
+ endtime=endtime,
+ observatory=station,
+ channel=channel,
+ type=self._get_data_type(location=location),
+ interval=self._get_interval(channel=channel),
+ network=network,
+ station=station,
+ location=location,
+ )
+ trace.data = numpy.ones(trace.stats.npts)
+ return Stream([trace])
+
+ def _get_interval(self, channel: str) -> str:
+ channel_start = channel[0]
+ if channel_start == "B":
+ return "tenhertz"
+ elif channel_start == "L":
+ return "second"
+ elif channel_start == "U":
+ return "minute"
+ elif channel_start == "R":
+ return "hour"
+ elif channel_start == "P":
+ return "day"
+ else:
+ raise ValueError(f"Unexpected channel start: {channel_start}")
+
+ def _get_data_type(self, location: str) -> str:
+ location_start = location[0]
+ if location_start == "R":
+ return "variation"
+ elif location_start == "A":
+ return "adjusted"
+ elif location_start == "Q":
+ return "quasi-definitive"
+ elif location_start == "D":
+ return "definitive"
+ else:
+ raise ValueError(f"Unexpected location start: {location_start}")
+
+ yield MockMiniSeedClient
--
GitLab
From 3f53ab4aa78e3ceaa4c0407be415025825efa857 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 10:19:48 -0600
Subject: [PATCH 04/15] Support location as data type for SNCL's
---
geomagio/edge/LegacySNCL.py | 2 ++
geomagio/edge/SNCL.py | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/geomagio/edge/LegacySNCL.py b/geomagio/edge/LegacySNCL.py
index 1b8cd2169..69527bd26 100644
--- a/geomagio/edge/LegacySNCL.py
+++ b/geomagio/edge/LegacySNCL.py
@@ -62,6 +62,8 @@ def get_channel(element: str, interval: str) -> str:
def get_location(element: str, data_type: str) -> str:
+ if len(data_type) == 2 and data_type[0] in ["R", "A", "Q", "D"]:
+ return data_type
return _get_location_start(data_type=data_type) + _get_location_end(element=element)
diff --git a/geomagio/edge/SNCL.py b/geomagio/edge/SNCL.py
index 8dc8a8e25..e5aec17d5 100644
--- a/geomagio/edge/SNCL.py
+++ b/geomagio/edge/SNCL.py
@@ -95,6 +95,8 @@ def get_channel(element: str, interval: str, data_type: str) -> str:
def get_location(element: str, data_type: str) -> str:
+ if len(data_type) == 2 and data_type[0] in ["R", "A", "Q", "D"]:
+ return data_type
return _get_location_start(data_type=data_type) + _get_location_end(element=element)
@@ -166,7 +168,7 @@ def _get_channel_end(element: str, data_type: str) -> str:
elif "_Temp" in element:
channel_middle = "K"
channel_end = element.split("_")[0]
- if data_type == "variation":
+ if data_type in ["variation", "R0", "R1", "RD", "RQ", "RV"]:
if channel_end == "H":
channel_end = "U"
elif channel_end == "E":
--
GitLab
From e6a50456701d1e17d426c5c71c8d3ebc04ca8742 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 10:20:29 -0600
Subject: [PATCH 05/15] Set type to SNCL's type in mseed/edge factories
---
geomagio/edge/EdgeFactory.py | 1 +
geomagio/edge/MiniSeedFactory.py | 1 +
2 files changed, 2 insertions(+)
diff --git a/geomagio/edge/EdgeFactory.py b/geomagio/edge/EdgeFactory.py
index c25955fe1..189a40b11 100644
--- a/geomagio/edge/EdgeFactory.py
+++ b/geomagio/edge/EdgeFactory.py
@@ -346,6 +346,7 @@ class EdgeFactory(TimeseriesFactory):
for trace in data:
trace.data = trace.data.astype("i4")
data.merge()
+ type = sncl.data_type
if data.count() == 0 and add_empty_channels:
data += self._get_empty_trace(
starttime=starttime,
diff --git a/geomagio/edge/MiniSeedFactory.py b/geomagio/edge/MiniSeedFactory.py
index 94103caee..d8f18768e 100644
--- a/geomagio/edge/MiniSeedFactory.py
+++ b/geomagio/edge/MiniSeedFactory.py
@@ -352,6 +352,7 @@ class MiniSeedFactory(TimeseriesFactory):
sncl.network, sncl.station, sncl.location, sncl.channel, starttime, endtime
)
data.merge()
+ type = sncl.data_type
if data.count() == 0 and add_empty_channels:
data += self._get_empty_trace(
starttime=starttime,
--
GitLab
From 7a81f47de6a1cad1858b3193ff8c50042ad15501 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 10:58:41 -0600
Subject: [PATCH 06/15] Remove unused import
---
test/edge_test/MiniSeedFactory_test.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/test/edge_test/MiniSeedFactory_test.py b/test/edge_test/MiniSeedFactory_test.py
index 22475d60b..ed3901904 100644
--- a/test/edge_test/MiniSeedFactory_test.py
+++ b/test/edge_test/MiniSeedFactory_test.py
@@ -1,6 +1,5 @@
"""Tests for MiniSeedFactory.py"""
import io
-from test.edge_test.conftest import MockMiniSeedClient
import numpy
from numpy.testing import assert_equal
--
GitLab
From f2604fb481dc1acd2b179f83b203190798026eef Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 12:00:52 -0600
Subject: [PATCH 07/15] Separate location code tests
---
test/edge_test/EdgeFactory_test.py | 51 ++++++++++++++++++++-----
test/edge_test/MiniSeedFactory_test.py | 53 +++++++++++++++++++++-----
2 files changed, 86 insertions(+), 18 deletions(-)
diff --git a/test/edge_test/EdgeFactory_test.py b/test/edge_test/EdgeFactory_test.py
index 7f92b7ecc..52a4f662f 100644
--- a/test/edge_test/EdgeFactory_test.py
+++ b/test/edge_test/EdgeFactory_test.py
@@ -34,7 +34,11 @@ def test_get_timeseries():
"variation",
"Expect timeseries stats data_type to be equal to variation",
)
- # timeseries requested with location code as type
+
+
+def test_get_timeseries_by_location():
+ """test.edge_test.EdgeFactory_test.test_get_timeseries_by_location()"""
+ edge_factory = EdgeFactory(host="TODO", port="TODO")
timeseries = edge_factory.get_timeseries(
UTCDateTime(2015, 3, 1, 0, 0, 0),
UTCDateTime(2015, 3, 1, 1, 0, 0),
@@ -44,19 +48,48 @@ def test_get_timeseries():
"minute",
)
assert_equal(
- timeseries.select(channel="H")[0].stats.station,
+ timeseries.select(channel="H")[0].stats.data_type,
+ "R0",
+ "Expect timeseries stats data_type to be equal to R0",
+ )
+ timeseries = edge_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
"BOU",
- "Expect timeseries to have stats",
+ ("H"),
+ "A0",
+ "minute",
)
assert_equal(
- timeseries.select(channel="H")[0].stats.channel,
- "H",
- "Expect timeseries stats channel to be equal to H",
+ timeseries.select(channel="H")[0].stats.data_type,
+ "A0",
+ "Expect timeseries stats data_type to be equal to A0",
+ )
+ timeseries = edge_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
+ "BOU",
+ ("X"),
+ "Q0",
+ "minute",
)
assert_equal(
- timeseries.select(channel="H")[0].stats.data_type,
- "variation",
- "Expect timeseries stats data_type to be equal to variation",
+ timeseries.select(channel="X")[0].stats.data_type,
+ "Q0",
+ "Expect timeseries stats data_type to be equal to Q0",
+ )
+ timeseries = edge_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
+ "BOU",
+ ("X"),
+ "D0",
+ "minute",
+ )
+ assert_equal(
+ timeseries.select(channel="X")[0].stats.data_type,
+ "D0",
+ "Expect timeseries stats data_type to be equal to D0",
)
diff --git a/test/edge_test/MiniSeedFactory_test.py b/test/edge_test/MiniSeedFactory_test.py
index ed3901904..00a96a509 100644
--- a/test/edge_test/MiniSeedFactory_test.py
+++ b/test/edge_test/MiniSeedFactory_test.py
@@ -116,7 +116,13 @@ def test_get_timeseries(MockMiniSeedClient):
"variation",
"Expect timeseries stats data_type to be equal to variation",
)
- # timeseries requested with location code as type
+
+
+def test_get_timeseries_by_location(MockMiniSeedClient):
+ """test.edge_test.MiniSeedFactory_test.test_get_timeseries_by_location()"""
+ miniseed_factory = MiniSeedFactory()
+ miniseed_factory.client = MockMiniSeedClient()
+
timeseries = miniseed_factory.get_timeseries(
UTCDateTime(2015, 3, 1, 0, 0, 0),
UTCDateTime(2015, 3, 1, 1, 0, 0),
@@ -126,19 +132,48 @@ def test_get_timeseries(MockMiniSeedClient):
"minute",
)
assert_equal(
- timeseries.select(channel="H")[0].stats.station,
+ timeseries.select(channel="H")[0].stats.data_type,
+ "R0",
+ "Expect timeseries stats data_type to be equal to R0",
+ )
+ timeseries = miniseed_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
"BOU",
- "Expect timeseries to have stats",
+ ("H"),
+ "A0",
+ "minute",
)
assert_equal(
- timeseries.select(channel="H")[0].stats.channel,
- "H",
- "Expect timeseries stats channel to be equal to H",
+ timeseries.select(channel="H")[0].stats.data_type,
+ "A0",
+ "Expect timeseries stats data_type to be equal to A0",
+ )
+ timeseries = miniseed_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
+ "BOU",
+ ("X"),
+ "Q0",
+ "minute",
)
assert_equal(
- timeseries.select(channel="H")[0].stats.data_type,
- "variation",
- "Expect timeseries stats data_type to be equal to variation",
+ timeseries.select(channel="X")[0].stats.data_type,
+ "Q0",
+ "Expect timeseries stats data_type to be equal to Q0",
+ )
+ timeseries = miniseed_factory.get_timeseries(
+ UTCDateTime(2015, 3, 1, 0, 0, 0),
+ UTCDateTime(2015, 3, 1, 1, 0, 0),
+ "BOU",
+ ("X"),
+ "D0",
+ "minute",
+ )
+ assert_equal(
+ timeseries.select(channel="X")[0].stats.data_type,
+ "D0",
+ "Expect timeseries stats data_type to be equal to D0",
)
--
GitLab
From 8da711d2130f2f05ad27455c456fd799eb697003 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 12:01:18 -0600
Subject: [PATCH 08/15] Maintain data type as location
---
geomagio/edge/EdgeFactory.py | 1 -
geomagio/edge/MiniSeedFactory.py | 1 -
2 files changed, 2 deletions(-)
diff --git a/geomagio/edge/EdgeFactory.py b/geomagio/edge/EdgeFactory.py
index 189a40b11..c25955fe1 100644
--- a/geomagio/edge/EdgeFactory.py
+++ b/geomagio/edge/EdgeFactory.py
@@ -346,7 +346,6 @@ class EdgeFactory(TimeseriesFactory):
for trace in data:
trace.data = trace.data.astype("i4")
data.merge()
- type = sncl.data_type
if data.count() == 0 and add_empty_channels:
data += self._get_empty_trace(
starttime=starttime,
diff --git a/geomagio/edge/MiniSeedFactory.py b/geomagio/edge/MiniSeedFactory.py
index d8f18768e..94103caee 100644
--- a/geomagio/edge/MiniSeedFactory.py
+++ b/geomagio/edge/MiniSeedFactory.py
@@ -352,7 +352,6 @@ class MiniSeedFactory(TimeseriesFactory):
sncl.network, sncl.station, sncl.location, sncl.channel, starttime, endtime
)
data.merge()
- type = sncl.data_type
if data.count() == 0 and add_empty_channels:
data += self._get_empty_trace(
starttime=starttime,
--
GitLab
From 4fab9678443eb4a2b1ec5bed988e982a5d10c125 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 12:02:13 -0600
Subject: [PATCH 09/15] custom locations, no mapping for custom locations
---
geomagio/edge/LegacySNCL.py | 2 +-
geomagio/edge/SNCL.py | 4 ++--
test/edge_test/SNCL_test.py | 8 ++++----
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/geomagio/edge/LegacySNCL.py b/geomagio/edge/LegacySNCL.py
index 69527bd26..f80b438a6 100644
--- a/geomagio/edge/LegacySNCL.py
+++ b/geomagio/edge/LegacySNCL.py
@@ -62,7 +62,7 @@ def get_channel(element: str, interval: str) -> str:
def get_location(element: str, data_type: str) -> str:
- if len(data_type) == 2 and data_type[0] in ["R", "A", "Q", "D"]:
+ if len(data_type) == 2:
return data_type
return _get_location_start(data_type=data_type) + _get_location_end(element=element)
diff --git a/geomagio/edge/SNCL.py b/geomagio/edge/SNCL.py
index e5aec17d5..003dcc7fc 100644
--- a/geomagio/edge/SNCL.py
+++ b/geomagio/edge/SNCL.py
@@ -95,7 +95,7 @@ def get_channel(element: str, interval: str, data_type: str) -> str:
def get_location(element: str, data_type: str) -> str:
- if len(data_type) == 2 and data_type[0] in ["R", "A", "Q", "D"]:
+ if len(data_type) == 2:
return data_type
return _get_location_start(data_type=data_type) + _get_location_end(element=element)
@@ -168,7 +168,7 @@ def _get_channel_end(element: str, data_type: str) -> str:
elif "_Temp" in element:
channel_middle = "K"
channel_end = element.split("_")[0]
- if data_type in ["variation", "R0", "R1", "RD", "RQ", "RV"]:
+ if data_type in "variation":
if channel_end == "H":
channel_end = "U"
elif channel_end == "E":
diff --git a/test/edge_test/SNCL_test.py b/test/edge_test/SNCL_test.py
index 1f963526b..0ce923683 100644
--- a/test/edge_test/SNCL_test.py
+++ b/test/edge_test/SNCL_test.py
@@ -121,11 +121,11 @@ def test_get_channel():
assert get_channel(element="V", interval="hour", data_type="variation") == "RFV"
assert get_channel(element="W", interval="hour", data_type="variation") == "RFW"
assert get_channel(element="H", interval="hour", data_type="variation") == "RFU"
- assert get_channel(element="H", interval="hour", data_type="R0") == "RFU"
+ assert get_channel(element="H", interval="hour", data_type="R0") == "RFH"
assert get_channel(element="E", interval="hour", data_type="variation") == "RFV"
- assert get_channel(element="E", interval="hour", data_type="R0") == "RFV"
+ assert get_channel(element="E", interval="hour", data_type="R0") == "RFE"
assert get_channel(element="Z", interval="hour", data_type="variation") == "RFW"
- assert get_channel(element="Z", interval="hour", data_type="R0") == "RFW"
+ assert get_channel(element="Z", interval="hour", data_type="R0") == "RFZ"
# not variation data_type, test that H,Z is not converted to U,V
assert get_channel(element="H", interval="hour", data_type="adjusted") == "RFH"
assert get_channel(element="H", interval="hour", data_type="A0") == "RFH"
@@ -193,7 +193,7 @@ def test_get_sncl():
) == SNCL(station="BOU", network="NT", channel="LFU", location="R0")
assert SNCL.get_sncl(
station="BOU", data_type="R0", interval="second", element="H"
- ) == SNCL(station="BOU", network="NT", channel="LFU", location="R0")
+ ) == SNCL(station="BOU", network="NT", channel="LFH", location="R0")
assert SNCL.get_sncl(
station="BOU", data_type="A0", interval="second", element="H"
) == SNCL(station="BOU", network="NT", channel="LFH", location="A0")
--
GitLab
From 71477b6351ae2fe49307afa0f15bbf144a623107 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 17:22:18 -0600
Subject: [PATCH 10/15] decbase for locations starting with R
---
geomagio/iaga2002/IAGA2002Writer.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/geomagio/iaga2002/IAGA2002Writer.py b/geomagio/iaga2002/IAGA2002Writer.py
index 2e9bbdfdd..bd215ef98 100644
--- a/geomagio/iaga2002/IAGA2002Writer.py
+++ b/geomagio/iaga2002/IAGA2002Writer.py
@@ -117,7 +117,11 @@ class IAGA2002Writer(object):
if (
"declination_base" in stats
and stats.declination_base is not None
- and (stats.data_type == "variation" or stats.data_type == "reported")
+ and (
+ stats.data_type == "variation"
+ or stats.data_type == "reported"
+ or stats.data_type[0] == "R"
+ )
):
comments.append(
"DECBAS {:<8d}"
--
GitLab
From c7031be3b599332ce4626f9a2c60dead618bca61 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 1 Sep 2021 17:37:23 -0600
Subject: [PATCH 11/15] fix if statement for SNCL channel mapping
---
geomagio/edge/SNCL.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/geomagio/edge/SNCL.py b/geomagio/edge/SNCL.py
index 003dcc7fc..e137b2b16 100644
--- a/geomagio/edge/SNCL.py
+++ b/geomagio/edge/SNCL.py
@@ -168,7 +168,7 @@ def _get_channel_end(element: str, data_type: str) -> str:
elif "_Temp" in element:
channel_middle = "K"
channel_end = element.split("_")[0]
- if data_type in "variation":
+ if data_type == "variation":
if channel_end == "H":
channel_end = "U"
elif channel_end == "E":
--
GitLab
From 640a8f2d7f1e1005c72204415e929cef0b3351bf Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Thu, 2 Sep 2021 08:57:46 -0600
Subject: [PATCH 12/15] add SNCL utilities to MockMiniSeedClient
---
test/edge_test/conftest.py | 39 +++++++++-----------------------------
1 file changed, 9 insertions(+), 30 deletions(-)
diff --git a/test/edge_test/conftest.py b/test/edge_test/conftest.py
index 4801e6ab8..b391a4a3a 100644
--- a/test/edge_test/conftest.py
+++ b/test/edge_test/conftest.py
@@ -4,6 +4,7 @@ from obspy.clients.neic.client import Client
import pytest
from geomagio import TimeseriesUtility
+from geomagio.edge import SNCL
@pytest.fixture(scope="class")
@@ -20,13 +21,19 @@ def MockMiniSeedClient() -> Client:
starttime: UTCDateTime,
endtime: UTCDateTime,
):
+ sncl = SNCL(
+ station=station,
+ network=network,
+ channel=channel,
+ location=location,
+ )
trace = TimeseriesUtility.create_empty_trace(
starttime=starttime,
endtime=endtime,
observatory=station,
channel=channel,
- type=self._get_data_type(location=location),
- interval=self._get_interval(channel=channel),
+ type=sncl.data_type,
+ interval=sncl.interval,
network=network,
station=station,
location=location,
@@ -34,32 +41,4 @@ def MockMiniSeedClient() -> Client:
trace.data = numpy.ones(trace.stats.npts)
return Stream([trace])
- def _get_interval(self, channel: str) -> str:
- channel_start = channel[0]
- if channel_start == "B":
- return "tenhertz"
- elif channel_start == "L":
- return "second"
- elif channel_start == "U":
- return "minute"
- elif channel_start == "R":
- return "hour"
- elif channel_start == "P":
- return "day"
- else:
- raise ValueError(f"Unexpected channel start: {channel_start}")
-
- def _get_data_type(self, location: str) -> str:
- location_start = location[0]
- if location_start == "R":
- return "variation"
- elif location_start == "A":
- return "adjusted"
- elif location_start == "Q":
- return "quasi-definitive"
- elif location_start == "D":
- return "definitive"
- else:
- raise ValueError(f"Unexpected location start: {location_start}")
-
yield MockMiniSeedClient
--
GitLab
From 5a24e2ba7eee80c0b82afd8af1fbf3774eb563bb Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 8 Sep 2021 16:27:19 +0000
Subject: [PATCH 13/15] remove conftest.py
---
test/edge_test/conftest.py | 44 --------------------------------------
1 file changed, 44 deletions(-)
delete mode 100644 test/edge_test/conftest.py
diff --git a/test/edge_test/conftest.py b/test/edge_test/conftest.py
deleted file mode 100644
index b391a4a3a..000000000
--- a/test/edge_test/conftest.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import numpy
-from obspy import Stream, UTCDateTime
-from obspy.clients.neic.client import Client
-import pytest
-
-from geomagio import TimeseriesUtility
-from geomagio.edge import SNCL
-
-
-@pytest.fixture(scope="class")
-def MockMiniSeedClient() -> Client:
- """replaces default obspy miniseed client's get_waveforms method to return trace of ones"""
-
- class MockMiniSeedClient(Client):
- def get_waveforms(
- self,
- network: str,
- station: str,
- location: str,
- channel: str,
- starttime: UTCDateTime,
- endtime: UTCDateTime,
- ):
- sncl = SNCL(
- station=station,
- network=network,
- channel=channel,
- location=location,
- )
- trace = TimeseriesUtility.create_empty_trace(
- starttime=starttime,
- endtime=endtime,
- observatory=station,
- channel=channel,
- type=sncl.data_type,
- interval=sncl.interval,
- network=network,
- station=station,
- location=location,
- )
- trace.data = numpy.ones(trace.stats.npts)
- return Stream([trace])
-
- yield MockMiniSeedClient
--
GitLab
From c7a942db45d05aab6833030dfe01d006defda911 Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 8 Sep 2021 16:27:37 +0000
Subject: [PATCH 14/15] Define mock client in separate file
---
test/edge_test/MockMiniSeedClient.py | 39 ++++++++++++++++++++++++++++
1 file changed, 39 insertions(+)
create mode 100644 test/edge_test/MockMiniSeedClient.py
diff --git a/test/edge_test/MockMiniSeedClient.py b/test/edge_test/MockMiniSeedClient.py
new file mode 100644
index 000000000..88c6a4b61
--- /dev/null
+++ b/test/edge_test/MockMiniSeedClient.py
@@ -0,0 +1,39 @@
+import numpy
+from obspy import Stream, UTCDateTime
+from obspy.clients.neic.client import Client
+
+from geomagio import TimeseriesUtility
+from geomagio.edge import SNCL
+
+
+class MockMiniSeedClient(Client):
+ """replaces default obspy miniseed client's get_waveforms method to return trace of ones"""
+
+ def get_waveforms(
+ self,
+ network: str,
+ station: str,
+ location: str,
+ channel: str,
+ starttime: UTCDateTime,
+ endtime: UTCDateTime,
+ ):
+ sncl = SNCL(
+ station=station,
+ network=network,
+ channel=channel,
+ location=location,
+ )
+ trace = TimeseriesUtility.create_empty_trace(
+ starttime=starttime,
+ endtime=endtime,
+ observatory=station,
+ channel=channel,
+ type=sncl.data_type,
+ interval=sncl.interval,
+ network=network,
+ station=station,
+ location=location,
+ )
+ trace.data = numpy.ones(trace.stats.npts)
+ return Stream([trace])
--
GitLab
From d063029db95641449816847bfb44feda1fcea0db Mon Sep 17 00:00:00 2001
From: pcain <pcain@usgs.gov>
Date: Wed, 8 Sep 2021 16:28:21 +0000
Subject: [PATCH 15/15] Create miniseed_factory fixture within mseed test
---
test/edge_test/MiniSeedFactory_test.py | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/test/edge_test/MiniSeedFactory_test.py b/test/edge_test/MiniSeedFactory_test.py
index 00a96a509..516850f79 100644
--- a/test/edge_test/MiniSeedFactory_test.py
+++ b/test/edge_test/MiniSeedFactory_test.py
@@ -8,6 +8,7 @@ import pytest
from geomagio import TimeseriesUtility
from geomagio.edge import MiniSeedFactory, MiniSeedInputClient
+from .MockMiniSeedClient import MockMiniSeedClient
class MockMiniSeedInputClient(object):
@@ -22,6 +23,14 @@ class MockMiniSeedInputClient(object):
self.last_sent = stream
+@pytest.fixture(scope="class")
+def miniseed_factory() -> MiniSeedFactory:
+ """instance of MiniSeedFactory with MockMiniseedClient"""
+ factory = MiniSeedFactory()
+ factory.client = MockMiniSeedClient()
+ yield factory
+
+
def test__put_timeseries():
"""edge_test.MiniSeedFactory_test.test__put_timeseries()"""
trace1 = __create_trace([0, 1, 2, 3, numpy.nan, 5, 6, 7, 8, 9], channel="H")
@@ -86,13 +95,11 @@ def test__set_metadata():
assert_equal(stream[1].stats["channel"], "H")
-def test_get_timeseries(MockMiniSeedClient):
+def test_get_timeseries(miniseed_factory):
"""edge_test.MiniSeedFactory_test.test_get_timeseries()"""
# Call get_timeseries, and test stats for comfirmation that it came back.
# TODO, need to pass in host and port from a config file, or manually
# change for a single test.
- miniseed_factory = MiniSeedFactory()
- miniseed_factory.client = MockMiniSeedClient()
timeseries = miniseed_factory.get_timeseries(
UTCDateTime(2015, 3, 1, 0, 0, 0),
UTCDateTime(2015, 3, 1, 1, 0, 0),
@@ -118,11 +125,8 @@ def test_get_timeseries(MockMiniSeedClient):
)
-def test_get_timeseries_by_location(MockMiniSeedClient):
+def test_get_timeseries_by_location(miniseed_factory):
"""test.edge_test.MiniSeedFactory_test.test_get_timeseries_by_location()"""
- miniseed_factory = MiniSeedFactory()
- miniseed_factory.client = MockMiniSeedClient()
-
timeseries = miniseed_factory.get_timeseries(
UTCDateTime(2015, 3, 1, 0, 0, 0),
UTCDateTime(2015, 3, 1, 1, 0, 0),
--
GitLab