Fixed npts issues in create missing and channels as well as a unit test. Also...

Fixed npts issues in create missing and channels as well as a unit test. Also cleaned up the unit test code for syntax and linting.

geomagio/edge/EdgeFactory.py

@@ -221,7 +221,6 @@ class EdgeFactory(TimeseriesFactory):
             trace_starttime = obspy.core.UTCDateTime(trace.stats.starttime)
             trace_endtime = obspy.core.UTCDateTime(trace.stats.endtime)
             trace_delta = trace.stats.delta
-
             if trace_starttime > starttime:
                 cnt = int((trace_starttime - starttime) / trace_delta)
                 if cnt > 0:
@@ -230,15 +229,12 @@ class EdgeFactory(TimeseriesFactory):
                             trace.data])
                     trace_starttime = trace_starttime - trace_delta * cnt
                     trace.stats.starttime = trace_starttime
-
             if trace_endtime < endtime:
                 cnt = int((endtime - trace_endtime) / trace.stats.delta)
                 if cnt > 0:
                     trace.data = numpy.concatenate([
                             trace.data,
                             numpy.full(cnt, numpy.nan, dtype=numpy.float64)])
-                    trace_endtime = trace_endtime + trace_delta * cnt
-                    trace.stats.endttime = trace_endtime
 
     def _convert_timeseries_to_decimal(self, stream):
         """convert geomag edge timeseries data stored as ints, to decimal by
@@ -326,24 +322,28 @@ class EdgeFactory(TimeseriesFactory):
         obspy.core.Stream
             stream of the requested channel data
         """
-        stats = obspy.core.Stats()
-        stats.channel = channel
-        stats.starttime = starttime
-        stats.network = network
-        stats.station = station
-        stats.location = location
         if interval == 'second':
-            stats.sampling_rate = 1.
+            delta = 1.
         elif interval == 'minute':
-            stats.sampling_rate = 1. / 60.
+            delta = 60.
         elif interval == 'hourly':
-            stats.sampling_rate = 1. / 3600.
+            delta = 3600.
         elif interval == 'daily':
-            stats.sampling_rate = 1. / 86400.
-        length = int((endtime - starttime) * stats.sampling_rate)
+            delta = 86400.
+        stats = obspy.core.Stats()
+        stats.network = network
+        stats.station = station
+        stats.location = location
+        stats.channel = channel
+        # Calculate first valid sample time based on interval
+        trace_starttime = obspy.core.UTCDateTime(
+            numpy.ceil(starttime.timestamp / delta) * delta)
+        stats.starttime = trace_starttime
+        stats.delta = delta
+        # Calculate number of valid samples up to or before endtime
+        length = int((endtime - trace_starttime) / delta)
         stats.npts = length + 1
-
-        data = numpy.full(length, numpy.nan, dtype=numpy.float64)
+        data = numpy.full(stats.npts, numpy.nan, dtype=numpy.float64)
         return obspy.core.Stream(obspy.core.Trace(data, stats))
 
     def _get_edge_channel(self, observatory, channel, type, interval):

test/edge_test/EdgeFactory_test.py

@@ -99,28 +99,78 @@ def test_clean_timeseries():
     """
     edge_factory = EdgeFactory()
     trace1 = _create_trace([1, 1, 1, 1, 1], 'H', UTCDateTime("2018-01-01"))
-    trace2 = _create_trace([2, 2],'E',UTCDateTime("2018-01-01"))
+    trace2 = _create_trace([2, 2], 'E', UTCDateTime("2018-01-01"))
     timeseries = Stream(traces=[trace1, trace2])
-    edge_factory._clean_timeseries(timeseries,trace1.stats.starttime,trace1.stats.endtime)
-    assert_equals(len(trace1.data),len(trace2.data))
-    assert_equals(trace1.stats.starttime,trace2.stats.starttime)
-    assert_equals(trace1.stats.endtime,trace2.stats.endtime)
+    edge_factory._clean_timeseries(
+        timeseries=timeseries,
+        starttime=trace1.stats.starttime,
+        endtime=trace1.stats.endtime)
+    assert_equals(len(trace1.data), len(trace2.data))
+    assert_equals(trace1.stats.starttime, trace2.stats.starttime)
+    assert_equals(trace1.stats.endtime, trace2.stats.endtime)
     # change starttime by less than 1 delta
     starttime = trace1.stats.starttime
     endtime = trace1.stats.endtime
-    edge_factory._clean_timeseries(timeseries,starttime-30,endtime+30)
-    assert_equals(trace1.stats.starttime,starttime)
+    edge_factory._clean_timeseries(timeseries, starttime - 30, endtime + 30)
+    assert_equals(trace1.stats.starttime, starttime)
     # Change starttime by more than 1 delta
-    edge_factory._clean_timeseries(timeseries,starttime-90,endtime+90)
-    assert_equals(trace1.stats.starttime,starttime-60)
-    assert_equals(np.isnan(trace1.data[0]),np.isnan(np.NaN))
+    edge_factory._clean_timeseries(timeseries, starttime - 90, endtime + 90)
+    assert_equals(trace1.stats.starttime, starttime - 60)
+    assert_equals(np.isnan(trace1.data[0]), np.isnan(np.NaN))
 
 
-def _create_trace(data,channel,starttime,delta=60.):
+def test_create_missing_channel():
+    """edge_test.EdgeFactory_test.test_create_missing_channel()
+    """
+    edge_factory = EdgeFactory()
+    trace1 = _create_trace([1, 1, 1, 1, 1], 'H', UTCDateTime("2018-01-01"))
+    trace2 = _create_trace([2, 2], 'E', UTCDateTime("2018-01-01"))
+    observatory = 'Test'
+    type = 'variation'
+    interval = 'minute'
+    network = 'NT'
+    location = 'R0'
+    trace3 = edge_factory._create_missing_channel(
+        starttime=trace1.stats.starttime,
+        endtime=trace1.stats.endtime,
+        observatory=observatory,
+        channel='F',
+        type=type,
+        interval=interval,
+        network=network,
+        station=trace1.stats.station,
+        location=location)
+    timeseries = Stream(traces=[trace1, trace2])
+    # For continuity set stats to be same for all traces
+    for trace in timeseries:
+        trace.stats.observatory = observatory
+        trace.stats.type = type
+        trace.stats.interval = interval
+        trace.stats.network = network
+        trace.stats.station = trace1.stats.station
+        trace.stats.location = location
+    timeseries += trace3
+    assert_equals(len(trace3[0].data), trace3[0].stats.npts)
+    assert_equals(timeseries[0].stats.starttime, timeseries[2].stats.starttime)
+    edge_factory._clean_timeseries(
+        timeseries=timeseries,
+        starttime=trace1.stats.starttime,
+        endtime=trace1.stats.endtime)
+    assert_equals(len(trace3[0].data), trace3[0].stats.npts)
+    assert_equals(timeseries[0].stats.starttime, timeseries[2].stats.starttime)
+    # Change starttime by more than 1 delta
+    starttime = trace1.stats.starttime
+    endtime = trace1.stats.endtime
+    edge_factory._clean_timeseries(timeseries, starttime - 90, endtime + 90)
+    assert_equals(len(trace3[0].data), trace3[0].stats.npts)
+    assert_equals(timeseries[0].stats.starttime, timeseries[2].stats.starttime)
+
+
+def _create_trace(data, channel, starttime, delta=60.):
     stats = Stats()
     stats.channel = channel
     stats.delta = delta
     stats.starttime = starttime
     stats.npts = len(data)
-    data = np.array(data,dtype=np.float64)
-    return Trace(data,stats)
+    data = np.array(data, dtype=np.float64)
+    return Trace(data, stats)