From e74751f06eb1466c8603bc8ea2c46a981957a808 Mon Sep 17 00:00:00 2001 From: Jeremy Fee <jmfee@usgs.gov> Date: Fri, 14 Feb 2020 12:58:20 -0700 Subject: [PATCH] Change how conversions are configured, in line with future metadata service --- geomagio/Metadata.py | 90 ++++++++++++++++++ geomagio/ObservatoryMetadata.py | 148 ------------------------------ geomagio/edge/MiniSeedFactory.py | 151 ++++++++++++++++++++----------- 3 files changed, 187 insertions(+), 202 deletions(-) create mode 100644 geomagio/Metadata.py diff --git a/geomagio/Metadata.py b/geomagio/Metadata.py new file mode 100644 index 000000000..40686a246 --- /dev/null +++ b/geomagio/Metadata.py @@ -0,0 +1,90 @@ +"""Simulate metadata service until it is implemented. +""" + + +def get_instrument(observatory, start_time=None, end_time=None, metadata=None): + """Get instrument metadata + + Args: + observatory: observatory code + start_time: start time to match, or None to match any. + end_time: end time to match, or None to match any. + metadata: use custom list, defaults to _INSTRUMENT_METADATA + Returns: + list of matching metadata + """ + metadata = metadata or _INSTRUMENT_METADATA + return [ + m + for m in metadata + if m["station"] == observatory and + (end_time is None or + m["start_time"] is None or + m["start_time"] < end_time) and + (start_time is None or + m["end_time"] is None or + m["end_time"] > start_time) + ] + + +""" +To make this list easier to maintain: + - List NT network stations first, then other networks in alphabetical order + - Within networks, alphabetize by station, then start_time. +""" +_INSTRUMENT_METADATA = [ + { + "network": "NT", + "station": "BDT", + "start_time": None, + "end_time": None, + "instrument": { + "type": "FGE", + "channels": { + # each channel maps to a list of components to calculate nT + # TODO: calculate these lists based on "FGE" type + "U": [{"channel": "U_Volt", "offset": 0, "scale": 313.2}], + "V": [{"channel": "V_Volt", "offset": 0, "scale": 312.3}], + "W": [{"channel": "Z_Volt", "offset": 0, "scale": 312.0}], + }, + "electronics": { + "serial": "E0542", + # these scale values are used to convert voltage + "x-scale": 313.2, # V/nT + "y-scale": 312.3, # V/nT + "z-scale": 312.0, # V/nT + "temperature-scale": 0.01, # V/K + }, + "sensor": { + "serial": "S0419", + # these constants combine with instrument setting for offset + "x-constant": 36958, # nT/mA + "y-constant": 36849, # nT/mA + "z-constant": 36811, # nT/mA + }, + }, + }, + { + "network": "NT", + "station": "LLO", + "start_time": None, + "end_time": None, + "instrument": { + "type": "Narod", + "channels": { + "U": [ + {"channel": "U_Volt", "offset": 0, "scale": 100}, + {"channel": "U_Bin", "offset": 0, "scale": 500}, + ], + "V": [ + {"channel": "V_Volt", "offset": 0, "scale": 100}, + {"channel": "V_Bin", "offset": 0, "scale": 500}, + ], + "W": [ + {"channel": "W_Volt", "offset": 0, "scale": 100}, + {"channel": "W_Bin", "offset": 0, "scale": 500}, + ], + }, + }, + }, +] diff --git a/geomagio/ObservatoryMetadata.py b/geomagio/ObservatoryMetadata.py index 142170fab..a35776b8e 100644 --- a/geomagio/ObservatoryMetadata.py +++ b/geomagio/ObservatoryMetadata.py @@ -13,10 +13,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 5527, - 'fge': True, - 'U_scale': 313.2, - 'V_scale': 312.3, - 'W_scale': 312.0, 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -47,10 +43,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 5527, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -81,10 +73,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 10000.0, 'declination_base': 5527, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -115,10 +103,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 10589, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -149,10 +133,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 10000.0, 'declination_base': 10589, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -183,10 +163,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 215772, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -217,10 +193,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 12151, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -238,10 +210,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 10000.0, 'declination_base': 12151, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -259,10 +227,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 10755, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -293,10 +257,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 10000.0, 'declination_base': 10755, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -327,10 +287,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 209690, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -361,10 +317,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 209690, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -395,10 +347,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 8097, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -429,10 +377,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 764, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -463,10 +407,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 5982, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -497,10 +437,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'reported': 'HDZF', 'sensor_sampling_rate': 0.01, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -526,10 +462,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 9547, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -560,10 +492,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 7386, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -594,10 +522,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 12349, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -628,10 +552,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 208439, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -662,10 +582,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 5863, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -696,10 +612,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'sensor_sampling_rate': 100.0, 'declination_base': 0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -726,10 +638,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -755,10 +663,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -784,10 +688,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -813,10 +713,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -842,10 +738,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -871,10 +763,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'reported': 'HDZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -900,10 +788,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'HDZF', 'reported': 'HDZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -929,10 +813,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -958,10 +838,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -987,10 +863,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -1016,10 +888,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -1045,10 +913,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -1074,10 +938,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -1103,10 +963,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + @@ -1132,10 +988,6 @@ DEFAULT_METADATA = { 'sensor_orientation': 'XYZF', 'reported': 'XYZF', 'sensor_sampling_rate': 100.0, - 'fge': False, - 'U_scale': 0., - 'V_scale': 0., - 'W_scale': 0., 'is_gin': False, 'is_intermagnet': False, 'conditions_of_use': 'The Conditions of Use for data provided' + diff --git a/geomagio/edge/MiniSeedFactory.py b/geomagio/edge/MiniSeedFactory.py index 2f9ad74c0..97edd82c1 100644 --- a/geomagio/edge/MiniSeedFactory.py +++ b/geomagio/edge/MiniSeedFactory.py @@ -18,6 +18,7 @@ import obspy.core from obspy.clients.neic import client as miniseed from .. import ChannelConverter, TimeseriesUtility +from ..Metadata import get_instrument from ..TimeseriesFactory import TimeseriesFactory from ..TimeseriesFactoryException import TimeseriesFactoryException from ..ObservatoryMetadata import ObservatoryMetadata @@ -79,8 +80,6 @@ class MiniSeedFactory(TimeseriesFactory): self.volt_conv = volt_conv self.bin_conv = bin_conv self.write_client = MiniSeedInputClient(self.host, self.write_port) - if isinstance(self.observatory, list) is True: - self.observatory = self.observatory[0] def get_timeseries(self, starttime, endtime, observatory=None, channels=None, type=None, interval=None): @@ -129,30 +128,17 @@ class MiniSeedFactory(TimeseriesFactory): # get the timeseries timeseries = obspy.core.Stream() for channel in channels: - data = self._get_timeseries(starttime, endtime, observatory, - channel, type, interval) + if channel in self.convert_channels: + data = self._convert_timeseries(starttime, endtime, + observatory, channel, type, interval) + else: + data = self._get_timeseries(starttime, endtime, + observatory, channel, type, interval) timeseries += data finally: # restore stdout sys.stdout = original_stdout - mdata = self.observatoryMetadata.metadata - # check for presence of observatory in metadata - if self.observatory not in mdata.keys(): - fge = False - - else: - fge = mdata[self.observatory]['metadata']['fge'] - - if self.convert_channels is not None: - out = obspy.core.Stream() - for channel in self.convert_channels: - _in_ = timeseries.select(channel=channel + "_Volt") - if fge is not True: - _in_ += timeseries.select(channel=channel + '_Bin') - out += self.convert_voltbin(channel, _in_) - timeseries = out - self._post_process(timeseries, starttime, endtime, channels) return timeseries @@ -466,42 +452,100 @@ class MiniSeedFactory(TimeseriesFactory): observatory, channel, type, interval) return data - def convert_voltbin(self, channel, stream): - """Convert miniseed data from bins and volts to nT. - Converts all traces in stream. + def _convert_timeseries(self, starttime, endtime, observatory, + channel, type, interval): + """Generate a single channel using multiple components. + + Finds metadata, then calls _get_converted_timeseries for actual + conversion. + Parameters ---------- - stream: obspy.core.Stream - stream of data to convert - channel: string - channel string(U ,V ,W) + starttime: obspy.core.UTCDateTime + the starttime of the requested data + endtime: obspy.core.UTCDateTime + the endtime of the requested data + observatory : str + observatory code + channel : str + single character channel {H, E, D, Z, F} + type : str + data type {definitive, quasi-definitive, variation} + interval : str + interval length {'day', 'hour', 'minute', 'second', 'tenhertz'} + + Returns + ------- + obspy.core.trace + timeseries trace of the requested channel data + """ + out = obspy.core.Stream() + metadata = get_instrument(observatory, starttime, endtime) + # loop in case request spans different configurations + for instrument in metadata: + instrument_channels = instrument["channels"] + instrument_endtime = instrument["end_time"] + instrument_starttime = instrument["start_time"] + if channel not in instrument_channels: + # no idea how to convert + continue + # determine metadata overlap with request + start = (starttime + if instrument_starttime is None or + instrument_starttime < starttime + else instrument_starttime) + end = (endtime + if instrument_endtime is None or + instrument_endtime > endtime + else instrument_endtime) + # now convert + out += self._get_converted_timeseries(start, end, + observatory, channel, type, interval, + instrument_channels[channel]) + return out + + def _get_converted_timeseries(self, starttime, endtime, observatory, + channel, type, interval, components): + """Generate a single channel using multiple components. + + Parameters + ---------- + starttime: obspy.core.UTCDateTime + the starttime of the requested data + endtime: obspy.core.UTCDateTime + the endtime of the requested data + observatory : str + observatory code + channel : str + single character channel {H, E, D, Z, F} + type : str + data type {definitive, quasi-definitive, variation} + interval : str + interval length {'day', 'hour', 'minute', 'second', 'tenhertz'} + components: list + each component is a dictionary with the following keys: + channel: str + offset: float + scale: float + Returns ------- - out : obspy.core.Trace - Trace containing 1 trace per 2 original traces. + obspy.core.trace + timeseries trace of the converted channel data """ - out = obspy.core.Trace() - mdata = self.observatoryMetadata.metadata - # Checks for presence of observatory in metadata - if self.observatory not in mdata.keys(): - # selects volts from input Trace - volts = stream.select(channel=channel + "_Volt")[0] - # selects bins from input Trace - bins = stream.select(channel=channel + "_Bin")[0] - # conversion from bins/volts to nT - data = self.volt_conv * volts.data \ - + self.bin_conv * bins.data - else: - mdata = mdata[self.observatory]['metadata'] - # get scaling factor from observatory metadata - scale = mdata[channel + '_scale'] - # selects volts from input Trace - volts = stream.select(channel=channel + "_Volt")[0] - # conversion from bins/volts to nT - data = 150 * (volts.data + scale) - - # copy stats from original Trace - stats = obspy.core.Stats(volts.stats) + # sum channels + stats = None + converted = None + for component in components: + # load component + data = self._get_timeseries(starttime, endtime, observatory, + component["channel"], type, interval) + # save stats from first component + stats = stats or obspy.core.Stats(data.stats) + # convert to nT + nt = data.data * component["scale"] + component["offset"] + # add to converted + converted = converted and converted + nt or nt # set channel parameter to U, V, or W stats.channel = channel # create empty trace with adapted stats @@ -509,8 +553,7 @@ class MiniSeedFactory(TimeseriesFactory): stats.endtime, stats.station, stats.channel, stats.data_type, stats.data_interval, stats.network, stats.station, stats.location) - # set data for empty trace as nT converted data - out.data = data + out.data = converted return out def _post_process(self, timeseries, starttime, endtime, channels): -- GitLab