diff --git a/geomagio/Controller.py b/geomagio/Controller.py index e5ab736b3a961583feceb6cd9ebb3f5924466aff..8dcb3eaea1d13f13783e73729c328fd47682df2d 100644 --- a/geomagio/Controller.py +++ b/geomagio/Controller.py @@ -11,6 +11,7 @@ import TimeseriesUtility import edge import iaga2002 import pcdcp +import imfv122 import imfv283 # factories for new filetypes @@ -300,6 +301,13 @@ def get_input_factory(args): elif input_url is not None: input_factory = iaga2002.IAGA2002Factory( **input_factory_args) + elif input_type == 'imfv122': + if input_stream is not None: + input_factory = imfv122.StreamIMFV122Factory( + **input_factory_args) + elif input_url is not None: + input_factory = imfv122.IMFV122Factory( + **input_factory_args) elif input_type == 'imfv283': if input_stream is not None: input_factory = imfv283.StreamIMFV283Factory( @@ -632,6 +640,7 @@ def parse_args(args): 'edge', 'goes', 'iaga2002', + 'imfv122', 'imfv283', 'pcdcp')) diff --git a/geomagio/imfv122/IMFV122Factory.py b/geomagio/imfv122/IMFV122Factory.py new file mode 100644 index 0000000000000000000000000000000000000000..667f124b33976c3ef67f9216c0f8ecc5398e2dcf --- /dev/null +++ b/geomagio/imfv122/IMFV122Factory.py @@ -0,0 +1,58 @@ +"""Factory that loads IMFV122 Files.""" + +import obspy.core +from .. import ChannelConverter +from ..TimeseriesFactory import TimeseriesFactory +from IMFV122Parser import IMFV122Parser + + +class IMFV122Factory(TimeseriesFactory): + """TimeseriesFactory for IMFV122 formatted files. + + Parameters + ---------- + See TimeseriesFactory + + See Also + -------- + IMFV122Parser + """ + + def __init__(self, **kwargs): + TimeseriesFactory.__init__(self, **kwargs) + + def parse_string(self, data, observatory=None, **kwargs): + """Parse the contents of a string in the format of an IAGA2002 file. + + Parameters + ---------- + iaga2002String : str + string containing IAGA2002 content. + observatory : str + observatory in case headers are unavailable. + parses observatory from headers when available. + Returns + ------- + obspy.core.Stream + parsed data. + """ + parser = IMFV122Parser(observatory=observatory) + parser.parse(data) + metadata = parser.metadata + starttime = obspy.core.UTCDateTime(parser.times[0]) + endtime = obspy.core.UTCDateTime(parser.times[-1]) + data = parser.data + length = len(data[data.keys()[0]]) + rate = (length - 1) / (endtime - starttime) + stream = obspy.core.Stream() + for channel in data.keys(): + stats = obspy.core.Stats(metadata) + stats.starttime = starttime + stats.sampling_rate = rate + stats.npts = length + stats.channel = channel + if channel == 'D': + data[channel] = ChannelConverter.get_radians_from_minutes( + data[channel]) + stream += obspy.core.Trace(data[channel], stats) + return stream diff --git a/geomagio/imfv122/IMFV122Parser.py b/geomagio/imfv122/IMFV122Parser.py new file mode 100644 index 0000000000000000000000000000000000000000..9f8e943712ae072a3a52fd27bd92dd9b16421309 --- /dev/null +++ b/geomagio/imfv122/IMFV122Parser.py @@ -0,0 +1,144 @@ +"""Parsing methods for the IMFV122 Format.""" + + +import numpy +from obspy.core import UTCDateTime + +# values that represent missing data points in IAGA2002 +EIGHTS = numpy.float64('88888.88') +NINES = numpy.float64('99999.99') + + +class IMFV122Parser(object): + """IMFV122 parser. + + Based on documentation at: + http://www.intermagnet.org/data-donnee/formats/imfv122-eng.php + + Attributes + ---------- + metadata : dict + parsed IMFV122 metadata. + channels : array + parsed channel names. + times : array + parsed timeseries times. + data : dict + keys are channel names (order listed in ``self.channels``). + values are ``numpy.array`` of timeseries values, array values are + ``numpy.nan`` when values are missing. + """ + + def __init__(self, observatory=None): + """Create a new IAGA2002 parser.""" + # header fields + self.metadata = { + 'network': 'NT', + 'station': observatory + } + # array of channel names + self.channels = [] + # timestamps of data (datetime.datetime) + self.times = [] + # dictionary of data (channel : numpy.array<float64>) + self.data = {} + # temporary storage for data being parsed + self._parsedata = ([], [], [], [], []) + + def parse(self, data): + """Parse a string containing IAGA2002 formatted data. + + Parameters + ---------- + data : str + IAGA 2002 formatted file contents. + """ + station = data[0:3] + lines = data.splitlines() + for line in lines: + if line.startswith(station): + self._parse_header(line) + else: + self._parse_data(line) + self._post_process() + + def _parse_header(self, line): + """Parse header line. + + Adds value to ``self.headers``. + """ + (observatory, + date, + doy, + start, + components, + type, + gin, + colalong, + decbas, + reserved) = line.split() + + self.channels = list(components) + self.metadata['declination_base'] = int(decbas) + self.metadata['geodetic_latitude'] = float(colalong[:4]) / 10 + self.metadata['geodetic_longitude'] = float(colalong[4:]) / 10 + self.metadata['station'] = observatory + self.metadata['gin'] = gin + + year = 1900 + int(date[-2:]) + julday = int(doy) + hour = 0 + minute = 0 + if year < 1971: + year = year + 100 + if len(start) == 2: + # minutes data + hour = int(start) + self._delta = 60 + else: + # seconds data + dayminute = int(start) + hour = int(dayminute / 60) + minute = dayminute % 60 + self._delta = 1 + self._nexttime = UTCDateTime( + year=year, + julday=julday, + hour=hour, + minute=minute) + + def _parse_data(self, line): + """Parse one data point in the timeseries. + + Adds time to ``self.times``. + Adds channel values to ``self.data``. + """ + (d11, d21, d31, d41, d12, d22, d32, d42) = line.split() + t, d1, d2, d3, d4 = self._parsedata + t.append(self._nexttime) + d1.append(d11) + d2.append(d21) + d3.append(d31) + d4.append(d41) + self._nexttime = self._nexttime + self._delta + t.append(self._nexttime) + d1.append(d12) + d2.append(d22) + d3.append(d32) + d4.append(d42) + self._nexttime = self._nexttime + self._delta + + def _post_process(self): + """Post processing after data is parsed. + + Converts data to numpy arrays. + Replaces empty values with ``numpy.nan``. + """ + self.times = self._parsedata[0] + for channel, data in zip(self.channels, self._parsedata[1:]): + data = numpy.array(data, dtype=numpy.float64) + data[data == int(EIGHTS)] = numpy.nan + data[data == EIGHTS] = numpy.nan + data[data == NINES] = numpy.nan + self.data[channel] = data / 10 + self._parsedata = None diff --git a/geomagio/imfv122/StreamIMFV122Factory.py b/geomagio/imfv122/StreamIMFV122Factory.py new file mode 100644 index 0000000000000000000000000000000000000000..4b60063d6177c2f8b36b6afcdc283976f16570d8 --- /dev/null +++ b/geomagio/imfv122/StreamIMFV122Factory.py @@ -0,0 +1,33 @@ +"""Factory to load IMFV122 files from an input stream.""" + +from IMFV122Factory import IMFV122Factory + + +class StreamIMFV122Factory(IMFV122Factory): + """Timeseries Factory for IMFV122 formatted files loaded via a stream. + normally either a single file, or stdio. + + Parameters + ---------- + stream: file object + io stream, normally either a file, or stdio + + See Also + -------- + IMFV122Factory + Timeseriesfactory + """ + def __init__(self, stream, **kwargs): + IMFV122Factory.__init__(self, **kwargs) + self._stream = stream + + def get_timeseries(self, starttime, endtime, observatory=None, + channels=None, type=None, interval=None): + """Implements get_timeseries + + Notes: Calls IMFV122Factory.parse_string in place of + IMFV122Factory.get_timeseries. + """ + return IMFV122Factory.parse_string(self, + data=self._stream.read(), + observatory=observatory) diff --git a/geomagio/imfv122/__init__.py b/geomagio/imfv122/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3b45ab11e20b9bd3cf25c2f2547ebd431a36644b --- /dev/null +++ b/geomagio/imfv122/__init__.py @@ -0,0 +1,16 @@ +"""IO Module for IMFV122 Format + +Based on documentation at: + http://www.intermagnet.org/data-donnee/formats/imfv122-eng.php +""" + +from IMFV122Factory import IMFV122Factory +from IMFV122Parser import IMFV122Parser +from StreamIMFV122Factory import StreamIMFV122Factory + + +__all__ = [ + 'IMFV122Factory', + 'IMFV122Parser', + 'StreamIMFV122Factory' +] diff --git a/test/imfv122_test/IMFV122Parser_test.py b/test/imfv122_test/IMFV122Parser_test.py new file mode 100644 index 0000000000000000000000000000000000000000..fea9af44ef32aa874d49607c81c65fc8ec07b660 --- /dev/null +++ b/test/imfv122_test/IMFV122Parser_test.py @@ -0,0 +1,76 @@ +"""Tests for the IMFV122 Parser class.""" + +from nose.tools import assert_equals +from geomagio.imfv122 import IMFV122Parser +from obspy.core import UTCDateTime + + +def test_imfv122_parse_header__minutes(): + """imfv122_test.test_imfv122_parse_header__minutes. + """ + parser = IMFV122Parser() + parser._parse_header( + 'KAK MAY0216 123 03 HDZF A KYO 05381402 000000 RRRRRRRRRRRRRRRR') + assert_equals(parser.channels, ['H', 'D', 'Z', 'F']) + metadata = parser.metadata + assert_equals(metadata['declination_base'], 0) + assert_equals(metadata['geodetic_latitude'], 53.8) + assert_equals(metadata['geodetic_longitude'], 140.2) + assert_equals(metadata['station'], 'KAK') + assert_equals(parser._delta, 60) + assert_equals(parser._nexttime, UTCDateTime('2016-05-02T03:00:00Z')) + + +def test_imfv122_parse_header__seconds(): + """imfv122_test.test_imfv122_parse_header__seconds. + """ + parser = IMFV122Parser() + parser._parse_header( + 'HER JAN0116 001 0123 HDZF R EDI 12440192 -14161 DRRRRRRRRRRRRRRR') + assert_equals(parser.channels, ['H', 'D', 'Z', 'F']) + metadata = parser.metadata + assert_equals(metadata['declination_base'], -14161) + assert_equals(metadata['geodetic_latitude'], 124.4) + assert_equals(metadata['geodetic_longitude'], 19.2) + assert_equals(metadata['station'], 'HER') + assert_equals(parser._delta, 1) + assert_equals(parser._nexttime, UTCDateTime('2016-01-01T02:03:00Z')) + + +def test_imfv122_parse_data(): + """imfv122_test.test_imfv122_parse_data. + """ + parser = IMFV122Parser() + parser._parse_header( + 'HER JAN0116 001 0123 HDZF R EDI 12440192 -14161 DRRRRRRRRRRRRRRR') + parser._parse_data('1234 5678 9101 1121 3141 5161 7181 9202') + assert_equals(parser._parsedata[0][0], UTCDateTime('2016-01-01T02:03:00Z')) + assert_equals(parser._parsedata[1][0], '1234') + assert_equals(parser._parsedata[2][0], '5678') + assert_equals(parser._parsedata[3][0], '9101') + assert_equals(parser._parsedata[4][0], '1121') + assert_equals(parser._parsedata[0][1], UTCDateTime('2016-01-01T02:03:01Z')) + assert_equals(parser._parsedata[1][1], '3141') + assert_equals(parser._parsedata[2][1], '5161') + assert_equals(parser._parsedata[3][1], '7181') + assert_equals(parser._parsedata[4][1], '9202') + + +def test_imfv122_post_process(): + """imfv122_test.test_imfv122_post_process. + """ + parser = IMFV122Parser() + parser._parse_header( + 'HER JAN0116 001 0123 HDZF R EDI 12440192 -14161 DRRRRRRRRRRRRRRR') + parser._parse_data('1234 5678 9101 1121 3141 5161 7181 9202') + parser._post_process() + assert_equals(parser.times[0], UTCDateTime('2016-01-01T02:03:00Z')) + assert_equals(parser.data['H'][0], 123.4) + assert_equals(parser.data['D'][0], 567.8) + assert_equals(parser.data['Z'][0], 910.1) + assert_equals(parser.data['F'][0], 112.1) + assert_equals(parser.times[1], UTCDateTime('2016-01-01T02:03:01Z')) + assert_equals(parser.data['H'][1], 314.1) + assert_equals(parser.data['D'][1], 516.1) + assert_equals(parser.data['Z'][1], 718.1) + assert_equals(parser.data['F'][1], 920.2) diff --git a/test/imfv122_test/__init__.py b/test/imfv122_test/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391