From 2cddc482d6ea24d44aa19484d9124765e26b2b59 Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Wed, 18 May 2022 10:43:32 -0600
Subject: [PATCH 1/7] Attempt StringIO after failed BytesIO file read
This is a bandaid to allow the `--input-url` option to be used to
read in single files. It adds zero functionality beyond what already
existed when the `--input-file` option was used.
---
geomagio/Controller.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/geomagio/Controller.py b/geomagio/Controller.py
index d05f45ae..41bffe20 100644
--- a/geomagio/Controller.py
+++ b/geomagio/Controller.py
@@ -1,7 +1,7 @@
"""Controller class for geomag algorithms"""
import argparse
-from io import BytesIO
+from io import BytesIO, StringIO
import sys
from typing import List, Optional, Tuple, Union
@@ -506,7 +506,12 @@ def get_input_factory(args):
input_factory_args["urlInterval"] = args.input_url_interval
input_factory_args["urlTemplate"] = args.input_url
else:
- input_stream = BytesIO(Util.read_url(args.input_url))
+ try:
+ input_stream = BytesIO(Util.read_url(args.input_url))
+ except TypeError as e:
+ print(str(e), file=sys.stderr)
+ print("Warning: reading url as BytesIO failed; attempting StringIO", file=sys.stderr)
+ input_stream = StringIO(Util.read_url(args.input_url))
input_type = args.input
if input_type == "edge":
input_factory = edge.EdgeFactory(
--
GitLab
From 747487e27478af4eca7d4deaf132c99a5808c4c4 Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Wed, 18 May 2022 13:37:58 -0600
Subject: [PATCH 2/7] Fix Black-recommended linting issue
---
geomagio/Controller.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/geomagio/Controller.py b/geomagio/Controller.py
index 41bffe20..18634056 100644
--- a/geomagio/Controller.py
+++ b/geomagio/Controller.py
@@ -510,7 +510,10 @@ def get_input_factory(args):
input_stream = BytesIO(Util.read_url(args.input_url))
except TypeError as e:
print(str(e), file=sys.stderr)
- print("Warning: reading url as BytesIO failed; attempting StringIO", file=sys.stderr)
+ print(
+ "Warning: reading url as BytesIO failed; attempting StringIO",
+ file=sys.stderr,
+ )
input_stream = StringIO(Util.read_url(args.input_url))
input_type = args.input
if input_type == "edge":
--
GitLab
From 03125a5c95e64b41f92f75654062f0b1bfc35c08 Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Wed, 18 May 2022 15:17:35 -0600
Subject: [PATCH 3/7] Add test for --input-url to work with single file
It worked when a URL template was used, but not if a
single simple filename was passed
---
test/Controller_test.py | 79 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 79 insertions(+)
diff --git a/test/Controller_test.py b/test/Controller_test.py
index 4a722ada..2870645a 100644
--- a/test/Controller_test.py
+++ b/test/Controller_test.py
@@ -33,6 +33,85 @@ def test_controller():
assert_equal(isinstance(controller._algorithm, Algorithm), True)
+def test_controller_input_url():
+ """Controller_test.test_controller_input_url()
+
+ Test `--input-url` option in controller with both a URL template, and
+ and a simple, single file url. The latter was especially problematic after
+ switching to Py3 because BytesIO choked on data read in as a string.
+ NOTE: the bug this was designed to catch was TypeError, so a "pass" is
+ simply to read in the data file. There is no need to compare it
+ with anything.
+ """
+ # define folder for testing
+ tmp_dir = gettempdir()
+
+ # TEST 1 - read in interval using the --input-url option set to a
+ # URL template (i.e., it has a '{' in the string)
+
+ # create list of string command line arguments
+ fake_argv = [
+ "--input",
+ "iaga2002",
+ "--input-url",
+ "file://etc/controller/{obs}{date:%Y%m%d}_XYZF_{t}{i}.{i}",
+ "--observatory",
+ "BOU",
+ "--inchannels",
+ "X",
+ "Y",
+ "Z",
+ "F",
+ "--interval",
+ "minute",
+ "--type",
+ "variation",
+ "--output",
+ "iaga2002",
+ "--output-url",
+ "file://" + tmp_dir + "/{obs}{date:%Y%m%d}_XYZF_{t}{i}.{i}",
+ ]
+
+ # parse arguments and create initial args object
+ args = parse_args(fake_argv)
+
+ starttime1 = args.starttime = UTCDateTime("2018-10-24T00:00:00Z")
+ endtime1 = args.endtime = UTCDateTime("2018-10-24T00:19:00Z")
+ _main(args)
+
+ # TEST 2 - read in interval using the --input-url option with no
+ # URL template (i.e., no '{' in the string, just a single filename)
+ # create list of string command line arguments
+ fake_argv = [
+ "--input",
+ "iaga2002",
+ "--input-url",
+ "file://etc/controller/bou20181024_XYZF_vmin.min",
+ "--observatory",
+ "BOU",
+ "--inchannels",
+ "X",
+ "Y",
+ "Z",
+ "F",
+ "--interval",
+ "minute",
+ "--type",
+ "variation",
+ "--output",
+ "iaga2002",
+ "--output-url",
+ "file://" + tmp_dir + "/bou20181024_XYZF_noURL_vmin.min",
+ ]
+
+ # parse arguments and create initial args object
+ args = parse_args(fake_argv)
+
+ starttime1 = args.starttime = UTCDateTime("2018-10-24T00:00:00Z")
+ endtime1 = args.endtime = UTCDateTime("2018-10-24T00:19:00Z")
+ _main(args)
+
+
def test_controller_update_sqdist():
"""Controller_test.test_controller_update_sqdist().
--
GitLab
From 1605f929c7786c3b11364eb2f5e2da9a32ec012a Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Wed, 18 May 2022 15:20:00 -0600
Subject: [PATCH 4/7] Fix Black-recommended linting issue
---
test/Controller_test.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/Controller_test.py b/test/Controller_test.py
index 2870645a..2e51338a 100644
--- a/test/Controller_test.py
+++ b/test/Controller_test.py
@@ -71,7 +71,7 @@ def test_controller_input_url():
"--output-url",
"file://" + tmp_dir + "/{obs}{date:%Y%m%d}_XYZF_{t}{i}.{i}",
]
-
+
# parse arguments and create initial args object
args = parse_args(fake_argv)
@@ -103,7 +103,7 @@ def test_controller_input_url():
"--output-url",
"file://" + tmp_dir + "/bou20181024_XYZF_noURL_vmin.min",
]
-
+
# parse arguments and create initial args object
args = parse_args(fake_argv)
--
GitLab
From e918b4c9029a97efa10f919eb58ca194d32f368c Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Fri, 27 May 2022 13:04:57 -0600
Subject: [PATCH 5/7] Remove BYTESIO object creation
It turns out that none of the factories currently being used in
production expects bytes as input, and in fact, Util.read_url()
converts everthing to a string anyway, so a BYTES-oriented input
would couldn't not possibly be working. So, just remove all use
of BYTESIO, and instead use only STRINGIO.
---
geomagio/Controller.py | 12 ++----------
1 file changed, 2 insertions(+), 10 deletions(-)
diff --git a/geomagio/Controller.py b/geomagio/Controller.py
index 18634056..aa717974 100644
--- a/geomagio/Controller.py
+++ b/geomagio/Controller.py
@@ -1,7 +1,7 @@
"""Controller class for geomag algorithms"""
import argparse
-from io import BytesIO, StringIO
+from io import StringIO
import sys
from typing import List, Optional, Tuple, Union
@@ -506,15 +506,7 @@ def get_input_factory(args):
input_factory_args["urlInterval"] = args.input_url_interval
input_factory_args["urlTemplate"] = args.input_url
else:
- try:
- input_stream = BytesIO(Util.read_url(args.input_url))
- except TypeError as e:
- print(str(e), file=sys.stderr)
- print(
- "Warning: reading url as BytesIO failed; attempting StringIO",
- file=sys.stderr,
- )
- input_stream = StringIO(Util.read_url(args.input_url))
+ input_stream = StringIO(Util.read_url(args.input_url))
input_type = args.input
if input_type == "edge":
input_factory = edge.EdgeFactory(
--
GitLab
From 2be6c90c68616c99e9754c803cf9b399afe1214a Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Fri, 27 May 2022 13:32:19 -0600
Subject: [PATCH 6/7] Empty commit
Trying to dislodge a plugged pipeline in Gitlab
--
GitLab
From 0a33ca7198580b411d44b775395f004cdf9f7931 Mon Sep 17 00:00:00 2001
From: "E. Joshua Rigler" <erigler@usgs.gov>
Date: Fri, 27 May 2022 13:41:46 -0600
Subject: [PATCH 7/7] Tweak .gitlab-ci.yml for new Gitlab requirment
Gitlab 15 requires slightly different configuration to generate
Cobertura coverage reports. See:
https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report
---
.gitlab-ci.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f9d27295..c7e28786 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -153,7 +153,9 @@ Python Lint:
Python Test:
artifacts:
reports:
- cobertura: coverage.xml
+ coverage_report:
+ coverage_format: cobertura
+ path: coverage.xml
junit: junit.xml
needs:
- Poetry
--
GitLab