From 635400dfda794490d9fc25a2669ab62a67682359 Mon Sep 17 00:00:00 2001
From: Travis Rivers <travrivers88@gmail.com>
Date: Wed, 4 Mar 2020 15:44:22 -0700
Subject: [PATCH] Fix parsing and validation errors, clean up
---
geomagio/TimeseriesUtility.py | 9 +-
geomagio/webservice/data.py | 154 ++++++++++--------
geomagio/webservice/static/usage.css | 4 -
geomagio/webservice/templates/data/usage.html | 126 ++++++++------
4 files changed, 167 insertions(+), 126 deletions(-)
diff --git a/geomagio/TimeseriesUtility.py b/geomagio/TimeseriesUtility.py
index dab3e801..8b0d3ee4 100644
--- a/geomagio/TimeseriesUtility.py
+++ b/geomagio/TimeseriesUtility.py
@@ -82,6 +82,7 @@ def get_delta_from_interval(data_interval):
delta = None
return delta
+
def get_interval_from_delta(delta):
"""Convert delta to an interval name
@@ -95,15 +96,15 @@ def get_interval_from_delta(delta):
interval : str
interval length {day, hour, minute, second, tenhertz}
"""
- if delta == "0.1":
+ if delta == 0.1:
data_interval = "tenhertz"
elif delta == 1:
data_interval = "second"
- elif delta == "60":
+ elif delta == 60:
data_interval = "minute"
- elif delta == "3600":
+ elif delta == 3600:
data_interval = "hour"
- elif delta == "86400":
+ elif delta == 86400:
data_interval = "day"
else:
data_interval = delta
diff --git a/geomagio/webservice/data.py b/geomagio/webservice/data.py
index 66f33688..5811db60 100644
--- a/geomagio/webservice/data.py
+++ b/geomagio/webservice/data.py
@@ -11,10 +11,10 @@ from ..imfjson import IMFJSONWriter
from ..TimeseriesUtility import get_interval_from_delta
-DEFAULT_DATA_TYPE = 'variation'
-DEFAULT_ELEMENTS = ('X', 'Y', 'Z', 'F')
-DEFAULT_OUTPUT_FORMAT = 'iaga2002'
-DEFAULT_SAMPLING_PERIOD = '60'
+DEFAULT_DATA_TYPE = "variation"
+DEFAULT_ELEMENTS = ("X", "Y", "Z", "F")
+DEFAULT_OUTPUT_FORMAT = "iaga2002"
+DEFAULT_SAMPLING_PERIOD = "60"
ERROR_CODE_MESSAGES = {
204: "No Data",
400: "Bad Request",
@@ -22,20 +22,41 @@ ERROR_CODE_MESSAGES = {
409: "Conflict",
500: "Internal Server Error",
501: "Not Implemented",
- 503: "Service Unavailable"
+ 503: "Service Unavailable",
}
VALID_DATA_TYPES = ["variation", "adjusted", "quasi-definitive", "definitive"]
VALID_INTERVALS = ["tenhertz", "second", "minute", "hour", "day"]
-VALID_OBSERVATORIES = ["BRT", "BRW", "DED", "DHT", "CMO", "CMT", "SIT", "SHU", "NEW",
-"BDT", "BOU", "TST", "USGS", "FDT", "FRD", "FRN", "TUC", "BSL", "HON", "SJG",
-"GUA", "SJT"]
+VALID_OBSERVATORIES = [
+ "BRT",
+ "BRW",
+ "DED",
+ "DHT",
+ "CMO",
+ "CMT",
+ "SIT",
+ "SHU",
+ "NEW",
+ "BDT",
+ "BOU",
+ "TST",
+ "USGS",
+ "FDT",
+ "FRD",
+ "FRN",
+ "TUC",
+ "BSL",
+ "HON",
+ "SJG",
+ "GUA",
+ "SJT",
+]
VALID_OUTPUT_FORMATS = ["iaga2002", "json"]
VALID_SAMPLING_PERIODS = ["0.1", "1", "60", "3600", "86400"]
blueprint = Blueprint("data", __name__)
input_factory = None
-VERSION = 'version'
+VERSION = "version"
def init_app(app: Flask):
@@ -72,6 +93,7 @@ def get_data():
class WebServiceException(Exception):
"""Base class for exceptions thrown by web services."""
+
pass
@@ -97,6 +119,7 @@ class WebServiceQuery(object):
output format.
default 'iaga2002'.
"""
+
def __init__(
self,
observatory_id=None,
@@ -105,7 +128,7 @@ class WebServiceQuery(object):
elements=("X", "Y", "Z", "F"),
sampling_period=60,
data_type="variation",
- output_format="iaga2002"
+ output_format="iaga2002",
):
self.observatory_id = observatory_id
self.starttime = starttime
@@ -118,15 +141,10 @@ class WebServiceQuery(object):
def format_error(status_code, exception):
"""Assign error_body value based on error format."""
-
- if request.args.get("output_format") == 'json':
- return Response(
- json_error(status_code, exception, request.url),
- mimetype="application/json")
+ if request.args.get("format") == "json":
+ return Response(json_error(status_code, exception), mimetype="application/json")
else:
- return Response(
- iaga2002_error(status_code, exception, request.query_string),
- mimetype="text/plain")
+ return Response(iaga2002_error(status_code, exception), mimetype="text/plain",)
def format_timeseries(timeseries, query):
@@ -148,11 +166,12 @@ def format_timeseries(timeseries, query):
if query.output_format == "json":
return Response(
IMFJSONWriter.format(timeseries, query.elements),
- mimetype="application/json")
+ mimetype="application/json",
+ )
else:
return Response(
- IAGA2002Writer.format(timeseries, query.elements),
- mimetype="text/plain")
+ IAGA2002Writer.format(timeseries, query.elements), mimetype="text/plain"
+ )
def get_input_factory():
@@ -163,15 +182,12 @@ def get_input_factory():
input_factory
Edge or miniseed factory object
"""
- data_type = os.getenv('DATA_TYPE', 'edge')
- host = os.getenv('DATA_HOST', 'cwbpub.cr.usgs.gov')
- port = os.getenv('DATA_PORT', 2060)
-
- if data_type == 'edge':
- input_factory = EdgeFactory(
- host=host,
- port=port,
- type=data_type)
+ data_type = os.getenv("DATA_TYPE", "edge")
+ host = os.getenv("DATA_HOST", "cwbpub.cr.usgs.gov")
+ port = os.getenv("DATA_PORT", 2060)
+
+ if data_type == "edge":
+ input_factory = EdgeFactory(host=host, port=port, type=data_type)
return input_factory
else:
return None
@@ -196,7 +212,8 @@ def get_timeseries(query):
query.observatory_id,
query.elements,
query.data_type,
- query.sampling_period)
+ query.sampling_period,
+ )
return timeseries
@@ -245,11 +262,10 @@ def json_error(code, message):
"generated": date,
"url": request.url,
"title": status_message,
- "error": message
- }
+ "error": message,
+ },
}
- return dumps(error_dict,
- sort_keys=True).encode('utf8')
+ return dumps(error_dict, sort_keys=True).encode("utf8")
def parse_query(query):
@@ -272,40 +288,52 @@ def parse_query(query):
"""
# Get values
observatory_id = query.get("observatory")
- elements = query.get("channels", DEFAULT_ELEMENTS)
+ start_time = query.get("starttime")
+ end_time = query.get("endtime")
+ elements = query.getlist("channels")
sampling_period = query.get("sampling_period", DEFAULT_SAMPLING_PERIOD)
data_type = query.get("type", DEFAULT_DATA_TYPE)
output_format = query.get("format", DEFAULT_OUTPUT_FORMAT)
# Format values and get time values
output_format.lower()
observatory_id.upper()
-
- try:
- start_time = UTCDateTime(query.get('starttime'))
- except:
- start_time = query.get('starttime')
+ elements = [e.split(",") for e in elements]
+ elements = sum(elements, [])
+ if not elements:
+ elements = DEFAULT_ELEMENTS
+ if observatory_id not in VALID_OBSERVATORIES:
+ raise WebServiceException(
+ f"""Bad observatory id "{observatory_id}". Valid values are: {', '.join(VALID_OBSERVATORIES)}."""
+ )
if not start_time:
now = datetime.now()
- start_time = UTCDateTime(
- year=now.year,
- month=now.month,
- day=now.day,
- hour=0)
- try:
- end_time = UTCDateTime(query.get("endtime"))
- except:
- end_time = query.get("endtime")
+ start_time = UTCDateTime(year=now.year, month=now.month, day=now.day, hour=0)
+ else:
+ try:
+ start_time = UTCDateTime(start_time)
+ except Exception:
+ raise WebServiceException(
+ 'Bad starttime value "%s".'
+ " Valid values are ISO-8601 timestamps." % start_time
+ )
if not end_time:
end_time = start_time + (24 * 60 * 60 - 1)
end_time = UTCDateTime(end_time)
-
+ else:
+ try:
+ end_time = UTCDateTime(end_time)
+ except Exception:
+ raise WebServiceException(
+ 'Bad end_time value "%s".'
+ " Valid values are ISO-8601 timestamps." % end_time
+ )
# Create WebServiceQuery object and set properties
params = WebServiceQuery()
params.observatory_id = observatory_id
params.starttime = start_time
params.endtime = end_time
params.elements = elements
- params.sampling_period = get_interval_from_delta(sampling_period)
+ params.sampling_period = get_interval_from_delta(float(sampling_period))
params.data_type = data_type
params.output_format = output_format
return params
@@ -324,33 +352,21 @@ def validate_query(query):
WebServiceException
if any parameters are not supported.
"""
- if type(query.endtime) == str:
- raise WebServiceException(
- 'Bad end_time value "%s".'
- ' Valid values are ISO-8601 timestamps.' % query.endtime)
- if type(query.starttime) == str:
- raise WebServiceException(
- 'Bad end_time value "%s".'
- ' Valid values are ISO-8601 timestamps.' % query.starttime)
if len(query.elements) > 4 and query.output_format == "iaga2002":
raise WebServiceException(
"No more than four elements allowed for iaga2002 format."
)
- if query.observatory_id not in VALID_OBSERVATORIES:
- raise WebServiceException(
- f"""Bad observatory id "{query.observatory_id}". Valid values are: {', '.join(VALID_OBSERVATORIES)}."""
- )
if query.starttime > query.endtime:
raise WebServiceException("Starttime must be before endtime.")
if query.data_type not in VALID_DATA_TYPES:
raise WebServiceException(
- f"""Bad data type value "{query.data_type}". Valid values are: {', '.join(VALID_DATA_TYPES)}."""
- )
+ f"""Bad data type value "{query.data_type}". Valid values are: {', '.join(VALID_DATA_TYPES)}."""
+ )
if query.sampling_period not in VALID_INTERVALS:
raise WebServiceException(
f"""Bad sampling_period value {query.sampling_period}. Valid values are: {', '.join(VALID_SAMPLING_PERIODS)}."""
- )
+ )
if query.output_format not in VALID_OUTPUT_FORMATS:
raise WebServiceException(
- f"""Bad format value "{query.output_format}". Valid values are: {', '.join(VALID_OUTPUT_FORMATS)}."""
- )
\ No newline at end of file
+ f"""Bad format value "{query.output_format}". Valid values are: {', '.join(VALID_OUTPUT_FORMATS)}."""
+ )
diff --git a/geomagio/webservice/static/usage.css b/geomagio/webservice/static/usage.css
index 0bb4b75d..05ab6950 100644
--- a/geomagio/webservice/static/usage.css
+++ b/geomagio/webservice/static/usage.css
@@ -1,7 +1,3 @@
-p {
- font-weight: bold;
-}
-
.space-between {
margin: 1em 0 0;
}
diff --git a/geomagio/webservice/templates/data/usage.html b/geomagio/webservice/templates/data/usage.html
index 476a041e..fc189df5 100644
--- a/geomagio/webservice/templates/data/usage.html
+++ b/geomagio/webservice/templates/data/usage.html
@@ -3,55 +3,83 @@
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='usage.css') }}"/>
{% endblock %}
{% block content %}
- <div class="views-field views-field-field-intro"> <div class="field-content lead pane-border"><p><strong>Example Requests</strong><br /></p><dl><dt>BOU observatory data for current UTC day in IAGA2002 format</dt>
-<dd><a href="http://geomag.usgs.gov/ws/edge/?id=BOU">http://geomag.usgs.gov/ws/edge/?id=BOU</a></dd>
-<dt>BOU observatory data for current UTC day in JSON format</dt>
-<dd><a href="http://geomag.usgs.gov/ws/edge/?id=BOU&format=json">http://geomag.usgs.gov/ws/edge/?id=BOU&format=json</a></dd>
-<dt>BOU electric field data for current UTC day in IAGA2002 format</dt>
-<dd><a href="http://geomag.usgs.gov/ws/edge/?id=BOU&elements=E-N,E-E">http://geomag.usgs.gov/ws/edge/?id=BOU&elements=E-N,E-E</a></dd>
-</dl><p><a href="/natural-hazards/geomagnetism/science/more-examples">See more examples</a></p>
-</div> </div>
- <div class="views-field views-field-field-science-tab-intro"> <div class="field-content"></div> </div>
- <div class="views-field views-field-field-science-object-body"> <div class="field-content"><div class="tex2jax"><p><strong>Request Limits</strong></p>
-<p>To ensure availability for users, the web service restricts the amount of data that can be retrieved in one request. The amount of data requested is computed as follows, where an interval is the number of seconds between start time and end time:</p>
-<p>samples = count(elements) * interval / sampling_period</p>
-<p><strong>Limits by the output format</strong></p>
-<p><strong>json</strong></p>
-<ul><li>172800 samples = 4 elements * 12 hours * 3600 samples/hour.</li>
-</ul><p><strong>iaga2002</strong></p>
-<ul><li>345600 samples = 4 elements * 24 hours * 3600 samples/hour.</li>
-<li>NOTE: while the json format supports fewer total samples per request, users may request fewer elements to retrieve longer intervals.</li>
-</ul><p><strong>Parameters</strong></p>
-<p><strong>id</strong></p>
-<ul><li>Observatory code. Required.</li>
-<li>Valid values: BDT, BOU, TST, BRW, BRT, BSL, CMO, CMT, DED, DHT, FRD, FRN, GUA, HON, NEW, SHU, SIT, SJG, TUC, USGS, BLC, BRD, CBB, EUA, FCC, IQA, MEA, OTT, RES, SNK, STJ, VIC, YKC, HAD, HER, KAK</li>
-</ul><p><strong>starttime</strong></p>
-<ul><li>Time of first requested data.</li>
-<li>Default: start of current UTC day</li>
-<li>Format: ISO8601 (YYYY-MM-DDTHH:MM:SSZ)</li>
-<li>Example: 2018-08-06T22:10:14Z</li>
-</ul><p><strong>endtime</strong></p>
-<ul><li>Time of last requested data.</li>
-<li>Default: starttime + 24 hours</li>
-<li>Format: ISO8601 (YYYY-MM-DDTHH:MM:SSZ)</li>
-<li>Example: 2018-08-06T22:10:14Z</li>
-</ul><p><strong>elements</strong></p>
-<ul><li>Comma separated list of requested elements.</li>
-<li>Default: X,Y,Z,F</li>
-<li>Valid values: D, DIST, DST, E, E-E, E-N, F, G, H, SQ, SV, UK1, UK2, UK3, UK4, X, Y, Z</li>
-</ul><p><strong>sampling_period</strong></p>
-<ul><li>Interval in seconds between values.</li>
-<li>Default: 60</li>
-<li>Valid values: 1, 60, 3600</li>
-</ul><p><strong>type</strong></p>
-<ul><li>Type of data.</li>
-<li>Default: variation Valid values: variation, adjusted, quasi-definitive,definitive</li>
-<li>NOTE: the USGS web service also supports specific EDGE location codes. For example: R0 is "internet variation",R1 is "satellite variation".</li>
-</ul><p><strong>format</strong></p>
-<ul><li>Output format.</li>
-<li>Default: iaga2002</li>
-<li>Valid values: iaga2002, json.</li>
-</ul></div></div> </div>
+<div>
+ <div>
+ <p><strong>Example Requests</strong><br /></p>
+ <dl>
+ <dt>BOU observatory data for current UTC day in IAGA2002 format</dt>
+ <dd><a href="http://geomag.usgs.gov/ws/edge/?id=BOU">http://geomag.usgs.gov/ws/edge/?id=BOU</a>
+ </dd>
+ <dt>BOU observatory data for current UTC day in JSON format
+ </dt>
+ <dd><a href="http://geomag.usgs.gov/ws/edge/?id=BOU&format=json">http://geomag.usgs.gov/ws/edge/?id=BOU&format=json</a>
+ </dd>
+ <dt>BOU electric field data for current UTC day in IAGA2002 format
+ </dt>
+ <dd><a href="http://geomag.usgs.gov/ws/edge/?id=BOU&elements=E-N,E-E">http://geomag.usgs.gov/ws/edge/?id=BOU&elements=E-N,E-E</a>
+ </dd>
+ </dl>
+ <p><a href="/natural-hazards/geomagnetism/science/more-examples">See more examples</a>
+ </p>
+</div>
+<div>
+ <p><strong>Request Limits</strong></p>
+ <p>To ensure availability for users, the web service restricts the amount of data that can be retrieved in one request. The amount of data requested is computed as follows, where an interval is the number of seconds between start time and end time:</p>
+ <p>samples = count(elements) * interval / sampling_period</p>
+ <p><strong>Limits by the output format</strong></p>
+ <p><strong>json</strong></p>
+ <ul>
+ <li>172800 samples = 4 elements * 12 hours * 3600 samples/hour.</li>
+ </ul>
+ <p><strong>iaga2002</strong></p>
+ <ul>
+ <li>345600 samples = 4 elements * 24 hours * 3600 samples/hour.</li>
+ <li>NOTE: while the json format supports fewer total samples per request, users may request fewer elements to retrieve longer intervals.</li>
+ </ul>
+ <p><strong>Parameters</strong></p>
+ <p><strong>id</strong></p>
+ <ul>
+ <li>Observatory code. Required.</li>
+ <li>Valid values: BDT, BOU, TST, BRW, BRT, BSL, CMO, CMT, DED, DHT, FRD, FRN, GUA, HON, NEW, SHU, SIT, SJG, TUC, USGS, BLC, BRD, CBB, EUA, FCC, IQA, MEA, OTT, RES, SNK, STJ, VIC, YKC, HAD, HER, KAK</li>
+ </ul>
+ <p><strong>starttime</strong></p>
+ <ul>
+ <li>Time of first requested data.</li>
+ <li>Default: start of current UTC day</li>
+ <li>Format: ISO8601 (YYYY-MM-DDTHH:MM:SSZ)</li>
+ <li>Example: 2018-08-06T22:10:14Z</li>
+ </ul>
+ <p><strong>endtime</strong></p>
+ <ul>
+ <li>Time of last requested data.</li>
+ <li>Default: starttime + 24 hours</li>
+ <li>Format: ISO8601 (YYYY-MM-DDTHH:MM:SSZ)</li>
+ <li>Example: 2018-08-06T22:10:14Z</li>
+ </ul><p><strong>elements</strong></p>
+ <ul>
+ <li>Comma separated list of requested elements.</li>
+ <li>Default: X,Y,Z,F</li>
+ <li>Valid values: D, DIST, DST, E, E-E, E-N, F, G, H, SQ, SV, UK1, UK2, UK3, UK4, X, Y, Z</li>
+ </ul>
+ <p><strong>sampling_period</strong></p>
+ <ul>
+ <li>Interval in seconds between values.</li>
+ <li>Default: 60</li>
+ <li>Valid values: 1, 60, 3600</li>
+ </ul>
+ <p><strong>type</strong></p>
+ <ul>
+ <li>Type of data.</li>
+ <li>Default: variation Valid values: variation, adjusted, quasi-definitive,definitive</li>
+ <li>NOTE: the USGS web service also supports specific EDGE location codes. For example: R0 is "internet variation",R1 is "satellite variation".</li>
+ </ul>
+ <p><strong>format</strong></p>
+ <ul>
+ <li>Output format.</li>
+ <li>Default: iaga2002</li>
+ <li>Valid values: iaga2002, json.</li>
+ </ul>
+</div></div>
</body>
</html>
{% endblock %}
\ No newline at end of file
--
GitLab