Updates to use async calls for the NWIS and the sifta services.

wdfn-server/waterdata/services/nwissite.py

@@ -21,17 +21,18 @@ async def get(session, params):
         - site_data - list of dictionaries
     """
     endpoint = app.config["SITE_DATA_ENDPOINT"]
-    app.logger.debug(f'Requesting data from {endpoint}')
+    app.logger.debug(f'Requesting data from {endpoint} and {params}')
     default_params = {
         'format': 'rdb'
     }
     default_params.update(params)
-    async with session.get(endpoint, params=default_params) as response:
-        if response.status_code == 200:
-            async with response.iter_lines(decode_unicode=True) as lines:
-                return 200, response.reason, list(parse_rdb(lines))
+    response = await session.get(endpoint, params=default_params)
+    if response.status == 200:
+        lines = await response.text()
+        app.logger.debug(f'Received data from {endpoint} and {params}')
+        return 200, response.reason, list(parse_rdb(iter(lines.splitlines())))
 
-        return response.status_code, response.reason, []
+    return response.status_code, response.reason, []
 
 
 async def get_site_data(session, site_no, agency_cd=''):
@@ -68,7 +69,7 @@ async def get_period_of_record(session, site_no, agency_cd=''):
     """
     params = {
         'sites': site_no,
-        'seriesCatalogOutput': True,
+        'seriesCatalogOutput': 'true',
         'siteStatus': 'all'
     }
     if agency_cd:

wdfn-server/waterdata/services/sifta.py

@@ -20,10 +20,10 @@ async def get_cooperators(session, site_no):
         # TODO: Add exception handling
         app.logger.debug(f'Retrieved data from {url}')  # pylint: disable=no-member
 
-        if response.status_code != 200:
+        if response.status != 200:
             return []
         try:
-            resp_json = response.json()
+            resp_json = await response.json()
         except ValueError:
             return []
         else:

wdfn-server/waterdata/services/timezone.py

@@ -25,6 +25,8 @@ class TimeZoneService:
         :return str
         """
         url = f'{self.endpoint}/points/{latitude},{longitude}'
+
+        app.logger.debug(f'Requesting timezone')
         try:
             response = self.session.get(url)
         except (request_exceptions.Timeout, request_exceptions.ConnectionError) as err:
@@ -33,5 +35,7 @@ class TimeZoneService:
         if response.status_code != 200:
             return None
 
+        app.logger.debug(f'Received timezone')
+
         json_data = response.json()
         return json_data['properties'].get('timeZone', None) if 'properties' in json_data else None

wdfn-server/waterdata/utils.py

@@ -2,9 +2,10 @@
 Utility functions
 
 """
-from functools import update_wrapper
-from urllib.parse import urlencode, urljoin
+import asyncio
 from email.message import EmailMessage
+from functools import update_wrapper, wraps
+from urllib.parse import urlencode, urljoin
 
 from flask import request
 
@@ -101,7 +102,7 @@ def parse_rdb(rdb_iter_lines):
     """
     Parse records in an RDB file into dictionaries.
 
-    :param iterator rdb_iter_lines: iterator containing lines from an RDB file
+    :param str rdb_iter_lines: iterator containing lines from an RDB file
     :rtype: Iterator
 
     """
@@ -124,23 +125,3 @@ def parse_rdb(rdb_iter_lines):
             continue
         record_values = record.split('\t')
         yield dict(zip(headers, record_values))
-
-
-def defined_when(condition, fallback):
-    """
-    Decorator that fallsback to a specified function if `condition` is False.
-    :param condition: bool Decorated function will be called if True, otherwise
-                           fallback will be called
-    :param fallback: function to be called if condition is False
-    :return: Decorated function
-    :rtype: function
-    """
-    def wrap(f):  # pylint: disable=invalid-name
-        if condition:
-            # pylint:disable=do-not-assign-a-lambda-expression-use-a-def, unnecessary-lambda
-            func = lambda *args, **kwargs: f(*args, **kwargs)  # flake8: noqa
-        else:
-            func = fallback
-        return update_wrapper(func, f)
-
-    return wrap

wdfn-server/waterdata/views.py

@@ -16,7 +16,7 @@ from markdown import markdown
 from . import app, __version__
 from .location_utils import build_linked_data, get_disambiguated_values, rollup_dataseries, \
     get_period_of_record_by_parm_cd, get_default_parameter_code
-from .utils import defined_when, set_cookie_for_banner_message, create_message
+from .utils import set_cookie_for_banner_message, create_message
 from .services.camera import get_monitoring_location_camera_details
 from .services.nwissite import get_county_sites, get_huc_sites, get_site_data, get_period_of_record
 from .services.ogc import MonitoringLocationNetworkService
@@ -233,7 +233,6 @@ def return_404():
 
 @app.route('/hydrological-unit/', defaults={'huc_cd': None}, methods=['GET'])
 @app.route('/hydrological-unit/<huc_cd>/', methods=['GET'])
-@defined_when(app.config['HYDROLOGIC_PAGES_ENABLED'], return_404)
 async def hydrological_unit(huc_cd, show_locations=False):
     """
     Hydrological unit view
@@ -242,6 +241,9 @@ async def hydrological_unit(huc_cd, show_locations=False):
     """
 
     # Get the data corresponding to this HUC
+    if not app.config['HYDROLOGIC_PAGES_ENABLED']:
+        return return_404()
+
     monitoring_locations = []
     if huc_cd:
         huc = app.config['HUC_LOOKUP']['hucs'].get(huc_cd, None)
@@ -271,12 +273,11 @@ async def hydrological_unit(huc_cd, show_locations=False):
 
 
 @app.route('/hydrological-unit/<huc_cd>/monitoring-locations/', methods=['GET'])
-@defined_when(app.config['HYDROLOGIC_PAGES_ENABLED'], return_404)
-def hydrological_unit_locations(huc_cd):
+async def hydrological_unit_locations(huc_cd):
     """
     Returns a HUC page with a list of monitoring locations included.
     """
-    return hydrological_unit(huc_cd, show_locations=True)
+    return await hydrological_unit(huc_cd, show_locations=True)
 
 
 @app.route('/networks/', defaults={'network_cd': ''}, methods=['GET'])
@@ -322,7 +323,6 @@ def networks(network_cd):
 @app.route('/states/', defaults={'state_cd': None, 'county_cd': None}, methods=['GET'])
 @app.route('/states/<state_cd>/', defaults={'county_cd': None}, methods=['GET'])
 @app.route('/states/<state_cd>/counties/<county_cd>/', methods=['GET'])
-@defined_when(app.config['STATE_COUNTY_PAGES_ENABLED'], return_404)
 async def states_counties(state_cd, county_cd, show_locations=False):
     """
     State unit view
@@ -331,6 +331,8 @@ async def states_counties(state_cd, county_cd, show_locations=False):
     :param county_cd: ID for this political unit - 'county'
     :param bool show_locations:
     """
+    if not app.config['STATE_COUNTY_PAGES_ENABLED']:
+        return return_404()
 
     monitoring_locations = []
     political_unit = {}
@@ -370,18 +372,18 @@ async def states_counties(state_cd, county_cd, show_locations=False):
 
 
 @app.route('/states/<state_cd>/counties/<county_cd>/monitoring-locations/', methods=['GET'])
-@defined_when(app.config['STATE_COUNTY_PAGES_ENABLED'], return_404)
-def county_station_locations(state_cd, county_cd):
+async def county_station_locations(state_cd, county_cd):
     """
     Returns a page listing monitoring locations within a county.
     """
-    return states_counties(state_cd, county_cd, show_locations=True)
+    return await states_counties(state_cd, county_cd, show_locations=True)
 
 
 @app.route('/components/time-series/<site_no>/', methods=['GET'])
-@defined_when(app.config['EMBED_IMAGE_FEATURE_ENABLED'], return_404)
 def time_series_component(site_no):
     """
     Returns an unadorned page with the time series component for a site.
     """
+    if not app.config['EMBED_IMAGE_FEATURE_ENABLED']:
+        return_404()
     return render_template('monitoring_location_embed.html', site_no=site_no)