cleanup

.gitlab/Dockerfile

@@ -34,9 +34,9 @@ RUN yum install -y epel-release dnf-plugins-core \
 WORKDIR /python
 
 RUN curl -sS  "${PYTHON_URL}" | tar xvz --strip-components=1 \
-    && ./configure --enable-optimizations \
-    && make \
-    && make install \
+    && ./configure --enable-optimizations --silent \
+    && make --silent\
+    && make install --silent \
     && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3
 
 WORKDIR /

Dockerfile

@@ -59,6 +59,7 @@ ENV PROJECT ${project}
 ENV NETCDF_FILE ${netcdf_file}
 
 ENV PYTHON_URL="https://www.python.org/ftp/python/3.9.2/Python-3.9.2.tgz"
+ENV PIP_CERT=${SSL_CERT_FILE}
 
 RUN yum install -y epel-release dnf-plugins-core \
     && yum config-manager --set-enabled powertools \
@@ -74,10 +75,10 @@ RUN yum install -y epel-release dnf-plugins-core \
 
 WORKDIR /python
 
-RUN curl -sS  "${PYTHON_URL}" | tar xvz --strip-components=1 \
-    && ./configure --enable-optimizations \
-    && make \
-    && make install \
+RUN curl -sS  "${PYTHON_URL}" | tar xz --strip-components=1 \
+    && ./configure --enable-optimizations --silent \
+    # && make --silent \
+    && make install --silent\
     && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3
 
 WORKDIR ${workdir}
@@ -87,7 +88,7 @@ COPY src/main/python src/main/python
 COPY src/main/resources src/main/resources
 COPY poetry.lock .
 COPY pyproject.toml .
-COPY bash-entrypoint.sh .
+COPY docker-entrypoint.sh .
 
 EXPOSE 8080
 ENTRYPOINT [ "bash", "docker-entrypoint.sh" ]

docker-entrypoint.sh

@@ -7,7 +7,10 @@ PROJECT="${PROJECT:-nshmp-netcdf}";
 
 # TODO: Remove option to install database in container when deployed to AWS
 if [ "${INSTALL_DATABASE}" == "true" ]; then
-  "${HOME}"/.poetry/bin/poetry install;
+  export REQUESTS_CA_BUNDLE="${SSL_CERT_FILE}";
+  export PATH="$HOME/.poetry/bin:$PATH";
+
+  poetry install;
 
   case "${NSHM}" in
     "CONUS_2018A")

src/main/python/gov/usgs/earthquake/nshmp/netcdf/__main__.py

@@ -10,7 +10,7 @@ Supported NSHMs for download and conversion:
 from rich.traceback import install
 
 from .application_inputs import ApplicationInputs
-from .converters.convert_2018a import Convert2018A
+from .converters.convert import Convert
 from .converters.preprocess import Preprocess
 from .database.database import Database
 from .parsers.args_parser import parser
@@ -38,6 +38,6 @@ def run():
             if not inputs.download_only:
                 for database_info in database.database_info:
                     preprocess = Preprocess(database_info=database_info)
-                    Convert2018A(inputs=inputs, metadata=preprocess.netcdf_metadata)
+                    Convert(inputs=inputs, metadata=preprocess.netcdf_metadata)
     except Exception as error:
         raise error

src/main/python/gov/usgs/earthquake/nshmp/netcdf/converters/convert.py

@@ -7,6 +7,11 @@ from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import Union
+from rich.progress import (
+    BarColumn,
+    Progress,
+    TimeElapsedColumn,
+)
 
 import netCDF4 as netcdf
 import numpy as np
@@ -46,7 +51,6 @@ class Convert:
         if (nshm != Nshm.NSHM_2018A):
             raise ValueError(f"NSHM [{self.metadata.database_info.nshm.value}] not supported")
 
-        # hazard_parameters = NetcdfParameters().HazardParameters()
         console.print(f"\n[blue]Converting {nshm.label}")
 
         self._imt_indices: dict[Imt, int] = self._set_imt_indices()
@@ -76,18 +80,12 @@ class Convert:
         self._site_class_mask_array = np.zeros(
             [self._dimensions.lat.size, self._dimensions.lon.size], int
         )
-        # self._data_array = np.full(
-        #     [
-        #         self._dimensions.site_class.size,
-        #         self._dimensions.imt.size,
-        #         self._dimensions.lat.size,
-        #         self._dimensions.lon.size,
-        #         self._dimensions.iml.size,
-        #     ],
-        #     hazard_parameters.fill_value,
-        #     float,
-        # )
-
+        self._progress = Progress(
+            "[progress.description]{task.description}",
+            BarColumn(),
+            "[progress.percentage]{task.percentage:>3.0f}%",
+            TimeElapsedColumn()
+        )
         self._write_netcdf_file()
         self._root_group.close()
         self._clean_ascii()
@@ -171,25 +169,24 @@ class Convert:
 
     def _get_hazard_data(
         self,
-        hazard_netcdf_var: netcdf.Variable,
-        netcdf_info: list[NetcdfInfo],
+        hazard_netcdf_var: netcdf.Variable
     ):
         futures: list[Future] = []
-
-        with ThreadPoolExecutor() as executor:
-            for index, _info in enumerate(netcdf_info):
-
-                info: NetcdfInfo = _info
-                futures.append(
-                    executor.submit(
-                        self._read_curves_file,
-                        hazard_netcdf_var=hazard_netcdf_var,
-                        netcdf_info=info,
+        status_msg = f"[bold green]Converting {self.metadata.database_info.nshm.label} files"
+
+        with self._progress:
+            with ThreadPoolExecutor() as executor:
+                for _info in self.metadata.netcdf_info:
+                    info: NetcdfInfo = _info
+                    futures.append(
+                        executor.submit(
+                            self._read_curves_file,
+                            hazard_netcdf_var=hazard_netcdf_var,
+                            netcdf_info=info,
+                        )
                     )
-                )
-
-        for future in futures:
-            future.result(timeout=120)
+                for future in self._progress.track(futures, description=status_msg):
+                    future.result(timeout=120)
 
     def _get_imt_index(self, imt: Imt):
         return self._imt_indices.get(imt)
@@ -366,15 +363,6 @@ class Convert:
             enum_dict=site_class_dict,
         )
 
-    def _write_hazard_data(self, hazard_netcdf_var: netcdf.Variable):
-        status_msg = f"[bold green]Converting {self.metadata.database_info.nshm.label} files ..."
-        with console.status(status_msg, spinner="pong") as status:
-            self._get_hazard_data(
-                netcdf_info=self.metadata.netcdf_info,
-                hazard_netcdf_var=hazard_netcdf_var,
-            )
-        status.stop()
-
     def _write_netcdf_file(self):
         self._root_group.setncatts(_ROOT_ATTRIBUTES)
         self._dataset_group.description = self.metadata.database_info.description
@@ -387,6 +375,5 @@ class Convert:
             )
 
         grid_mask_netcdf_var, hazard_netcdf_var = self._create_netcdf_variables()
-        self._write_hazard_data(hazard_netcdf_var=hazard_netcdf_var)
-        # hazard_netcdf_var[:, :, :, :, :] = self._data_array
+        self._get_hazard_data(hazard_netcdf_var=hazard_netcdf_var)
         grid_mask_netcdf_var[:, :] = self._site_class_mask_array

src/main/python/gov/usgs/earthquake/nshmp/netcdf/converters/preprocess.py

@@ -2,7 +2,11 @@ from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
 
 import numpy as np
-from rich.progress import track
+from rich.progress import (
+    BarColumn,
+    Progress,
+    TimeElapsedColumn,
+)
 
 from ..database.database_info import DatabaseInfo, DataInfo, NetcdfInfo, NetcdfMetadata
 from ..geo.location import Locations
@@ -35,19 +39,27 @@ class Preprocess:
         futures: list[Future[DataInfo]] = []
         data_info_list: list[DataInfo] = []
 
-        with ThreadPoolExecutor() as executor:
-            for _curve_file in database_info.curve_files:
-                curve_file: Path = _curve_file
-                future = executor.submit(
-                    DataPathParser.parse_path,
-                    database_info=database_info,
-                    curve_file=curve_file,
-                )
-                futures.append(future)
-
-            for _future in track(futures, description=status_msg):
-                future: Future[DataInfo] = _future
-                data_info_list.append(future.result(timeout=120))
+        progress = Progress(
+            "[progress.description]{task.description}",
+            BarColumn(),
+            "[progress.percentage]{task.percentage:>3.0f}%",
+            TimeElapsedColumn()
+        )
+
+        with progress:
+            with ThreadPoolExecutor() as executor:
+                for _curve_file in database_info.curve_files:
+                    curve_file: Path = _curve_file
+                    future = executor.submit(
+                        DataPathParser.parse_path,
+                        database_info=database_info,
+                        curve_file=curve_file,
+                    )
+                    futures.append(future)
+
+                for _future in progress.track(futures, description=status_msg):
+                    future: Future[DataInfo] = _future
+                    data_info_list.append(future.result(timeout=120))
 
         console.print("Preprocessing done", style="bold yellow")
         return self._condense_data_info(database_info=database_info, data_info_list=data_info_list)