diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1455086cecb275c5133ed093bd818b84b2e8f2f..c2d2340afc091d47d15ecff7f0d4d0eb329989b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,6 @@ variables: JACOCO_HTML_DIR: build/reports/jacoco/test/html JUNIT_FILES: build/test-results/test/TEST-*.xml - GIT_NSHMP_USERNAME: ${GITLAB_USER_EMAIL} - GIT_NSHMP_PASSWORD: ${GITLAB_API_TOKEN} include: - project: 'ghsc/hazdev/pipeline-build-template' @@ -11,7 +9,7 @@ include: stages: - build - - deploy + - publish #### # Templates @@ -25,62 +23,19 @@ stages: .docker-build: extends: - - .build + - .dind + - .tags::build + script: + - apk add bash + - scripts/gitlab-ci/docker-build.sh + stage: build variables: DOCKER_BUILD_ARGS: | BUILD_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 FROM_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 ci_job_token=${CI_JOB_TOKEN} - git_username=${GITLAB_USER_EMAIL} - git_password=${GITLAB_API_TOKEN} - -.templates: - # TODO: Remove tag "beta" when ready - adjust-ref: &adjust-ref | - if [[ \ - ${CI_COMMIT_REF_SLUG} == "master" || \ - ${CI_COMMIT_REF_SLUG} == "production" || \ - -n "${CI_COMMIT_TAG}" \ - ]]; then - CI_COMMIT_REF_SLUG="beta"; - else - CI_COMMIT_REF_SLUG="beta--${CI_COMMIT_REF_SLUG}" - fi - -.deploy: - cache: {} - image: ${CODE_REGISTRY}/ghsc/hazdev/cloud-formation/hazdev-build-runner:latest - dependencies: - script: - - *adjust-ref - - git clone ${GENERIC_SWARM_DEPLOY_REPO} generic-deploy - - export REGISTRY=${CODE_REGISTRY_IMAGE} - - cp -v - generic-deploy/default.config.sh - generic-deploy/default.funcs.sh - generic-deploy/deploy.sh - scripts/. - - scripts/deploy.sh - stage: deploy - tags: - - deploy - - swarm - variables: - APP_NAME: nshmp-haz - STACK_NAME: nshmp-haz - -.onprem-staging: - only: - - master@ghsc/nshmp/nshmp-haz-v2 - - tags@ghsc/nshmp/nshmp-haz-v2 - -.staging01: - tags: - - staging01 - -.staging02: - tags: - - staging02 + DOCKERFILE: Dockerfile + IMAGE_NAME: ${CI_PROJECT_NAME} #### # Stage: build @@ -120,22 +75,15 @@ Unit Tests: Build Haz Image: extends: - .docker-build - before_script: - - apk add git - - *adjust-ref variables: - CI_PROJECT_NAME: nshmp-haz + IMAGE_NAME: nshmp-haz Build WS Image: extends: - .docker-build - before_script: - - mv Dockerfile haz.Dockerfile - - mv ws.Dockerfile Dockerfile - - *adjust-ref - - apk add git variables: - CI_PROJECT_NAME: nshmp-haz-ws + IMAGE_NAME: nshmp-haz-ws + DOCKERFILE: ws.Dockerfile #### # Stage: Publish @@ -144,14 +92,14 @@ Build WS Image: Maven: extends: .gradle - stage: deploy + stage: publish only: - tags@ghsc/nshmp/nshmp-ws script: - ./gradlew publish -P version=${CI_COMMIT_TAG} Trigger nshmp-deploy: - stage: deploy + stage: publish only: - master@ghsc/nshmp/nshmp-haz-v2 - tags@ghsc/nshmp/nshmp-haz-v2 @@ -162,19 +110,3 @@ Trigger nshmp-deploy: -F token=${NSHMP_DEPLOY_TRIGGER_TOKEN} \ -F ref=master \ https://code.chs.usgs.gov/api/v4/projects/5047/trigger/pipeline - -#### -# Stage: Deploy -#### - -# Staging 01: -# extends: -# - .deploy -# - .staging -# - .staging01 - -# Staging 02: -# extends: -# - .deploy -# - .staging -# - .staging02 diff --git a/scripts/docker-entrypoint.ws.sh b/scripts/docker-entrypoint.ws.sh deleted file mode 100644 index 480843de112113fb90b25975a02d2a40c74adf66..0000000000000000000000000000000000000000 --- a/scripts/docker-entrypoint.ws.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC1090 - -source "$(dirname "${0}")/docker-config.inc.sh"; -exit_status=${?}; -[ "${exit_status}" -eq 0 ] || exit "${exit_status}"; - -# Download models to use -model_path=$(get_models "${MODEL}" "${NSHM_VERSION}"); -exit_status=${?}; -check_exit_status ${exit_status}; - -# Run web services -java -jar "${PROJECT}.jar" \ - "-Dmicronaut.server.context-path=${CONTEXT_PATH}" \ - --models="${model_path}"; -exit_status=${?}; -check_exit_status ${exit_status}; - -exit ${exit_status}; diff --git a/scripts/docker-functions.inc.sh b/scripts/docker-functions.inc.sh index 22092cb65089b306f092cd4fda9f1d4eaffd6120..ae711e683f6ce28e5d6c1ec21d321bda2e3ec436 100644 --- a/scripts/docker-functions.inc.sh +++ b/scripts/docker-functions.inc.sh @@ -86,7 +86,7 @@ error_exit() { } #### -# Returns the model path for deagg-epsilon and hazard-2018. +# Returns the model path. # # @param $1 nshm {String} # The NSHM to download. @@ -229,58 +229,6 @@ get_model_path() { return ${exit_status}; } -#### -# Get NSHMs for web services. -# -# @param $1 nshm {String} -# The NSHM to download. -# @param $1 nshm_version {String} -# The version to download from GitHub. -# -# @return String -# The path to the model directory -# @status Integer -# The result for get_model -#### -get_models() { - local nshm=${1}; - local nshm_version=${2}; - local model_base_path="models"; - local nshm_name; - local exit_status; - - if [ ! -d "${model_base_path}" ]; then - mkdir ${model_base_path}; - fi - - cd ${model_base_path} || error_exit "Could not change directory [${model_base_path}]" 1; - nshm_name=$(get_model "${nshm}" "${nshm_version}"); - exit_status=${?}; - - if [ ${exit_status} -eq 0 ]; then - local model; - local year; - model="$(echo "${nshm}" | cut -d _ -f1 | awk \{'print tolower($0)'\})"; - year="$(echo "${nshm}" | cut -d _ -f2 | awk \{'print tolower($0)'\})"; - - if [ "${model}" == 'conus' ]; then - [[ -d wus ]] || mkdir wus; - [[ -d ceus ]]|| mkdir ceus; - mv "${nshm_name}/${CEUS}" "ceus/${year}"; - mv "${nshm_name}/${WUS}" "wus/${year}"; - rm -r "${nshm_name}"; - else - mkdir "${model}"; - mv "${nshm_name}" "${model}/${year}"; - fi - fi - - cd ../; - - echo ${model_base_path}; - return ${exit_status}; -} - #### # Returns to nshmp-haz Java class to call. # diff --git a/scripts/gitlab-ci/docker-build.sh b/scripts/gitlab-ci/docker-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..819b999ab5c0a586712baf8d03808fc24139ece8 --- /dev/null +++ b/scripts/gitlab-ci/docker-build.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Script to build Docker image and push to internal registry. +## + +BUILD_ARGS=""; + +for arg in ${DOCKER_BUILD_ARGS}; do + BUILD_ARGS="${BUILD_ARGS} --build-arg ${arg}"; +done + +# Build Docker image +# shellcheck disable=SC2086 +docker build \ + ${BUILD_ARGS} \ + --pull \ + --tag "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}" \ + --file "${DOCKERFILE}" \ + .; + +# Push image to internal registry +docker push "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}"; + +# Push latest tag +if [[ + ${CI_COMMIT_REF_SLUG} == "master" || + ${CI_COMMIT_REF_SLUG} == "production" || + -n "${CI_COMMIT_TAG}" +]]; then + docker tag \ + "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_REF_SLUG}" \ + "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:latest"; + docker push "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:latest"; +fi + +# Push specific tag +if [[ + -n "${CI_COMMIT_TAG}" +]]; then + docker tag \ + "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:latest" \ + "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_TAG}"; + docker push "${CODE_REGISTRY_IMAGE}/${IMAGE_NAME}:${CI_COMMIT_TAG}"; +fi diff --git a/settings.gradle b/settings.gradle index fddc289feaadbb730967c279b9d21f322f5f1337..5c938c14a9495694a68565a86d7cce0948dce711 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,11 +4,22 @@ plugins { rootProject.name = "nshmp-haz-v2" +/** + * Download default model, nshm-conus-2018. + * + * To download the model set environmnet variables: + * - GIT_NSHMP_USERNAME="The GitLab user name" + * - GIT_NSHMP_PASSWORD="The GitLab API token" + */ git { defaultAuthGroup "nshmp" + def user = System.getenv("GIT_NSHMP_USERNAME") + def pass = System.getenv("GIT_NSHMP_PASSWORD") - fetch("https://code.usgs.gov/ghsc/nshmp/nshm-conus-2018.git", { - name "nshmp-haz-dep--nshm-conus-2018" - tag "0.2.3" - }) + if (user && pass) { + fetch("https://code.usgs.gov/ghsc/nshmp/nshm-conus-2018.git", { + name "nshmp-haz-dep--nshm-conus-2018" + tag "0.2.3" + }) + } } diff --git a/src/main/java/gov/usgs/earthquake/nshmp/www/DeaggEpsilonController.java b/src/main/java/gov/usgs/earthquake/nshmp/www/DeaggEpsilonController.java index ab2976d4e2b23d2f3d1e8a54170458c3048a645e..478f8087ad759a3d4acd890ab3f437cdd0ecd4b1 100644 --- a/src/main/java/gov/usgs/earthquake/nshmp/www/DeaggEpsilonController.java +++ b/src/main/java/gov/usgs/earthquake/nshmp/www/DeaggEpsilonController.java @@ -11,7 +11,6 @@ import gov.usgs.earthquake.nshmp.www.services.DeaggEpsilonService; import gov.usgs.earthquake.nshmp.www.services.DeaggEpsilonService.Query; import gov.usgs.earthquake.nshmp.www.services.HazardService; -import io.micronaut.context.event.StartupEvent; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; import io.micronaut.http.MediaType; @@ -19,7 +18,6 @@ import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.annotation.QueryValue; -import io.micronaut.runtime.event.annotation.EventListener; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -32,11 +30,6 @@ public class DeaggEpsilonController { @Inject private NshmpMicronautServlet servlet; - @EventListener - public void init(StartupEvent event) { - DeaggEpsilonService.init(); - } - @Get(uri = "/usage", produces = MediaType.APPLICATION_JSON) public HttpResponse<String> doGetUsage(HttpRequest<?> request) { var urlHelper = servlet.urlHelper(request); diff --git a/src/main/java/gov/usgs/earthquake/nshmp/www/services/DeaggEpsilonService.java b/src/main/java/gov/usgs/earthquake/nshmp/www/services/DeaggEpsilonService.java index 2f744fe8631b0191a04d5db8ca49e9232e589208..02cb1f53bbed3603427b5e0c6576f204b7abfca4 100644 --- a/src/main/java/gov/usgs/earthquake/nshmp/www/services/DeaggEpsilonService.java +++ b/src/main/java/gov/usgs/earthquake/nshmp/www/services/DeaggEpsilonService.java @@ -1,19 +1,13 @@ package gov.usgs.earthquake.nshmp.www.services; -import static com.google.common.base.Preconditions.checkNotNull; - -import java.io.IOException; -import java.io.InputStream; import java.net.URL; import java.util.EnumMap; import java.util.List; -import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.function.Function; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; -import com.google.common.io.Resources; import gov.usgs.earthquake.nshmp.calc.CalcConfig; import gov.usgs.earthquake.nshmp.calc.Deaggregation; @@ -30,6 +24,7 @@ import gov.usgs.earthquake.nshmp.www.meta.Metadata; import gov.usgs.earthquake.nshmp.www.services.ServicesUtil.Key; import gov.usgs.earthquake.nshmp.www.services.SourceServices.SourceModel; +import io.micronaut.context.annotation.Value; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; @@ -43,26 +38,9 @@ public final class DeaggEpsilonService { /* Developer notes: See HazardService. */ private static final String NAME = "Epsilon Deaggregation"; - private static URL basinUrl; - public static void init() { - try (InputStream config = Resources.getResource("config.properties").openStream()) { - checkNotNull(config, "Missing config.properties"); - - Properties props = new Properties(); - props.load(config); - if (props.containsKey("basin_host")) { - /* - * TODO Site builder tests if service is working, which may be - * inefficient for single call services. - */ - var url = new URL(props.getProperty("basin_host") + "/nshmp/ws/data/basin"); - basinUrl = url; - } - } catch (IOException | NullPointerException e) { - throw new RuntimeException(e); - } - } + @Value("${nshmp-haz.basin-service-url}") + private static URL basinUrl; /** * Handler for {@link DeaggEpsilonController#doGetDeaggEpsilon}. Returns the diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 864c3d0da1b68393edb0a762c7af6db4e4514e73..2fef824ccdac59da5b0e11c2df20916a2f00deef 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -15,6 +15,12 @@ micronaut: mapping: /** nshmp-haz: + ## + # The basin service url + # To specify the url to use: + # java -jar build/libs/nshmp-haz-v2.jar --basin-service-url= + basin-service-url: ${basin-service-url:https://staging-earthquake.usgs.gov/nshmp/ws/data/basin} + ## # The path to the models. # To specify the model to use: diff --git a/ws.Dockerfile b/ws.Dockerfile index 4d9afdad56cca2cd0b290849c055c7252f41dda4..a901e47f44ec4afa75cedea57e7245475d0c0bf1 100644 --- a/ws.Dockerfile +++ b/ws.Dockerfile @@ -6,6 +6,11 @@ # -f ws.Dockerfile # --build-arg gitlab_token=<git-api-token> # -t nshmp-haz-ws . +# +# Run locally: +# docker run -p 8080:8080 +# -v "path/to/models:/models" +# nshmp-haz-ws #### ARG BUILD_IMAGE=usgs/java:11 @@ -17,20 +22,16 @@ ARG libs_dir=${builder_workdir}/build/libs ARG jar_file=${libs_dir}/${project}.jar #### -# Builder image: Build war file. +# Builder image: Build jar file. #### FROM ${BUILD_IMAGE} as builder ARG builder_workdir -ARG libs_dir -ARG jar_file -ARG git_username -ARG git_password + +# TODO +# Remove once nshmp-lib is public ARG gitlab_token=null ARG ci_job_token=null - -ENV GIT_NSHMP_USERNAME ${git_username} -ENV GIT_NSHMP_PASSWORD ${git_password} ENV GITLAB_TOKEN ${gitlab_token} ENV CI_JOB_TOKEN ${ci_job_token} @@ -41,7 +42,7 @@ COPY . . RUN ./gradlew assemble #### -# Application image: Run war file. +# Application image: Run jar file. #### FROM ${FROM_IMAGE} @@ -51,14 +52,19 @@ ARG libs_dir ARG builder_workdir ARG project -ENV DEBUG false ENV PROJECT ${project} ENV CONTEXT_PATH "/" +ENV BASIN_SERVICE_URL "https://staging-earthquake.usgs.gov/nshmp/ws/data/basin" WORKDIR /app COPY --from=builder ${libs_dir}/* ./ -COPY scripts scripts + +VOLUME [ "/models" ] EXPOSE 8080 -ENTRYPOINT [ "bash", "scripts/docker-entrypoint.ws.sh" ] + +ENTRYPOINT java -jar "${PROJECT}.jar" \ + "-Dmicronaut.server.context-path=${CONTEXT_PATH}" \ + --basin-service-url="${BASIN_SERVICE_URL}" \ + --models="/models";