diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ffd169b0bc4c7ae0b8e4808a78f11158bc000584..ecdec9a034d02574f8caf5167885963eb3d8e84e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,8 +4,6 @@ variables: GRADLE_USER_HOME: ${CI_PROJECT_DIR}/tmp/.gradle JACOCO_HTML_DIR: ${REPORTS_DIR}/jacoco/test/html JUNIT_FILES: build/test-results/test/TEST-*.xml - PIPELINE_IMAGE_NAME: ${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}--pipeline - POETRY_CACHE_DIR: ${CI_PROJECT_DIR}/tmp/.poetry REPORTS_DIR: build/reports # TODO: Remove hard coded url S3_BUCKET: https://nshmp-netcdf-lp-development-rbucket-1of3d1x45yfd9.s3-us-west-2.amazonaws.com @@ -23,7 +21,6 @@ include: file: "templates/library.yml" stages: - - image - init - build - database @@ -39,13 +36,6 @@ stages: stage: build image: ${PIPELINE_IMAGE_NAME} -.python: - before_script: - - source $HOME/.poetry/env - extends: - - .tags::development - image: ${PIPELINE_IMAGE_NAME} - .templates: adjust-ref: &adjust-ref | if [[ \ @@ -94,34 +84,6 @@ stages: tags: - staging02 -#### -# Stage: image -#### - -Image: - extends: - - .dind - rules: - - - changes: - - .gitlab/Dockerfile - when: on_success - - - allow_failure: true - when: manual - script: - - | - docker build \ - --build-arg FROM_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 \ - --file ".gitlab/Dockerfile" \ - --pull \ - --tag ${PIPELINE_IMAGE_NAME} \ - . - - docker push ${PIPELINE_IMAGE_NAME} - stage: image - tags: - - build - #### # Stage: init #### @@ -129,20 +91,64 @@ Image: Init: artifacts: paths: - - '${POETRY_CACHE_DIR}' - '${GRADLE_USER_HOME}' extends: - - .python + - .gradle script: - - poetry install - ./gradlew dependencies - stage: init + stage: #### # Stage: build #### -Java - Build Project: +.docker-build: + image: ${DEVOPS_REGISTRY}docker:19.03-git + script: + - BUILD_ARGS=''; + - | + for arg in ${DOCKER_BUILD_ARGS}; do + BUILD_ARGS="${BUILD_ARGS} --build-arg ${arg}"; + done + - | + docker build \ + ${BUILD_ARGS} \ + --pull \ + --tag "${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}:${IMAGE_TAG}" \ + --file "${CI_PROJECT_DIR}/${DOCKERFILE}" \ + "${CI_PROJECT_DIR}/."; + - docker push "${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}:${IMAGE_TAG}"; + - | + if [[ \ + ${CI_COMMIT_REF_SLUG} == "master" || \ + ${CI_COMMIT_REF_SLUG} == "production" || \ + -n "${CI_COMMIT_TAG}" \ + ]]; then + docker tag "${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}:${IMAGE_TAG}" \ + "${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}:latest"; + docker push "${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}:latest"; + docker tag "${CODE_REGISTRY_IMAGE}/${CI_PROJECT_NAME}:latest" \ + "usgs/${CI_PROJECT_NAME}:latest"; + docker push "usgs/${CI_PROJECT_NAME}:latest"; + fi + - | + printf " + -------- + Image Name - %s:%s + -------- + " "${CI_PROJECT_NAME}" "${IMAGE_TAG}"; + services: + - alias: docker + name: ${DEVOPS_REGISTRY}docker:19.03-dind + stage: build + tags: + - build + variables: + DOCKERFILE: Dockerfile + DOCKER_DRIVER: overlay2 + IMAGE_TAG: ${CI_COMMIT_REF_SLUG} + +Build Project: extends: - .gradle rules: @@ -160,35 +166,50 @@ Java - Build Project: - ./gradlew assemble stage: build -Java - Build Image 2018: +Build Image 2018: before_script: - apk add curl - mkdir DATA - curl -SL ${DATA_URL} -o DATA/data.nc + needs: [] + rules: + - if: $CI_PROJECT_PATH != 'ghsc/nshmp/nshmp-netcdf' + variables: + IMAGE_TAG: 'development-${CI_COMMIT_SHORT_SHA}' + - if: > + $CI_PROJECT_PATH == 'ghsc/nshmp/nshmp-netcdf' + && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + variables: + IMAGE_TAG: 'staging-${CI_COMMIT_SHORT_SHA}' + - if: > + $CI_PROJECT_PATH == 'ghsc/nshmp/nshmp-netcdf' + && ( $CI_COMMIT_BRANCH == 'production' || $CI_COMMIT_TAG != '' ) + variables: + IMAGE_TAG: 'production-${CI_COMMIT_SHORT_SHA}' + variables: + DOCKER_BUILD_ARGS: | + BUILD_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 + FROM_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 + ci_job_token=${CI_JOB_TOKEN} + netcdf_file=DATA/data.nc + +Build Latest Image: extends: - - .build + - .docker-build + needs: [] rules: - - - changes: - - 'Dockerfile' - - 'src/main/**' - - '**/*.gradle' - - '*gradle*' - - 'pyproject.toml' - - 'poetry.lock' - - 'scripts/*' - when: on_success - - - allow_failure: true - when: manual + - if: > + $CI_PROJECT_PATH == 'ghsc/nshmp/nshmp-netcdf' + && ( $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == 'production' ) variables: DOCKER_BUILD_ARGS: | BUILD_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 FROM_IMAGE=${DEVOPS_REGISTRY}usgs/java:11 ci_job_token=${CI_JOB_TOKEN} netcdf_file=DATA/data.nc + IMAGE_TAG: 'latest' -Java - Unit Tests: +Unit Tests: artifacts: paths: - ${REPORTS_DIR} @@ -229,43 +250,6 @@ Markdown Lint: - ./gradlew markdownlint; stage: build -Python - Lint: - extends: - - .python - rules: - - - changes: - - 'src/**/python/**' - - 'src/**/resources/**' - - 'pyproject.toml' - - 'poetry.lock' - when: on_success - - - allow_failure: true - when: manual - script: - - poetry run lint - stage: build - -Python - Format Check: - extends: - - .python - rules: - - - changes: - - '*/**.py' - - 'src/**/python/**' - - 'src/**/resources/**' - - 'pyproject.toml' - - 'poetry.lock' - when: on_success - - - allow_failure: true - when: manual - script: - - poetry run format_check - stage: build - YAML Lint: extends: - .gradle @@ -281,43 +265,6 @@ YAML Lint: - ./gradlew yamllint; stage: build -#### -# Stage: database -#### - -Python - Download Database: - extends: - - .python - rules: - - - allow_failure: true - when: manual - parallel: - matrix: - - NSHM: "NSHM_2018B" - - NSHM: "NSHM_2018" - - NSHM: "NSHM_2014A" - - NSHM: "NSHM_2014" - - NSHM: "NSHM_2008" - script: - - poetry run netcdf --download-only - stage: database - -Python - Convert Database: - extends: - - .python - rules: - - - allow_failure: true - when: manual - parallel: - matrix: - - NSHM: "NSHM_2018A" - script: - - poetry run netcdf - stage: database - timeout: 4h - #### # Stage: deploy ####