diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53f95126c619dd95e1a1734c76cf7b083c622d26..62726d9d35a3627b9674beb76b2b9c3a26d95262 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,3 @@ ---- -include: - - project: 'ghsc/hazdev/pipeline-build-template' - ref: 'master' - file: 'templates/library.yml' - image: ${DEVOPS_REGISTRY}usgs/centos:latest stages: @@ -14,6 +8,15 @@ stages: variables: CI_REGISTRY: ${CODE_REGISTRY} CI_REGISTRY_IMAGE: ${CODE_REGISTRY_IMAGE} + # docker variables + DOCKER_DRIVER: overlay2 + FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/centos:latest + # environment variables + APP_NAME: geomag-algorithms + DATA_HOST: 'cwbpub.cr.usgs.gov' + DATA_PORT: '2060' + DATA_TYPE: 'edge' + ## -------------------------------------------------- # Templates @@ -30,7 +33,11 @@ variables: cache: {} extends: - .adjust_image_names - - .dind + image: docker:19.03-git + only: + - master@ghsc/geomag/geomag-algorithms + - production@ghsc/geomag/geomag-algorithms + - tags@ghsc/geomag/geomag-algorithms script: - docker build --pull @@ -46,26 +53,18 @@ variables: - docker tag local/${IMAGE_NAME} ${INTERNAL_IMAGE_NAME} - docker push ${INTERNAL_IMAGE_NAME} - docker image rm ${INTERNAL_IMAGE_NAME} + services: + - docker:19.03-dind stage: integration tags: - build - variables: - FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/centos:latest .check_code: cache: {} image: ${DEVOPS_REGISTRY}usgs/conda:latest script: - # Add conda to path - - source /etc/profile.d/conda.sh - # Install Project Dependencies - - conda config --add channels conda-forge - - conda install python=${PYTHON_VERSION} obspy pycurl - - pip install pipenv - - pipenv --site-packages install --dev --pre --skip-lock - # Run Code Checks - - pipenv run black --check . - - pipenv run pytest --cov-report xml:cov.xml --cov=geomagio + - export PYTHON_VERSION=${PYTHON_VERSION:-3.8} + - scripts/ci_check_code.sh artifacts: paths: - cov.xml @@ -74,8 +73,6 @@ variables: stage: test tags: - development - variables: - PYTHON_VERSION: 3.8 .deploy: cache: {} @@ -89,6 +86,7 @@ variables: generic-deploy/default.funcs.sh generic-deploy/deploy.sh scripts/. + - export APP_NAME=${APP_NAME} - export IMAGE_NAME=${IMAGE_NAME} - export REGISTRY=${CI_REGISTRY_IMAGE} @@ -98,8 +96,6 @@ variables: tags: - deploy - swarm - variables: - APP_NAME: geomag-algorithms .mage: variables: @@ -107,13 +103,20 @@ variables: DATA_PORT: '2060' DATA_TYPE: 'edge' -.mage01: - tags: - - mage01 +.staging: + only: + - master@ghsc/geomag/geomag-algorithms + - tags@ghsc/geomag/geomag-algorithms + +.production: + except: + - ^.*beta.*$ + - ^.*-rc.*$ + only: + - tags@ghsc/geomag/geomag-algorithms + - production@ghsc/geomag/geomag-algorithms + when: manual -.mage02: - tags: - - mage02 ## -------------------------------------------------- # Test Stage @@ -122,38 +125,38 @@ variables: Check Python 3.6: extends: - .check_code - variables: - PYTHON_VERSION: '3.6' + script: + - export PYTHON_VERSION=3.6 + - scripts/ci_check_code.sh Check Python 3.7: extends: - .check_code - variables: - PYTHON_VERSION: '3.7' + script: + - export PYTHON_VERSION=3.7 + - scripts/ci_check_code.sh Check Python 3.8: extends: - .check_code - variables: - PYTHON_VERSION: '3.8' + script: + - export PYTHON_VERSION=3.8 + - scripts/ci_check_code.sh + ## -------------------------------------------------- # Integration Stage ## -------------------------------------------------- -Build Staging Docker Image: +Build Docker Image: extends: - .build_docker_image - - .staging variables: APP_NAME: geomag-algorithms -Build Production Docker Image: - extends: - - .build_docker_image - - .production - variables: - APP_NAME: geomag-algorithms + +# TODO: Penetration testing (e.g. OWASP ZAP etc...) + ## -------------------------------------------------- # Deploy Stage @@ -163,8 +166,10 @@ Deploy Mage01: extends: - .deploy - .staging - - .mage - - .mage01 + tags: + - deploy + - swarm + - mage01 variables: APP_NAME: geomag-algorithms @@ -172,7 +177,54 @@ Deploy Mage02: extends: - .deploy - .production - - .mage - - .mage02 + tags: + - deploy + - swarm + - mage02 + variables: + APP_NAME: geomag-algorithms + + +Deploy Algorithms Production01: + extends: + - .deploy + - .production + tags: + - deploy + - swarm + - production01 + variables: + APP_NAME: geomag-algorithms + +Deploy Algorithms Production02: + extends: + - .deploy + - .production + tags: + - deploy + - swarm + - production02 + variables: + APP_NAME: geomag-algorithms + +Deploy Algorithms Staging01: + extends: + - .deploy + - .staging + tags: + - deploy + - swarm + - staging01 + variables: + APP_NAME: geomag-algorithms + +Deploy Algorithms Staging02: + extends: + - .deploy + - .staging + tags: + - deploy + - swarm + - staging02 variables: APP_NAME: geomag-algorithms diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh new file mode 100755 index 0000000000000000000000000000000000000000..9177da920fc2f4e02c5c945a3d68c7f2d281ece0 --- /dev/null +++ b/scripts/ci_check_code.sh @@ -0,0 +1,19 @@ +#! /bin/bash -ex + +PYTHON_VERSION=${PYTHON_VERSION:-"3.8"} + + +if [ -f "/etc/profile.d/conda.sh" ]; then + # Add conda to path + source /etc/profile.d/conda.sh +fi + +# Install Project Dependencies +conda config --add channels conda-forge +conda install python=${PYTHON_VERSION} obspy pycurl +pip install pipenv +pipenv --site-packages install --dev --pre --skip-lock + +# Run Code Checks +pipenv run black --check . +pipenv run pytest --cov-report xml:cov.xml --cov=geomagio