From 7b0164e52e9f1833bb6e2152f243ae2b7730a8a9 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 18:26:55 -0600
Subject: [PATCH 1/8] Update pipeline
---
.gitlab-ci.yml | 116 ++++++++++++++++++++++++++++++++++++-------------
1 file changed, 87 insertions(+), 29 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 53f95126..6330f79d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,9 +1,3 @@
----
-include:
- - project: 'ghsc/hazdev/pipeline-build-template'
- ref: 'master'
- file: 'templates/library.yml'
-
image: ${DEVOPS_REGISTRY}usgs/centos:latest
stages:
@@ -14,6 +8,14 @@ stages:
variables:
CI_REGISTRY: ${CODE_REGISTRY}
CI_REGISTRY_IMAGE: ${CODE_REGISTRY_IMAGE}
+ # docker variables
+ FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/centos:latest
+ # environment variables
+ APP_NAME: geomag-algorithms
+ DATA_HOST: 'cwbpub.cr.usgs.gov'
+ DATA_PORT: '2060'
+ DATA_TYPE: 'edge'
+
## --------------------------------------------------
# Templates
@@ -30,7 +32,11 @@ variables:
cache: {}
extends:
- .adjust_image_names
- - .dind
+ image: docker:19.03-git
+ only:
+ - master@ghsc/geomag/geomag-algorithms
+ - production@ghsc/geomag/geomag-algorithms
+ - tags@ghsc/geomag/geomag-algorithms
script:
- docker build
--pull
@@ -46,11 +52,11 @@ variables:
- docker tag local/${IMAGE_NAME} ${INTERNAL_IMAGE_NAME}
- docker push ${INTERNAL_IMAGE_NAME}
- docker image rm ${INTERNAL_IMAGE_NAME}
+ services:
+ - docker:19.03-dind
stage: integration
tags:
- build
- variables:
- FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/centos:latest
.check_code:
cache: {}
@@ -89,6 +95,7 @@ variables:
generic-deploy/default.funcs.sh
generic-deploy/deploy.sh
scripts/.
+
- export APP_NAME=${APP_NAME}
- export IMAGE_NAME=${IMAGE_NAME}
- export REGISTRY=${CI_REGISTRY_IMAGE}
@@ -98,8 +105,6 @@ variables:
tags:
- deploy
- swarm
- variables:
- APP_NAME: geomag-algorithms
.mage:
variables:
@@ -107,13 +112,20 @@ variables:
DATA_PORT: '2060'
DATA_TYPE: 'edge'
-.mage01:
- tags:
- - mage01
+.staging:
+ only:
+ - master@ghsc/geomag/geomag-plots
+ - tags@ghsc/geomag/geomag-plots
+
+.production:
+ except:
+ - ^.*beta.*$
+ - ^.*-rc.*$
+ only:
+ - tags@ghsc/geomag/geomag-plots
+ - production@ghsc/geomag/geomag-plots
+ when: manual
-.mage02:
- tags:
- - mage02
## --------------------------------------------------
# Test Stage
@@ -137,23 +149,20 @@ Check Python 3.8:
variables:
PYTHON_VERSION: '3.8'
+
## --------------------------------------------------
# Integration Stage
## --------------------------------------------------
-Build Staging Docker Image:
+Build Docker Image:
extends:
- .build_docker_image
- - .staging
variables:
APP_NAME: geomag-algorithms
-Build Production Docker Image:
- extends:
- - .build_docker_image
- - .production
- variables:
- APP_NAME: geomag-algorithms
+
+# TODO: Penetration testing (e.g. OWASP ZAP etc...)
+
## --------------------------------------------------
# Deploy Stage
@@ -163,8 +172,10 @@ Deploy Mage01:
extends:
- .deploy
- .staging
- - .mage
- - .mage01
+ tags:
+ - deploy
+ - swarm
+ - mage01
variables:
APP_NAME: geomag-algorithms
@@ -172,7 +183,54 @@ Deploy Mage02:
extends:
- .deploy
- .production
- - .mage
- - .mage02
+ tags:
+ - deploy
+ - swarm
+ - mage02
+ variables:
+ APP_NAME: geomag-algorithms
+
+
+Deploy Algorithms Production01:
+ extends:
+ - .deploy
+ - .production
+ tags:
+ - deploy
+ - swarm
+ - production01
+ variables:
+ APP_NAME: geomag-algorithms
+
+Deploy Algorithms Production02:
+ extends:
+ - .deploy
+ - .production
+ tags:
+ - deploy
+ - swarm
+ - production02
+ variables:
+ APP_NAME: geomag-algorithms
+
+Deploy Algorithms Staging01:
+ extends:
+ - .deploy
+ - .staging
+ tags:
+ - deploy
+ - swarm
+ - staging01
+ variables:
+ APP_NAME: geomag-algorithms
+
+Deploy Algorithms Staging02:
+ extends:
+ - .deploy
+ - .staging
+ tags:
+ - deploy
+ - swarm
+ - staging02
variables:
APP_NAME: geomag-algorithms
--
GitLab
From ab49067bd93bb501808d77c068c2daa31505e752 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 18:32:57 -0600
Subject: [PATCH 2/8] Update staging/production targets
---
.gitlab-ci.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6330f79d..24ada48e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -114,16 +114,16 @@ variables:
.staging:
only:
- - master@ghsc/geomag/geomag-plots
- - tags@ghsc/geomag/geomag-plots
+ - master@ghsc/geomag/geomag-algorithms
+ - tags@ghsc/geomag/geomag-algorithms
.production:
except:
- ^.*beta.*$
- ^.*-rc.*$
only:
- - tags@ghsc/geomag/geomag-plots
- - production@ghsc/geomag/geomag-plots
+ - tags@ghsc/geomag/geomag-algorithms
+ - production@ghsc/geomag/geomag-algorithms
when: manual
--
GitLab
From 9759c900b3e69393749aede00063408834a71245 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 18:37:14 -0600
Subject: [PATCH 3/8] Update check_code to run script, to allow local
gitlab-runner exec
---
.gitlab-ci.yml | 29 +++++++++++------------------
scripts/ci_check_code.sh | 19 +++++++++++++++++++
2 files changed, 30 insertions(+), 18 deletions(-)
create mode 100644 scripts/ci_check_code.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 24ada48e..db210e7b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -62,16 +62,8 @@ variables:
cache: {}
image: ${DEVOPS_REGISTRY}usgs/conda:latest
script:
- # Add conda to path
- - source /etc/profile.d/conda.sh
- # Install Project Dependencies
- - conda config --add channels conda-forge
- - conda install python=${PYTHON_VERSION} obspy pycurl
- - pip install pipenv
- - pipenv --site-packages install --dev --pre --skip-lock
- # Run Code Checks
- - pipenv run black --check .
- - pipenv run pytest --cov-report xml:cov.xml --cov=geomagio
+ - export PYTHON_VERSION=${PYTHON_VERSION:-3.8}
+ - scripts/ci_check_code.sh
artifacts:
paths:
- cov.xml
@@ -80,8 +72,6 @@ variables:
stage: test
tags:
- development
- variables:
- PYTHON_VERSION: 3.8
.deploy:
cache: {}
@@ -134,20 +124,23 @@ variables:
Check Python 3.6:
extends:
- .check_code
- variables:
- PYTHON_VERSION: '3.6'
+ script:
+ - export PYTHON_VERSION=3.6
+ - scripts/ci_check_code.sh
Check Python 3.7:
extends:
- .check_code
- variables:
- PYTHON_VERSION: '3.7'
+ script:
+ - export PYTHON_VERSION=3.7
+ - scripts/ci_check_code.sh
Check Python 3.8:
extends:
- .check_code
- variables:
- PYTHON_VERSION: '3.8'
+ script:
+ - export PYTHON_VERSION=3.8
+ - scripts/ci_check_code.sh
## --------------------------------------------------
diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh
new file mode 100644
index 00000000..14cd8be1
--- /dev/null
+++ b/scripts/ci_check_code.sh
@@ -0,0 +1,19 @@
+#! /bin/bash
+
+$PYTHON_VERSION=${PYTHON_VERSION:3.8}
+
+
+if [ -f "/etc/profile.d/conda.sh" ]; then
+ # Add conda to path
+ source /etc/profile.d/conda.sh
+fi
+
+# Install Project Dependencies
+conda config --add channels conda-forge
+conda install python=${PYTHON_VERSION} obspy pycurl
+pip install pipenv
+pipenv --site-packages install --dev --pre --skip-lock
+
+# Run Code Checks
+pipenv run black --check .
+pipenv run pytest --cov-report xml:cov.xml --cov=geomagio
--
GitLab
From e3ddde632451e3de9384902ca960628048136a21 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 18:45:06 -0600
Subject: [PATCH 4/8] use "git update-index --chmod=+x" to make ci_check_code
executable
---
scripts/ci_check_code.sh | 0
1 file changed, 0 insertions(+), 0 deletions(-)
mode change 100644 => 100755 scripts/ci_check_code.sh
diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh
old mode 100644
new mode 100755
--
GitLab
From 9461fda34f787e1a1667f0980d712ab56a9f69b4 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 18:46:57 -0600
Subject: [PATCH 5/8] Add -x, fix variable default
---
scripts/ci_check_code.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh
index 14cd8be1..9d6824e0 100755
--- a/scripts/ci_check_code.sh
+++ b/scripts/ci_check_code.sh
@@ -1,6 +1,6 @@
-#! /bin/bash
+#! /bin/bash -x
-$PYTHON_VERSION=${PYTHON_VERSION:3.8}
+$PYTHON_VERSION=${PYTHON_VERSION:-"3.8"}
if [ -f "/etc/profile.d/conda.sh" ]; then
--
GitLab
From e2e61f091a4ca3e1bee10f616b6c883b502e3446 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 18:55:10 -0600
Subject: [PATCH 6/8] Update bash variable set
---
scripts/ci_check_code.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh
index 9d6824e0..c7ab192e 100755
--- a/scripts/ci_check_code.sh
+++ b/scripts/ci_check_code.sh
@@ -1,6 +1,6 @@
#! /bin/bash -x
-$PYTHON_VERSION=${PYTHON_VERSION:-"3.8"}
+PYTHON_VERSION=${PYTHON_VERSION:-"3.8"}
if [ -f "/etc/profile.d/conda.sh" ]; then
--
GitLab
From 9086365158c244c9ed542d22fcfe9a53aaa67323 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 19:03:17 -0600
Subject: [PATCH 7/8] Update bash command to be -ex
---
scripts/ci_check_code.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh
index c7ab192e..9177da92 100755
--- a/scripts/ci_check_code.sh
+++ b/scripts/ci_check_code.sh
@@ -1,4 +1,4 @@
-#! /bin/bash -x
+#! /bin/bash -ex
PYTHON_VERSION=${PYTHON_VERSION:-"3.8"}
--
GitLab
From 4c2c562f45e0a0b9de60e30e43f101d862236182 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Sep 2020 19:39:37 -0600
Subject: [PATCH 8/8] Add DOCKER_DRIVER variable
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index db210e7b..62726d9d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,6 +9,7 @@ variables:
CI_REGISTRY: ${CODE_REGISTRY}
CI_REGISTRY_IMAGE: ${CODE_REGISTRY_IMAGE}
# docker variables
+ DOCKER_DRIVER: overlay2
FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/centos:latest
# environment variables
APP_NAME: geomag-algorithms
--
GitLab