From a9498e9b336cb91bed740390fc408a359da5ba76 Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Mon, 19 Oct 2020 09:32:25 -0600
Subject: [PATCH 01/10] Add deploy library job
---
.gitlab-ci.yml | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9b07d2ca..668d2adc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -193,6 +193,17 @@ Deploy Mage01:
variables:
APP_NAME: geomag-algorithms
+Deploy Library Mage01:
+ extends:
+ - .deploy-library
+ - .staging
+ tags:
+ - deploy
+ - swarm
+ - mage01
+ variables:
+ APP_NAME: geomag-algorithms
+
Deploy Mage02:
extends:
- .deploy
--
GitLab
From 134353e08ff02694104af91f97dc590d2a883026 Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Mon, 19 Oct 2020 10:41:44 -0600
Subject: [PATCH 02/10] Add deploy library tag
---
.gitlab-ci.yml | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 668d2adc..e52ef86c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -98,6 +98,30 @@ variables:
- deploy
- swarm
+.deploy-library:
+ extends:
+ - .deploy
+ image: "docker:stable-git"
+ script:
+ - PREFIX_LENGTH=${#REQUIRED_PREFIX};
+ - if [[ "${APP_DEPLOY_DIR:0:${PREFIX_LENGTH}}" != "${REQUIRED_PREFIX}" ]]; then
+ echo "APP_DEPLOY_DIR does not contain correct path";
+ exit 255;
+ fi
+ - if [ ! -d "${APP_DEPLOY_DIR}" ]; then
+ cd "$(dirname "${APP_DEPLOY_DIR}")";
+ git clone "${CI_REPOSITORY_URL}" "$(basename "${APP_DEPLOY_DIR}")";
+ fi
+ - cd "${APP_DEPLOY_DIR}";
+ - git checkout "${CI_COMMIT_REF_NAME}" || git checkout -b "${CI_COMMIT_REF_NAME}";
+ - git pull --ff-only "${CI_REPOSITORY_URL}" "${CI_COMMIT_REF_NAME}";
+ stage: deploy
+ variables:
+ APP_DEPLOY_DIR: ''
+ REQUIRED_PREFIX: '/efs/www/apps/'
+ CI_REPOSITORY_URL: 'https://code.usgs.gov/ghsc/geomag/geomag-algorithms'
+
+
.mage:
variables:
DATA_HOST: "cwbpub.cr.usgs.gov"
--
GitLab
From 748c0d70f9e4fa0c75f19fe13444bd302f89750a Mon Sep 17 00:00:00 2001
From: pcain-usgs <pcain@usgs.gov>
Date: Mon, 19 Oct 2020 11:01:00 -0600
Subject: [PATCH 03/10] Set deploy directory and prefix
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e52ef86c..9267e8cb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -117,8 +117,8 @@ variables:
- git pull --ff-only "${CI_REPOSITORY_URL}" "${CI_COMMIT_REF_NAME}";
stage: deploy
variables:
- APP_DEPLOY_DIR: ''
- REQUIRED_PREFIX: '/efs/www/apps/'
+ APP_DEPLOY_DIR: '/home/geomag_user/geomag-algorithms'
+ REQUIRED_PREFIX: '/home/geomag_user/geomag-algorithms'
CI_REPOSITORY_URL: 'https://code.usgs.gov/ghsc/geomag/geomag-algorithms'
--
GitLab
From 06e2759c166288c6e1f6b457e0730104af52c932 Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 16:20:41 +0000
Subject: [PATCH 04/10] Rename deploy jobs, update deploy directory and prefix.
---
.gitlab-ci.yml | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9267e8cb..28e96fe8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -117,8 +117,8 @@ variables:
- git pull --ff-only "${CI_REPOSITORY_URL}" "${CI_COMMIT_REF_NAME}";
stage: deploy
variables:
- APP_DEPLOY_DIR: '/home/geomag_user/geomag-algorithms'
- REQUIRED_PREFIX: '/home/geomag_user/geomag-algorithms'
+ APP_DEPLOY_DIR: '/geomag/geomag-algorithms'
+ REQUIRED_PREFIX: '/geomag'
CI_REPOSITORY_URL: 'https://code.usgs.gov/ghsc/geomag/geomag-algorithms'
@@ -206,9 +206,9 @@ Scan Docker Image:
# Deploy Stage
## --------------------------------------------------
-Deploy Mage01:
+Deploy Mage01 Library:
extends:
- - .deploy
+ - .deploy-library
- .staging
tags:
- deploy
@@ -217,9 +217,9 @@ Deploy Mage01:
variables:
APP_NAME: geomag-algorithms
-Deploy Library Mage01:
+Deploy Mage01 Web Service:
extends:
- - .deploy-library
+ - .deploy
- .staging
tags:
- deploy
@@ -228,7 +228,7 @@ Deploy Library Mage01:
variables:
APP_NAME: geomag-algorithms
-Deploy Mage02:
+Deploy Mage02 Web Service:
extends:
- .deploy
- .production
@@ -239,7 +239,7 @@ Deploy Mage02:
variables:
APP_NAME: geomag-algorithms
-Deploy Algorithms Production01:
+Deploy Production01 Web Service:
extends:
- .deploy
- .production
@@ -250,7 +250,7 @@ Deploy Algorithms Production01:
variables:
APP_NAME: geomag-algorithms
-Deploy Algorithms Production02:
+Deploy Production02 Web Service:
extends:
- .deploy
- .production
@@ -261,7 +261,7 @@ Deploy Algorithms Production02:
variables:
APP_NAME: geomag-algorithms
-Deploy Algorithms Staging01:
+Deploy Staging01 Web Service:
extends:
- .deploy
- .staging
@@ -272,7 +272,7 @@ Deploy Algorithms Staging01:
variables:
APP_NAME: geomag-algorithms
-Deploy Algorithms Staging02:
+Deploy Staging02 Web Service:
extends:
- .deploy
- .staging
--
GitLab
From 58797bb8de5ab5f6566fef154baa1be783e7ac3c Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 16:22:30 +0000
Subject: [PATCH 05/10] Remove explicit CI_REPOSITORY_URL variable (already
provided by gitlab)
---
.gitlab-ci.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 28e96fe8..f8d34a08 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -119,8 +119,6 @@ variables:
variables:
APP_DEPLOY_DIR: '/geomag/geomag-algorithms'
REQUIRED_PREFIX: '/geomag'
- CI_REPOSITORY_URL: 'https://code.usgs.gov/ghsc/geomag/geomag-algorithms'
-
.mage:
variables:
--
GitLab
From 7df0c76f4a3f1cf8253e23f8d69b22f232e7e746 Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 16:27:09 +0000
Subject: [PATCH 06/10] Add pip caching for check_code step
---
.gitlab-ci.yml | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f8d34a08..ae27e650 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,10 @@
image: ${DEVOPS_REGISTRY}usgs/centos:latest
+cache:
+ paths:
+ # cache pip installed dependencies, see PIP_CACHE_DIR variable below
+ - .cache/pip
+
stages:
- test
- integration
@@ -9,6 +14,7 @@ stages:
variables:
CI_REGISTRY: ${CODE_REGISTRY}
CI_REGISTRY_IMAGE: ${CODE_REGISTRY_IMAGE}
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
TRIVY_VERSION: "0.11.0"
# docker variables
DOCKER_DRIVER: overlay2
@@ -61,7 +67,6 @@ variables:
- build
.check_code:
- cache: {}
image: ${DEVOPS_REGISTRY}usgs/conda:latest
script:
- export PYTHON_VERSION=${PYTHON_VERSION:-3.8}
@@ -99,8 +104,7 @@ variables:
- swarm
.deploy-library:
- extends:
- - .deploy
+ cache: {}
image: "docker:stable-git"
script:
- PREFIX_LENGTH=${#REQUIRED_PREFIX};
@@ -116,16 +120,13 @@ variables:
- git checkout "${CI_COMMIT_REF_NAME}" || git checkout -b "${CI_COMMIT_REF_NAME}";
- git pull --ff-only "${CI_REPOSITORY_URL}" "${CI_COMMIT_REF_NAME}";
stage: deploy
+ tags:
+ - deploy
+ - swarm
variables:
APP_DEPLOY_DIR: '/geomag/geomag-algorithms'
REQUIRED_PREFIX: '/geomag'
-.mage:
- variables:
- DATA_HOST: "cwbpub.cr.usgs.gov"
- DATA_PORT: "2060"
- DATA_TYPE: "edge"
-
.staging:
only:
- master@ghsc/geomag/geomag-algorithms
--
GitLab
From c7839eae18c48272f2d0c7943819167269e65b66 Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 19:26:37 +0000
Subject: [PATCH 07/10] Comment out pip caching
---
.gitlab-ci.yml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ae27e650..06b201da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,9 +1,9 @@
image: ${DEVOPS_REGISTRY}usgs/centos:latest
-cache:
- paths:
- # cache pip installed dependencies, see PIP_CACHE_DIR variable below
- - .cache/pip
+#cache:
+# paths:
+# # cache pip installed dependencies, see PIP_CACHE_DIR variable below
+# - .cache/pip
stages:
- test
@@ -14,7 +14,7 @@ stages:
variables:
CI_REGISTRY: ${CODE_REGISTRY}
CI_REGISTRY_IMAGE: ${CODE_REGISTRY_IMAGE}
- PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+# PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
TRIVY_VERSION: "0.11.0"
# docker variables
DOCKER_DRIVER: overlay2
--
GitLab
From eaf1c5690e85641fe97451150271acebd50ea620 Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 19:40:59 +0000
Subject: [PATCH 08/10] Lock in to databases v0.3.2 for now
which uses the simpler to install psycopg2-binary.
---
Pipfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Pipfile b/Pipfile
index 4414bbc3..93a51671 100644
--- a/Pipfile
+++ b/Pipfile
@@ -21,7 +21,7 @@ pycurl = "==7.43.0.5"
# web service dependencies
authlib = "*"
cryptography = "*"
-databases = {extras=["postgresql", "sqlite"], version = "*"}
+databases = {extras=["postgresql", "sqlite"], version = "0.3.2"}
fastapi = "*"
gunicorn = "*"
httpx = "==0.11.1"
--
GitLab
From 817c48bda3343f1ad793f6ff0fef3520ad96efbf Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 19:41:38 +0000
Subject: [PATCH 09/10] Re-add pip caching.
---
.gitlab-ci.yml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 06b201da..ae27e650 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,9 +1,9 @@
image: ${DEVOPS_REGISTRY}usgs/centos:latest
-#cache:
-# paths:
-# # cache pip installed dependencies, see PIP_CACHE_DIR variable below
-# - .cache/pip
+cache:
+ paths:
+ # cache pip installed dependencies, see PIP_CACHE_DIR variable below
+ - .cache/pip
stages:
- test
@@ -14,7 +14,7 @@ stages:
variables:
CI_REGISTRY: ${CODE_REGISTRY}
CI_REGISTRY_IMAGE: ${CODE_REGISTRY_IMAGE}
-# PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
TRIVY_VERSION: "0.11.0"
# docker variables
DOCKER_DRIVER: overlay2
--
GitLab
From f70566f39c53081a753c960b2aad07e09c4c8833 Mon Sep 17 00:00:00 2001
From: Jeremy M Fee <jmfee@usgs.gov>
Date: Tue, 20 Oct 2020 19:45:35 +0000
Subject: [PATCH 10/10] Update version syntax in pipfile
---
Pipfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Pipfile b/Pipfile
index 93a51671..0fe36b73 100644
--- a/Pipfile
+++ b/Pipfile
@@ -21,7 +21,7 @@ pycurl = "==7.43.0.5"
# web service dependencies
authlib = "*"
cryptography = "*"
-databases = {extras=["postgresql", "sqlite"], version = "0.3.2"}
+databases = {extras=["postgresql", "sqlite"], version = "==0.3.2"}
fastapi = "*"
gunicorn = "*"
httpx = "==0.11.1"
--
GitLab