From b131e133e471b6db40298d170b26f55a3f37a0b7 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Dec 2020 20:27:58 -0700
Subject: [PATCH 1/3] Update how container builds (remove conda)
---
.gitlab-ci.yml | 30 ++++++++-----------
Dockerfile | 64 ++++++++--------------------------------
Pipfile | 1 +
docker-entrypoint.sh | 5 ++--
scripts/ci_check_code.sh | 20 -------------
5 files changed, 28 insertions(+), 92 deletions(-)
delete mode 100755 scripts/ci_check_code.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 22edf12a8..d906b1ea1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
TRIVY_VERSION: "0.13.0"
# docker variables
DOCKER_DRIVER: overlay2
- FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/centos:7
+ FROM_IMAGE: ${CODE_REGISTRY}/devops/images/usgs/obspy:3.8
# environment variables
APP_NAME: geomag-algorithms
DATA_HOST: "cwbpub.cr.usgs.gov"
@@ -67,15 +67,20 @@ variables:
- build
.check_code:
- image: ${DEVOPS_REGISTRY}usgs/conda:latest
- script:
- - export PYTHON_VERSION=${PYTHON_VERSION:-3.8}
- - scripts/ci_check_code.sh
artifacts:
paths:
- cov.xml
reports:
junit: cov.xml
+ before_script:
+ # install dependencies
+ - pipenv --site-packages install --dev --pre --skip-lock
+ - pipenv run which python
+ image: ${DEVOPS_REGISTRY}usgs/obspy:latest
+ script:
+ - pipenv run black --check .
+ - pipenv run pytest --cov-report xml:cov.xml --cov=geomagio
+ - pipenv run safety check
stage: test
tags:
- development
@@ -148,23 +153,12 @@ variables:
Check Python 3.6:
extends:
- .check_code
- script:
- - export PYTHON_VERSION=3.6
- - scripts/ci_check_code.sh
-
-Check Python 3.7:
- extends:
- - .check_code
- script:
- - export PYTHON_VERSION=3.7
- - scripts/ci_check_code.sh
+ image: ${DEVOPS_REGISTRY}usgs/obspy:3
Check Python 3.8:
extends:
- .check_code
- script:
- - export PYTHON_VERSION=3.8
- - scripts/ci_check_code.sh
+ image: ${DEVOPS_REGISTRY}usgs/obspy:3.8
## --------------------------------------------------
# Integration Stage
diff --git a/Dockerfile b/Dockerfile
index 6b3666eb6..203f160ac 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,62 +1,17 @@
-ARG FROM_IMAGE=usgs/centos:7
+ARG FROM_IMAGE=usgs/obspy:3.8
-FROM ${FROM_IMAGE} as conda
+FROM ${FROM_IMAGE}
LABEL maintainer="Jeremy Fee <jmfee@usgs.gov>"
ARG GIT_BRANCH_NAME=none
ARG GIT_COMMIT_SHA=none
+ARG WEBSERVICE="true"
# set environment variables
ENV GIT_BRANCH_NAME=${GIT_BRANCH_NAME} \
GIT_COMMIT_SHA=${GIT_COMMIT_SHA} \
- LANG='en_US.UTF-8' \
- LC_ALL='en_US.UTF-8' \
- PATH=/conda/bin:$PATH \
- PIP_CERT=${SSL_CERT_FILE} \
- PYTHONDONTWRITEBYTECODE=1 \
- PYTHONUNBUFFERED=1 \
- REQUESTS_CA_BUNDLE=${SSL_CERT_FILE}
+ WEBSERVICE=${WEBSERVICE}
-# install conda
-RUN echo 'export PATH=/conda/bin:$PATH' > /etc/profile.d/conda.sh \
- && curl \
- https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
- -o ~/miniconda.sh \
- && /bin/bash ~/miniconda.sh -b -p /conda \
- && rm ~/miniconda.sh
-
-# install dependencies via conda
-RUN conda config --set ssl_verify $SSL_CERT_FILE \
- && conda config --add channels conda-forge \
- && conda install --yes python=3.7 jupyter obspy pycurl \
- && conda clean --all -y \
- && export PIP_CERT=$SSL_CERT_FILE \
- && pip install pipenv 'virtualenv!=20.0.22' \
- && yum install -y which \
- && yum clean all
-
-
-################################################################################
-## Development image
-
-# build by running
-# docker build -t geomag-algorithms-development --target development .
-
-FROM conda as development
-
-# install dependencies via pipenv
-WORKDIR /geomag-algorithms
-COPY Pipfile /geomag-algorithms
-RUN pipenv --site-packages install --dev --skip-lock
-
-# copy library (ignores set in .dockerignore)
-COPY . /geomag-algorithms
-
-
-################################################################################
-## Production image
-
-FROM conda
RUN useradd \
-c 'Docker image user' \
@@ -72,12 +27,17 @@ USER geomag_user
# install dependencies via pipenv
WORKDIR /data
COPY Pipfile /data/
-RUN pipenv --site-packages install --skip-lock
+RUN if [ "${WEBSERVICE}" = "true" ]; then \
+ # only install production packages for webservice
+ pipenv --site-packages install --skip-lock; \
+ else \
+ # install everything
+ pipenv --site-packages install --dev --skip-lock; \
+ fi
# copy library (ignores set in .dockerignore)
COPY . /geomag-algorithms
-EXPOSE 8000
-
# entrypoint needs double quotes
ENTRYPOINT [ "/geomag-algorithms/docker-entrypoint.sh" ]
+EXPOSE 8000
diff --git a/Pipfile b/Pipfile
index 074a95e9e..01fdf5748 100644
--- a/Pipfile
+++ b/Pipfile
@@ -6,6 +6,7 @@ verify_ssl = true
[dev-packages]
bandit = "*"
black = "==20.8b1"
+jupyter = "*"
pre-commit = "*"
pytest = "*"
pytest-cov = "*"
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index b79af220e..957e1a0a7 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -7,9 +7,10 @@ export WEBSERVICE=${WEBSERVICE:-false}
export PYTHONPATH=/geomag-algorithms
-if [ $WEBSERVICE = 'false' ]; then
+if [ "${WEBSERVICE}" = "false" ]; then
# run jupyter notebook server
- exec jupyter notebook \
+ exec pipenv run jupyter \
+ notebook \
--ip='*' \
--notebook-dir=/data \
--no-browser \
diff --git a/scripts/ci_check_code.sh b/scripts/ci_check_code.sh
deleted file mode 100755
index 5bba50af2..000000000
--- a/scripts/ci_check_code.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#! /bin/bash -ex
-
-PYTHON_VERSION=${PYTHON_VERSION:-"3.8"}
-
-
-if [ -f "/etc/profile.d/conda.sh" ]; then
- # Add conda to path
- source /etc/profile.d/conda.sh
-fi
-
-# Install Project Dependencies
-conda config --add channels conda-forge
-conda install python=${PYTHON_VERSION} obspy pycurl
-pip install pipenv
-pipenv --site-packages install --dev --pre --skip-lock
-pipenv run which python
-# Run Code Checks
-pipenv run black --check .
-pipenv run pytest --cov-report xml:cov.xml --cov=geomagio
-pipenv run safety check
--
GitLab
From 31cc7e107e0b503bb59853b3dd72c2e97e579af7 Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Dec 2020 20:37:48 -0700
Subject: [PATCH 2/3] Simplify build
---
.gitlab-ci.yml | 1 +
Dockerfile | 10 ++--------
2 files changed, 3 insertions(+), 8 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d906b1ea1..37902f1ad 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -78,6 +78,7 @@ variables:
- pipenv run which python
image: ${DEVOPS_REGISTRY}usgs/obspy:latest
script:
+ # run checks
- pipenv run black --check .
- pipenv run pytest --cov-report xml:cov.xml --cov=geomagio
- pipenv run safety check
diff --git a/Dockerfile b/Dockerfile
index 203f160ac..d78f30c1c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,7 +5,7 @@ LABEL maintainer="Jeremy Fee <jmfee@usgs.gov>"
ARG GIT_BRANCH_NAME=none
ARG GIT_COMMIT_SHA=none
-ARG WEBSERVICE="true"
+ARG WEBSERVICE="false"
# set environment variables
ENV GIT_BRANCH_NAME=${GIT_BRANCH_NAME} \
@@ -27,13 +27,7 @@ USER geomag_user
# install dependencies via pipenv
WORKDIR /data
COPY Pipfile /data/
-RUN if [ "${WEBSERVICE}" = "true" ]; then \
- # only install production packages for webservice
- pipenv --site-packages install --skip-lock; \
- else \
- # install everything
- pipenv --site-packages install --dev --skip-lock; \
- fi
+RUN pipenv --site-packages install --dev --skip-lock
# copy library (ignores set in .dockerignore)
COPY . /geomag-algorithms
--
GitLab
From 3a0af235c6c8991b1f6f5e5185f3d1f2a014cf2e Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Fri, 4 Dec 2020 20:55:15 -0700
Subject: [PATCH 3/3] Configure coverage/junit reports
---
.gitlab-ci.yml | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 37902f1ad..695a4da67 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -68,10 +68,9 @@ variables:
.check_code:
artifacts:
- paths:
- - cov.xml
reports:
- junit: cov.xml
+ cobertura: cov.xml
+ junit: junit.xml
before_script:
# install dependencies
- pipenv --site-packages install --dev --pre --skip-lock
@@ -80,7 +79,7 @@ variables:
script:
# run checks
- pipenv run black --check .
- - pipenv run pytest --cov-report xml:cov.xml --cov=geomagio
+ - pipenv run pytest --cov=geomagio --cov-report xml:cov.xml --junitxml junit.xml
- pipenv run safety check
stage: test
tags:
--
GitLab