Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ghsc/geomag/geomag-algorithms
1 result
Show changes
Commits on Source (21)
Expand all Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 504971ac
...@@ -109,12 +109,12 @@ variables: ...@@ -109,12 +109,12 @@ variables:
script: script:
- PREFIX_LENGTH=${#REQUIRED_PREFIX}; - PREFIX_LENGTH=${#REQUIRED_PREFIX};
- if [[ "${APP_DEPLOY_DIR:0:${PREFIX_LENGTH}}" != "${REQUIRED_PREFIX}" ]]; then - if [[ "${APP_DEPLOY_DIR:0:${PREFIX_LENGTH}}" != "${REQUIRED_PREFIX}" ]]; then
echo "APP_DEPLOY_DIR does not contain correct path"; echo "APP_DEPLOY_DIR does not contain correct path";
exit 255; exit 255;
fi fi
- if [ ! -d "${APP_DEPLOY_DIR}" ]; then - if [ ! -d "${APP_DEPLOY_DIR}" ]; then
cd "$(dirname "${APP_DEPLOY_DIR}")"; cd "$(dirname "${APP_DEPLOY_DIR}")";
git clone "${CI_REPOSITORY_URL}" "$(basename "${APP_DEPLOY_DIR}")"; git clone "${CI_REPOSITORY_URL}" "$(basename "${APP_DEPLOY_DIR}")";
fi fi
- cd "${APP_DEPLOY_DIR}"; - cd "${APP_DEPLOY_DIR}";
- git checkout "${CI_COMMIT_REF_NAME}" || git checkout -b "${CI_COMMIT_REF_NAME}"; - git checkout "${CI_COMMIT_REF_NAME}" || git checkout -b "${CI_COMMIT_REF_NAME}";
...@@ -124,8 +124,8 @@ variables: ...@@ -124,8 +124,8 @@ variables:
- deploy - deploy
- swarm - swarm
variables: variables:
APP_DEPLOY_DIR: '/geomag/geomag-algorithms' APP_DEPLOY_DIR: "/geomag/geomag-algorithms"
REQUIRED_PREFIX: '/geomag' REQUIRED_PREFIX: "/geomag"
.staging: .staging:
only: only:
...@@ -182,6 +182,9 @@ Build Docker Image: ...@@ -182,6 +182,9 @@ Build Docker Image:
Scan Docker Image: Scan Docker Image:
cache: {} cache: {}
# temporarily allow while cryptography dependency has CVE
# new version no less secure than old
allow_failure: true
extends: extends:
- .adjust_image_names - .adjust_image_names
image: docker:19.03-git image: docker:19.03-git
......
Pipfile.lock
View file @ 504971ac
This diff is collapsed.
code.json
View file @ 504971ac
...@@ -3,9 +3,9 @@ ...@@ -3,9 +3,9 @@
"name": "geomag-algorithms", "name": "geomag-algorithms",
"organization": "U.S. Geological Survey", "organization": "U.S. Geological Survey",
"description": "Library for processing Geomagnetic timeseries data.", "description": "Library for processing Geomagnetic timeseries data.",
"version": "1.2.1", "version": "1.3.5",
"status": "Development", "status": "Development",
"permissions": { "permissions": {
"usageType": "openSource", "usageType": "openSource",
"licenses": [ "licenses": [
...@@ -15,27 +15,26 @@ ...@@ -15,27 +15,26 @@
} }
] ]
}, },
"homepageURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms", "homepageURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms",
"downloadURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms/-/archive/master/geomag-algorithms-master.zip", "downloadURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms/-/archive/master/geomag-algorithms-master.zip",
"disclaimerURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms/raw/master/DISCLAIMER.md", "disclaimerURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms/raw/master/DISCLAIMER.md",
"repositoryURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms.git", "repositoryURL": "https://code.usgs.gov/ghsc/geomag/geomag-algorithms.git",
"vcs": "git", "vcs": "git",
"laborHours": 0, "laborHours": 0,
"tags": ["usgs", "geomagnetism", "timeseries", "processing"], "tags": ["usgs", "geomagnetism", "timeseries", "processing"],
"languages": ["Shell", "Python"], "languages": ["Shell", "Python"],
"contact": { "contact": {
"name": "HazDev Team", "name": "HazDev Team",
"email": "gs-haz_dev_team_group@usgs.gov" "email": "gs-haz_dev_team_group@usgs.gov"
}, },
"date": { "date": {
"metadataLastUpdated": "2020-10-07" "metadataLastUpdated": "2020-11-10"
} }
} }
] ]
\ No newline at end of file
geomagio/api/ws/Observatory.py
View file @ 504971ac
...@@ -171,7 +171,7 @@ OBSERVATORIES = [ ...@@ -171,7 +171,7 @@ OBSERVATORIES = [
declination_base=209690, declination_base=209690,
), ),
Observatory( Observatory(
id="FRT", id="FDT",
elevation=69, elevation=69,
latitude=38.205, latitude=38.205,
longitude=282.627, longitude=282.627,
......
geomagio/api/ws/app.py
View file @ 504971ac
import os import os
from typing import Dict, Union
from fastapi import FastAPI, Request, Response from fastapi import FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, PlainTextResponse, RedirectResponse from fastapi.responses import JSONResponse, PlainTextResponse, RedirectResponse
from obspy import UTCDateTime from obspy import UTCDateTime
...@@ -78,13 +76,22 @@ def format_error( ...@@ -78,13 +76,22 @@ def format_error(
status_code: int, exception: str, format: str, request: Request status_code: int, exception: str, format: str, request: Request
) -> Response: ) -> Response:
"""Assign error_body value based on error format.""" """Assign error_body value based on error format."""
# These urls are embedded in responses
# and app usually runs behind reverse proxy
url = str(request.url)
usage = f"http://{request.headers['host']}/ws/docs"
if "x-forwarded-proto" in request.headers:
proto = f"{request.headers['x-forwarded-proto']}:"
url = url.replace("http:", proto)
usage = usage.replace("http:", proto)
# serve error
if format == "json": if format == "json":
return json_error(status_code, exception, request.url) return json_error(code=status_code, error=exception, url=url, usage=usage)
else: else:
return text_error(status_code, exception, request.url) return text_error(code=status_code, error=exception, url=url, usage=usage)
def json_error(code: int, error: Exception, url: str) -> Response: def json_error(code: int, error: Exception, url: str, usage: str) -> Response:
"""Format json error message. """Format json error message.
Returns Returns
...@@ -100,7 +107,8 @@ def json_error(code: int, error: Exception, url: str) -> Response: ...@@ -100,7 +107,8 @@ def json_error(code: int, error: Exception, url: str) -> Response:
"status": code, "status": code,
"error": str(error), "error": str(error),
"generated": f"{UTCDateTime().isoformat()}Z", "generated": f"{UTCDateTime().isoformat()}Z",
"url": str(url), "url": url,
"usage": usage,
"version": VERSION, "version": VERSION,
}, },
}, },
...@@ -108,7 +116,7 @@ def json_error(code: int, error: Exception, url: str) -> Response: ...@@ -108,7 +116,7 @@ def json_error(code: int, error: Exception, url: str) -> Response:
) )
def text_error(code: int, error: Exception, url: str) -> Response: def text_error(code: int, error: Exception, url: str, usage: str = "") -> Response:
"""Format error message as plain text """Format error message as plain text
Returns Returns
...@@ -120,7 +128,7 @@ def text_error(code: int, error: Exception, url: str) -> Response: ...@@ -120,7 +128,7 @@ def text_error(code: int, error: Exception, url: str) -> Response:
{error} {error}
Usage details are available from Usage details are available from {usage}
Request: Request:
{url} {url}
......
geomagio/api/ws/elements.py
View file @ 504971ac
...@@ -10,15 +10,18 @@ router = APIRouter() ...@@ -10,15 +10,18 @@ router = APIRouter()
@router.get("/elements/") @router.get("/elements/")
def get_elements() -> Dict: def get_elements() -> Dict:
return { features = []
"type": "FeatureCollection", for e in ELEMENTS:
"features": [ feature = {
{ "type": "Feature",
"type": "Feature", "id": e.id,
"id": e.id, "properties": {
"properties": {"name": e.name, "units": e.units}, "name": e.name,
"geometry": None, "units": e.units,
} },
for e in ELEMENTS "geometry": None,
], }
} if e.abbreviation:
feature["properties"]["abbreviation"] = e.abbreviation
features.append(feature)
return {"type": "FeatureCollection", "features": features}
scripts/custom.config.sh
View file @ 504971ac
...@@ -2,9 +2,13 @@ ...@@ -2,9 +2,13 @@
export SITE_URL="${SITE_URL_PREFIX}geomag${SITE_URL_SUFFIX}"; export SITE_URL="${SITE_URL_PREFIX}geomag${SITE_URL_SUFFIX}";
export BASE_HREF=${BASE_HREF:-ws}; export BASE_HREF=${BASE_HREF:-ws};
export SERVICE_MAP=( export SERVICE_MAP=(
"/${BASE_HREF}":'web' "/${BASE_HREF}:web"
); );
# Algorithms Environment Variables # Algorithms Environment Variables
export DATA_HOST=${DATA_HOST:-cwbpub.cr.usgs.gov}; export DATA_HOST=${DATA_HOST:-cwbpub.cr.usgs.gov};
export DATA_PORT=${DATA_PORT:-2060}; export DATA_PORT=${DATA_PORT:-2060};
export DATA_TYPE=${DATA_TYPE:-edge}; export DATA_TYPE=${DATA_TYPE:-edge};
if [[ $TARGET_HOSTNAME == *"mage"* ]]; then
export DATA_HOST=${TARGET_HOSTNAME}
fi
scripts/custom.funcs.sh
View file @ 504971ac
...@@ -5,6 +5,71 @@ preStackDeployHook () { ...@@ -5,6 +5,71 @@ preStackDeployHook () {
writeYmlFile; writeYmlFile;
} }
##
# override the default updateRouting, which adds a trailing slash...
##
updateRouting () {
local appName=$1; shift;
local stackName=$1; shift;
local serviceMap=$@;
local dir="$(pwd -P)";
local stamp=$(date);
local configFile="${dir}/${appName}.conf";
local serverFile="${dir}/${appName}.server";
local upstreamIdx=0;
debug "Re-routing traffic to ${stackName} stack.";
echo "# Auto generated ${stamp} for ${stackName}" > $configFile;
echo "# Auto generated ${stamp} for ${stackName}" > $serverFile;
for service in ${serviceMap[@]}; do
local name="${stackName}_$(echo $service | awk -F: '{print $2}')";
local upstreamName="${name}_${upstreamIdx}";
local path="$(echo $service | awk -F: '{print $1}')";
local port=$(getPublishedPort $name 2> /dev/null);
if [ -z "${port}" ]; then
# No port exposed. Continue.
debug "No port exposed for ${name}. Not routing. Moving on.";
continue;
fi
echo "upstream ${upstreamName} {" >> $configFile;
for host in ${TARGET_HOSTS[@]}; do
echo " server ${host}:${port};" >> $configFile;
done
echo "}" >> $configFile;
cat <<- EO_SERVER_SNIP >> $serverFile
# do not include trailing slash here, map can if needed
location ${path} {
proxy_pass http://${upstreamName};
proxy_connect_timeout 5s;
proxy_set_header Host \$host;
proxy_set_header X-Client-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
# keep redirects on https
proxy_redirect off;
}
EO_SERVER_SNIP
# ^^ Note: TAB indentation required
upstreamIdx=$((upstreamIdx + 1));
done
if [ $(configsDiffer ${appName} ${configFile} ${serverFile}) ]; then
debug "Updating configuration for ${appName}";
routerConfig --update ${appName} ${configFile} ${serverFile};
else
debug "${appName} configuration not changed. Skipping router update.";
fi
}
## ##
# Write a customized YML file for deploying the stack. Necessary because # Write a customized YML file for deploying the stack. Necessary because
# by default, YML files do not allow variables for defining configs. # by default, YML files do not allow variables for defining configs.
...@@ -34,7 +99,7 @@ services: ...@@ -34,7 +99,7 @@ services:
- 8000 - 8000
environment: environment:
- BASE_HREF=${BASE_HREF} - BASE_HREF=${BASE_HREF}
- DATA_HOST=${TARGET_HOSTNAME} - DATA_HOST=${DATA_HOST}
- DATA_PORT=${DATA_PORT} - DATA_PORT=${DATA_PORT}
- DATA_TYPE=${DATA_TYPE} - DATA_TYPE=${DATA_TYPE}
- SITE_URL=${SITE_URL} - SITE_URL=${SITE_URL}
......
setup.py
View file @ 504971ac
...@@ -9,7 +9,7 @@ if ssl_cert_file: ...@@ -9,7 +9,7 @@ if ssl_cert_file:
setuptools.setup( setuptools.setup(
name="geomag-algorithms", name="geomag-algorithms",
version="1.2.1", version="1.3.5",
description="USGS Geomag Algorithms Library", description="USGS Geomag Algorithms Library",
url="https://github.com/usgs/geomag-algorithms", url="https://github.com/usgs/geomag-algorithms",
packages=setuptools.find_packages(exclude=["test*"]), packages=setuptools.find_packages(exclude=["test*"]),
......