From 21a1f2b8fad39dc3b8a30ae02361b9ca1617f7ec Mon Sep 17 00:00:00 2001
From: Jeremy Fee <jmfee@usgs.gov>
Date: Tue, 23 Mar 2021 23:29:30 -0600
Subject: [PATCH] Merge apiclient into metadata package, cleanup
---
.gitignore | 1 +
geomagio/api/secure/login.py | 66 +++++-----
geomagio/apiclient/__init__.py | 4 -
.../MetadataFactory.py | 50 +++++---
geomagio/metadata/__init__.py | 3 +-
.../metadata.py => metadata/main.py} | 114 +++++++++++++-----
setup.py | 2 +-
7 files changed, 151 insertions(+), 89 deletions(-)
delete mode 100644 geomagio/apiclient/__init__.py
rename geomagio/{apiclient => metadata}/MetadataFactory.py (52%)
rename geomagio/{apiclient/metadata.py => metadata/main.py} (65%)
diff --git a/.gitignore b/.gitignore
index a8cadd0b8..12c925f9c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
.coverage
cov.xml
.DS_Store
+.eggs
node_modules
*.pyc
coverage.xml
diff --git a/geomagio/api/secure/login.py b/geomagio/api/secure/login.py
index b687dec3b..9da5838a6 100644
--- a/geomagio/api/secure/login.py
+++ b/geomagio/api/secure/login.py
@@ -35,16 +35,20 @@ Usage:
"""
import logging
import os
-import requests
from typing import Callable, List, Optional
from authlib.integrations.starlette_client import OAuth
from fastapi import APIRouter, Depends, HTTPException
+import httpx
from pydantic import BaseModel
from starlette.requests import Request
from starlette.responses import RedirectResponse
+GITLAB_HOST = os.getenv("GITLAB_HOST", "code.usgs.gov")
+GITLAB_API_URL = os.getenv("GITLAB_API_URL", f"https://{GITLAB_HOST}/api/v4")
+
+
class User(BaseModel):
"""Information about a logged in user."""
@@ -64,45 +68,39 @@ async def current_user(request: Request) -> Optional[User]:
user: Optional[User] = Depends(current_user)
"""
+ user = None
if "user" in request.session:
- return User(**request.session["user"])
- if "apiuser" in request.session:
- return User(**request.session["apiuser"])
- if "Authorization" in request.headers:
- user = get_api_user(token=request.headers["Authorization"])
+ user = User(**request.session["user"])
+ elif "Authorization" in request.headers:
+ user = await get_gitlab_user(token=request.headers["Authorization"])
if user is not None:
- request.session["apiuser"] = user.dict()
- return user
- return None
+ request.session["user"] = user.dict()
+ return user
-def get_api_user(token: str) -> Optional[User]:
- url = os.getenv("GITLAB_API_URL")
+async def get_gitlab_user(token: str, url: str = GITLAB_API_URL) -> Optional[User]:
header = {"PRIVATE-TOKEN": token}
# request user information from gitlab api with access token
- userinfo_response = requests.get(
- f"{url}/user",
- headers=header,
- )
- userinfo = userinfo_response.json()
+
try:
- user = User(
- email=userinfo["email"],
- sub=userinfo["id"],
- name=userinfo["name"],
- nickname=userinfo["username"],
- picture=userinfo["avatar_url"],
- )
- except KeyError:
- logging.info(f"Invalid token: {userinfo_response.status_code}")
+ # use httpx/async so this doesn't block other requests
+ async with httpx.AsyncClient() as client:
+ userinfo_response = await client.get(f"{url}/user", headers=header)
+ userinfo = userinfo_response.json()
+ user = User(
+ email=userinfo["email"],
+ sub=userinfo["id"],
+ name=userinfo["name"],
+ nickname=userinfo["username"],
+ picture=userinfo["avatar_url"],
+ )
+ # use valid token to retrieve user's groups
+ groups_response = await client.get(f"{url}/groups", headers=header)
+ user.groups = [g["full_path"] for g in groups_response.json()]
+ return user
+ except Exception:
+ logging.exception(f"Unable to get gitlab user")
return None
- # use valid token to retrieve user's groups
- groups_response = requests.get(
- f"{url}/groups",
- headers=header,
- )
- user.groups = [g["full_path"] for g in groups_response.json()]
- return user
def require_user(
@@ -143,7 +141,9 @@ oauth.register(
name="openid",
client_id=os.getenv("OPENID_CLIENT_ID"),
client_secret=os.getenv("OPENID_CLIENT_SECRET"),
- server_metadata_url=os.getenv("OPENID_METADATA_URL"),
+ server_metadata_url=os.getenv(
+ "OPENID_METADATA_URL", f"https://{GITLAB_HOST}/.well-known/openid-configuration"
+ ),
client_kwargs={"scope": "openid email profile"},
)
# routes for login/logout
diff --git a/geomagio/apiclient/__init__.py b/geomagio/apiclient/__init__.py
deleted file mode 100644
index 53bb45e4c..000000000
--- a/geomagio/apiclient/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from .metadata import app
-from .MetadataFactory import MetadataFactory
-
-__all__ = ["app", "MetadataFactory"]
diff --git a/geomagio/apiclient/MetadataFactory.py b/geomagio/metadata/MetadataFactory.py
similarity index 52%
rename from geomagio/apiclient/MetadataFactory.py
rename to geomagio/metadata/MetadataFactory.py
index 288901c68..0558bcc20 100644
--- a/geomagio/apiclient/MetadataFactory.py
+++ b/geomagio/metadata/MetadataFactory.py
@@ -1,57 +1,73 @@
import os
import requests
-from typing import List, Union
+from typing import List
from obspy import UTCDateTime
from pydantic import parse_obj_as
-from ..metadata import Metadata, MetadataQuery
+from .Metadata import Metadata
+from .MetadataQuery import MetadataQuery
+
+
+GEOMAG_API_HOST = os.getenv("GEOMAG_API_HOST", "geomag.usgs.gov")
+GEOMAG_API_URL = f"https://{GEOMAG_API_HOST}/ws/secure/metadata"
+if "127.0.0.1" in GEOMAG_API_URL:
+ GEOMAG_API_URL = GEOMAG_API_URL.replace("https://", "http://")
class MetadataFactory(object):
def __init__(
self,
- url: str = "http://{}/ws/secure/metadata".format(
- os.getenv("GEOMAG_API_HOST", "127.0.0.1:8000")
- ),
+ url: str = GEOMAG_API_URL,
token: str = os.getenv("GITLAB_API_TOKEN"),
):
self.url = url
self.token = token
- self.header = self._get_headers()
def _get_headers(self):
return {"Authorization": self.token} if self.token else None
def delete_metadata(self, metadata: Metadata) -> bool:
- response = requests.delete(url=f"{self.url}/{metadata.id}", headers=self.header)
+ response = requests.delete(
+ url=f"{self.url}/{metadata.id}",
+ headers=self._get_headers(),
+ )
if response.status_code == 200:
return True
return False
def get_metadata(self, query: MetadataQuery) -> List[Metadata]:
if query.id:
- response = self.get_metadata_by_id(id=query.id)
+ metadata = [self.get_metadata_by_id(id=query.id)]
else:
- args = parse_params(query=query)
- response = requests.get(url=self.url, params=args, headers=self.header)
- metadata = parse_obj_as(Union[List[Metadata], Metadata], response.json())
- if isinstance(metadata, Metadata):
- metadata = [metadata]
+ response = requests.get(
+ url=self.url,
+ headers=self._get_headers(),
+ params=parse_params(query=query),
+ )
+ metadata = parse_obj_as(List[Metadata], response.json())
return metadata
- def get_metadata_by_id(self, id: int) -> requests.Response:
- return requests.get(f"{self.url}/{id}", headers=self.header)
+ def get_metadata_by_id(self, id: int) -> Metadata:
+ response = requests.get(
+ url=f"{self.url}/{id}",
+ headers=self._get_headers(),
+ )
+ return Metadata(**response.json())
def create_metadata(self, metadata: Metadata) -> Metadata:
response = requests.post(
- url=self.url, data=metadata.json(), headers=self.header
+ url=self.url,
+ data=metadata.json(),
+ headers=self._get_headers(),
)
return Metadata(**response.json())
def update_metadata(self, metadata: Metadata) -> Metadata:
response = requests.put(
- url=f"{self.url}/{metadata.id}", data=metadata.json(), headers=self.header
+ url=f"{self.url}/{metadata.id}",
+ data=metadata.json(),
+ headers=self._get_headers(),
)
return Metadata(**response.json())
diff --git a/geomagio/metadata/__init__.py b/geomagio/metadata/__init__.py
index 37b2adc33..592b2d159 100644
--- a/geomagio/metadata/__init__.py
+++ b/geomagio/metadata/__init__.py
@@ -1,6 +1,7 @@
from .Metadata import Metadata
from .MetadataCategory import MetadataCategory
+from .MetadataFactory import MetadataFactory
from .MetadataQuery import MetadataQuery
-__all__ = ["Metadata", "MetadataCategory", "MetadataQuery"]
+__all__ = ["Metadata", "MetadataCategory", "MetadataFactory", "MetadataQuery"]
diff --git a/geomagio/apiclient/metadata.py b/geomagio/metadata/main.py
similarity index 65%
rename from geomagio/apiclient/metadata.py
rename to geomagio/metadata/main.py
index afb3ea63e..79cf886ba 100644
--- a/geomagio/apiclient/metadata.py
+++ b/geomagio/metadata/main.py
@@ -1,13 +1,48 @@
import sys
import json
import os
+import textwrap
from typing import Dict, Optional
from obspy import UTCDateTime
import typer
-from ..metadata import Metadata, MetadataCategory, MetadataQuery
+from .Metadata import Metadata
+from .MetadataCategory import MetadataCategory
from .MetadataFactory import MetadataFactory
+from .MetadataQuery import MetadataQuery
+
+
+GEOMAG_API_HOST = os.getenv("GEOMAG_API_HOST", "geomag.usgs.gov")
+GEOMAG_API_URL = f"https://{GEOMAG_API_HOST}/ws/secure/metadata"
+if "127.0.0.1" in GEOMAG_API_URL:
+ GEOMAG_API_URL = GEOMAG_API_URL.replace("https://", "http://")
+
+
+ENVIRONMENT_VARIABLE_HELP = """Environment variables:
+
+ GITLAB_API_TOKEN
+
+ (Required) Personal access token with "read_api" scope. Create at
+ https://code.usgs.gov/profile/personal_access_tokens
+
+ GEOMAG_API_HOST
+
+ Default "geomag.usgs.gov"
+
+ REQUESTS_CA_BUNDLE
+
+ Use custom certificate bundle
+ """
+
+
+app = typer.Typer(
+ help=f"""
+ Command line interface for Metadata API
+
+ {ENVIRONMENT_VARIABLE_HELP}
+ """
+)
def load_metadata(input_file: str) -> Optional[Dict]:
@@ -21,14 +56,22 @@ def load_metadata(input_file: str) -> Optional[Dict]:
return data
-app = typer.Typer()
+def main():
+ """Command line interface for Metadata API.
+
+ Registered as "geomag-metadata" console script in setup.py.
+ """
+ app()
+
+@app.command(
+ help=f"""
+ Create new metadata.
-@app.command()
+ {ENVIRONMENT_VARIABLE_HELP}
+ """
+)
def create(
- url: str = "http://{}/ws/secure/metadata".format(
- os.getenv("GEOMAG_API_HOST", "127.0.0.1:8000")
- ),
category: MetadataCategory = None,
channel: str = None,
created_after: str = None,
@@ -42,6 +85,7 @@ def create(
network: str = None,
starttime: str = None,
station: str = None,
+ url: str = GEOMAG_API_URL,
wrap: bool = True,
):
input_metadata = load_metadata(input_file=input_file)
@@ -67,12 +111,16 @@ def create(
print(metadata.json())
-@app.command()
+@app.command(
+ help=f"""
+ Delete an existing metadata.
+
+ {ENVIRONMENT_VARIABLE_HELP}
+ """
+)
def delete(
input_file: str,
- url: str = "http://{}/ws/secure/metadata".format(
- os.getenv("GEOMAG_API_HOST", "127.0.0.1:8000")
- ),
+ url: str = GEOMAG_API_URL,
):
metadata_dict = load_metadata(input_file=input_file)
metadata = Metadata(**metadata_dict)
@@ -81,24 +129,28 @@ def delete(
sys.exit(1)
-@app.command()
+@app.command(
+ help=f"""
+ Search existing metadata.
+
+ {ENVIRONMENT_VARIABLE_HELP}
+ """
+)
def get(
- url: str = "http://{}/ws/secure/metadata".format(
- os.getenv("GEOMAG_API_HOST", "127.0.0.1:8000")
- ),
category: Optional[MetadataCategory] = None,
channel: Optional[str] = None,
created_after: Optional[str] = None,
created_before: Optional[str] = None,
- data_valid: Optional[bool] = True,
+ data_valid: Optional[bool] = None,
endtime: Optional[str] = None,
+ getone: bool = False,
id: Optional[int] = None,
location: Optional[str] = None,
- metadata_valid: Optional[bool] = True,
+ metadata_valid: Optional[bool] = None,
network: Optional[str] = None,
starttime: Optional[str] = None,
station: Optional[str] = None,
- getone: bool = False,
+ url: str = GEOMAG_API_URL,
):
query = MetadataQuery(
category=category,
@@ -115,30 +167,26 @@ def get(
station=station,
)
metadata = MetadataFactory(url=url).get_metadata(query=query)
- if not metadata:
- print([])
- return
-
if getone:
- if len(metadata) > 1:
- raise ValueError("More than one matching record")
+ if len(metadata) != 1:
+ raise ValueError(f"{len(metadata)} matching records")
print(metadata[0].json())
- return
- print("[" + ",".join([m.json() for m in metadata]) + "]")
+ else:
+ print("[" + ",\n".join([m.json() for m in metadata]) + "]")
-@app.command()
+@app.command(
+ help=f"""
+ Update an existing metadata.
+
+ {ENVIRONMENT_VARIABLE_HELP}
+ """
+)
def update(
input_file: str,
- url: str = "http://{}/ws/secure/metadata".format(
- os.getenv("GEOMAG_API_HOST", "127.0.0.1:8000")
- ),
+ url: str = GEOMAG_API_URL,
):
metadata_dict = load_metadata(input_file=input_file)
metadata = Metadata(**metadata_dict)
response = MetadataFactory(url=url).update_metadata(metadata=metadata)
print(response.json())
-
-
-def main():
- app()
diff --git a/setup.py b/setup.py
index a73fd11e7..9a9c5ace7 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@ setuptools.setup(
entry_points={
"console_scripts": [
"generate-matrix=geomagio.processing.adjusted:main",
- "geomag-apiclient=geomagio.apiclient.metadata:main",
+ "geomag-metadata=geomagio.metadata.main:main",
"magproc-prepfiles=geomagio.processing.magproc:main",
"obsrio-filter=geomagio.processing.obsrio:main",
],
--
GitLab