Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ghsc/geomag/geomag-algorithms
1 result
Show changes
Commits on Source (4)
Hide whitespace changes
Inline Side-by-side
docs/overview.drawio 0 → 100644
View file @ 624cfc95
<mxfile host="65bd71144e" modified="2021-01-21T17:16:38.931Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.52.1 Chrome/83.0.4103.122 Electron/9.3.5 Safari/537.36" etag="W-LIp3zateXR4xu9K7Xh" version="13.10.0" type="embed">
<diagram id="Vd_ur7joLU8uI5hJ4403" name="Page-1">
<mxGraphModel dx="832" dy="510" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0"/>
<mxCell id="1" parent="0"/>
<mxCell id="31" value="geomag-web-absolutes" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;fontStyle=2" parent="1" vertex="1">
<mxGeometry x="720" y="380" width="200" height="120" as="geometry"/>
</mxCell>
<mxCell id="19" value="Web Applications&lt;br&gt;(geomag-plots)" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;strokeWidth=1;" parent="1" vertex="1">
<mxGeometry x="720" y="80" width="200" height="260" as="geometry"/>
</mxCell>
<mxCell id="18" value="Web Services&lt;br&gt;(geomag-algorithms)" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;strokeWidth=1;" parent="1" vertex="1">
<mxGeometry x="480" y="80" width="200" height="260" as="geometry"/>
</mxCell>
<mxCell id="17" value="VM" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;" parent="1" vertex="1">
<mxGeometry x="240" y="80" width="200" height="260" as="geometry"/>
</mxCell>
<mxCell id="13" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;startArrow=none;startFill=0;entryPerimeter=0;" parent="1" source="3" target="37" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="340" y="180" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="27" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;startArrow=none;startFill=0;dashed=1;" parent="1" source="3" target="5" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="3" value="Data Processing&lt;br&gt;(geomag-algorithms)" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="280" y="240" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="12" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="4" target="37" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="400" y="150" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="4" value="Data Web Service" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="520" y="140" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="29" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;startArrow=none;startFill=0;" parent="1" source="5" target="36" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="580" y="360" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="5" value="Metadata Web Service" style="rounded=0;whiteSpace=wrap;html=1;dashed=1;" parent="1" vertex="1">
<mxGeometry x="520" y="240" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="10" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="6" target="4" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="11" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;" parent="1" source="6" target="5" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="6" value="Plots" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="760" y="140" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;" parent="1" source="7" target="5" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="9" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="7" target="4" edge="1">
<mxGeometry relative="1" as="geometry"/>
</mxCell>
<mxCell id="7" value="Operations" style="rounded=0;whiteSpace=wrap;html=1;dashed=1;" parent="1" vertex="1">
<mxGeometry x="760" y="240" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="16" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;startArrow=classic;startFill=1;endArrow=none;endFill=0;" parent="1" source="15" target="37" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="280" y="150" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="15" value="Observatories" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="40" y="140" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="32" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;startArrow=none;startFill=0;" parent="1" source="30" target="36" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="640" y="450" as="targetPoint"/>
</mxGeometry>
</mxCell>
<mxCell id="30" value="Web Absolutes" style="rounded=0;whiteSpace=wrap;html=1;fontStyle=2" parent="1" vertex="1">
<mxGeometry x="760" y="420" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="33" value="MagProc" style="rounded=0;whiteSpace=wrap;html=1;fontStyle=2" vertex="1" parent="1">
<mxGeometry x="280" y="440" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="34" value="Residual Spreadsheets" style="rounded=0;whiteSpace=wrap;html=1;fontStyle=2" vertex="1" parent="1">
<mxGeometry x="280" y="380" width="120" height="60" as="geometry"/>
</mxCell>
<mxCell id="36" value="MySQL" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;strokeWidth=1;" vertex="1" parent="1">
<mxGeometry x="550" y="410" width="60" height="80" as="geometry"/>
</mxCell>
<mxCell id="37" value="Edge CWB&lt;br&gt;Timeseries Data" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;strokeWidth=1;" vertex="1" parent="1">
<mxGeometry x="280" y="130" width="120" height="80" as="geometry"/>
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>
\ No newline at end of file
docs/overview.md 0 → 100644
View file @ 624cfc95
# Overview
## Background
Geomag observatories produce raw time series data based on measurements of
Earth’s magnetic field. These raw time series are adjusted in near-real time to
eliminate spikes and apply baseline corrections, and later adjusted again to
produce definitive data. Additionally, other algorithms combine time series to
produce derived time series.
## Architecture
![Overview Diagram](overview.png)
Observatories use a new ObsRIO system which records vector magnetometer data at
10Hz, generates MiniSEED formatted output, and supports SEEDLink for data
transfer. Data are reported using double precision floating point, which mirrors
the ObsRIO internal processing format, and MiniSEED blocks are generated every
few seconds to decrease latency.
Observatory data is acquired and stored using Edge Continuous Waveform Buffer
(CWB) software. Edge CWB acquires MiniSEED data using SEEDLink and other
protocols and provides TCP query services for data access. This software and
format is widely used by the Seismic community for high volume timeseries data
and supported by another team.
The [Geomag Algorithms](https://code.usgs.gov/ghsc/geomag/geomag-plots) Python
library includes algorithms for data processing as well as web services for data
access. Geomag Algorithms uses ObsPy for timeseries data, and FastAPI for web
services.
The [Geomag Plots](https://code.usgs.gov/ghsc/geomag/geomag-plots) Angular
project includes web applications that access web services to visualize and
eventually manage data and metadata.
Web Absolutes is a legacy application used to enter regular calibration
measurements recorded at observatories. This supports the "null" method, but
does not support the "residual" method used at high latitudes; which use
spreadsheet macros.
MagProc definitive data processing software was developed to process one-minute
data. It provides tools for reviewers to identify and remove spikes from data,
and requires manual effort to prepare and process data. A separate version of
MagProc was created to support one second data, but requires outputs from
processing one-minute data; instead of filtering processed one second data to
generate one minute data.
The Operations web application is under development to replace the legacy Web
Absolutes and MagProc applications. It will let users enter absolute
measurements, flag timeseries data, and manage other metadata used in data
processing. Reviewers confirm metadata is correct, and can manually process
quasi-definitive and definitive data using a new Adjusted algorithm.
## Adjusted Algorithm
The Adjusted algorithm applies a baseline correction transformation matrix that
both scales and rotates from raw data to corrected data. These transformation
matrices are computed manually, updated infrequently, and require additional
effort to deploy.
The initial plan is to create these transformation matrix "keyframe"s more
regularly, and build a web service to access multiple versions. The algorithm is
being updated to support a weighted least squares regression that is more
reliable. New matrices will be uploaded to the web service and stored for
reproducibility. An existing matrix can be "closed" in preparation for changes
at observatories, so adjusted data is not produced until a stable baseline
correction can be calculated.
Near-real time adjusted data will use the web service to look up the current
transformation matrix and process data as usual. When a new matrix is deployed
in real time, existing data is not recomputed and the change may result in small
steps or similar data artifacts.
Quasi-definitive data is processed at least one to two weeks after collection,
providing time for data to be manually flagged and additional observations to be
collected. Matrices used during Quasi-definitive can incorporate additional
observations when calculating keyframes. Each keyframe will use a consistent
window of observations - for example, 7 weeks before keyframe time and 1 week
after. A new algorithm to interpolate between keyframes is being developed to
prevent steps in data at keyframes.
Definitive data is processed at least one to two months after the end of the
calendar year, providing time for additional observations to be collected.
Matrices used during Definitive processing can incorporate a larger window of
observations, but processing of data once the matrices are calculated is
otherwise identical to Quasi-definitive.
## Flagging
Flagging information will be tracked in the Metadata web service database,
allowing automatic flags to be used for near-real time adjusted data. Manual
flags can be added and automatic flags reviewed using the Operations web
application before Quasi-definitive processing. In addition to flagging data
spikes, absolute observations and data offsets should be flagged/reviewed in a
similar manner for use when computing baseline transformation matrices.
docs/overview.png 0 → 100644
View file @ 624cfc95
docs/overview.png

46.2 KiB