diff --git a/docs/overview.drawio b/docs/overview.drawio new file mode 100644 index 0000000000000000000000000000000000000000..c818995414b53076f4db586475835ab7ba72d9eb --- /dev/null +++ b/docs/overview.drawio @@ -0,0 +1,95 @@ +<mxfile host="65bd71144e" modified="2021-01-21T17:16:38.931Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.52.1 Chrome/83.0.4103.122 Electron/9.3.5 Safari/537.36" etag="W-LIp3zateXR4xu9K7Xh" version="13.10.0" type="embed"> + <diagram id="Vd_ur7joLU8uI5hJ4403" name="Page-1"> + <mxGraphModel dx="832" dy="510" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0"> + <root> + <mxCell id="0"/> + <mxCell id="1" parent="0"/> + <mxCell id="31" value="geomag-web-absolutes" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;fontStyle=2" parent="1" vertex="1"> + <mxGeometry x="720" y="380" width="200" height="120" as="geometry"/> + </mxCell> + <mxCell id="19" value="Web Applications<br>(geomag-plots)" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;strokeWidth=1;" parent="1" vertex="1"> + <mxGeometry x="720" y="80" width="200" height="260" as="geometry"/> + </mxCell> + <mxCell id="18" value="Web Services<br>(geomag-algorithms)" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;strokeWidth=1;" parent="1" vertex="1"> + <mxGeometry x="480" y="80" width="200" height="260" as="geometry"/> + </mxCell> + <mxCell id="17" value="VM" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;" parent="1" vertex="1"> + <mxGeometry x="240" y="80" width="200" height="260" as="geometry"/> + </mxCell> + <mxCell id="13" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;startArrow=none;startFill=0;entryPerimeter=0;" parent="1" source="3" target="37" edge="1"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="340" y="180" as="targetPoint"/> + </mxGeometry> + </mxCell> + <mxCell id="27" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;startArrow=none;startFill=0;dashed=1;" parent="1" source="3" target="5" edge="1"> + <mxGeometry relative="1" as="geometry"/> + </mxCell> + <mxCell id="3" value="Data Processing<br>(geomag-algorithms)" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="280" y="240" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="12" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="4" target="37" edge="1"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="400" y="150" as="targetPoint"/> + </mxGeometry> + </mxCell> + <mxCell id="4" value="Data Web Service" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="520" y="140" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="29" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;startArrow=none;startFill=0;" parent="1" source="5" target="36" edge="1"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="580" y="360" as="targetPoint"/> + </mxGeometry> + </mxCell> + <mxCell id="5" value="Metadata Web Service" style="rounded=0;whiteSpace=wrap;html=1;dashed=1;" parent="1" vertex="1"> + <mxGeometry x="520" y="240" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="10" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="6" target="4" edge="1"> + <mxGeometry relative="1" as="geometry"/> + </mxCell> + <mxCell id="11" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;" parent="1" source="6" target="5" edge="1"> + <mxGeometry relative="1" as="geometry"/> + </mxCell> + <mxCell id="6" value="Plots" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="760" y="140" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;dashed=1;" parent="1" source="7" target="5" edge="1"> + <mxGeometry relative="1" as="geometry"/> + </mxCell> + <mxCell id="9" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="7" target="4" edge="1"> + <mxGeometry relative="1" as="geometry"/> + </mxCell> + <mxCell id="7" value="Operations" style="rounded=0;whiteSpace=wrap;html=1;dashed=1;" parent="1" vertex="1"> + <mxGeometry x="760" y="240" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="16" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;startArrow=classic;startFill=1;endArrow=none;endFill=0;" parent="1" source="15" target="37" edge="1"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="280" y="150" as="targetPoint"/> + </mxGeometry> + </mxCell> + <mxCell id="15" value="Observatories" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxGeometry x="40" y="140" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="32" style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;startArrow=none;startFill=0;" parent="1" source="30" target="36" edge="1"> + <mxGeometry relative="1" as="geometry"> + <mxPoint x="640" y="450" as="targetPoint"/> + </mxGeometry> + </mxCell> + <mxCell id="30" value="Web Absolutes" style="rounded=0;whiteSpace=wrap;html=1;fontStyle=2" parent="1" vertex="1"> + <mxGeometry x="760" y="420" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="33" value="MagProc" style="rounded=0;whiteSpace=wrap;html=1;fontStyle=2" vertex="1" parent="1"> + <mxGeometry x="280" y="440" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="34" value="Residual Spreadsheets" style="rounded=0;whiteSpace=wrap;html=1;fontStyle=2" vertex="1" parent="1"> + <mxGeometry x="280" y="380" width="120" height="60" as="geometry"/> + </mxCell> + <mxCell id="36" value="MySQL" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;strokeWidth=1;" vertex="1" parent="1"> + <mxGeometry x="550" y="410" width="60" height="80" as="geometry"/> + </mxCell> + <mxCell id="37" value="Edge CWB<br>Timeseries Data" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;strokeWidth=1;" vertex="1" parent="1"> + <mxGeometry x="280" y="130" width="120" height="80" as="geometry"/> + </mxCell> + </root> + </mxGraphModel> + </diagram> +</mxfile> \ No newline at end of file diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000000000000000000000000000000000000..6fb9466d03488d44f309a0fdeb199c5b7d7381dc --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,39 @@ +# Overview + +## Background + +Geomag observatories produce raw time series data based on measurements of Earth’s magnetic field. These raw time series are adjusted in near-real time to eliminate spikes and apply baseline corrections, and later adjusted again to produce definitive data. Additionally, other algorithms combine time series to produce derived time series. + +## Architecture + + + +Observatories use a new ObsRIO system which records vector magnetometer data at 10Hz, generates MiniSEED formatted output, and supports SEEDLink for data transfer. Data are reported using double precision floating point, which mirrors the ObsRIO internal processing format, and MiniSEED blocks are generated every few seconds to decrease latency. + +Observatory data is acquired and stored using Edge Continuous Waveform Buffer (CWB) software. Edge CWB acquires MiniSEED data using SEEDLink and other protocols and provides TCP query services for data access. This software and format is widely used by the Seismic community for high volume timeseries data and supported by another team. + +The Geomag Algorithms python library (this project) includes algorithms for data processing as well as web services for data access. Geomag Algorithms uses ObsPy for timeseries data, and FastAPI for web services. + +The Geomag Plots project includes web applications that access web services to visualize and eventually manage data and metadata. + +Web Absolutes is a legacy application used to enter regular calibration measurements recorded at observatories. This supports the "null" method, but does not support the "residual" method used at high latitudes; which use spreadsheet macros. + +MagProc definitive data processing software was developed to process one-minute data. It provides tools for reviewers to identify and remove spikes from data, and requires manual effort to prepare and process data. A separate version of MagProc was created to support one second data, but requires outputs from processing one-minute data; instead of filtering processed one second data to generate one minute data. + +The Operations web application is under development to replace the legacy Web Absolutes and MagProc applications. It will let users enter absolute measurements, flag timeseries data, and manage other metadata used in data processing. Reviewers confirm metadata is correct, and can manually process quasi-definitive and definitive data using a new Adjusted algorithm. + +## Adjusted Algorithm + +The Adjusted algorithm applies a baseline correction transformation matrix that both scales and rotates from raw data to corrected data. These transformation matrices are computed manually, updated infrequently, and require additional effort to deploy. + +The initial plan is to create these transformation matrix "keyframe"s more regularly, and build a web service to access multiple versions. The algorithm is being updated to support a weighted least squares regression that is more reliable. New matrices will be uploaded to the web service and stored for reproducibility. An existing matrix can be "closed" in preparation for changes at observatories, so adjusted data is not produced until a stable baseline correction can be calculated. + +Near-real time adjusted data will use the web service to look up the current transformation matrix and process data as usual. When a new matrix is deployed in real time, existing data is not recomputed and the change may result in small steps or similar data artifacts. + +Quasi-definitive data is processed at least one to two weeks after collection, providing time for data to be manually flagged and additional observations to be collected. Matrices used during Quasi-definitive can incorporate additional observations when calculating keyframes. Each keyframe will use a consistent window of observations - for example, 7 weeks before keyframe time and 1 week after. A new algorithm to interpolate between keyframes is being developed to prevent steps in data at keyframes. + +Definitive data is processed at least one to two months after the end of the calendar year, providing time for additional observations to be collected. Matrices used during Definitive processing can incorporate a larger window of observations, but processing of data once the matrices are calculated is otherwise identical to Quasi-definitive. + +## Flagging + +Flagging information will be tracked in the Metadata web service database, allowing automatic flags to be used for near-real time adjusted data. Manual flags can be added and automatic flags reviewed using the Operations web application before Quasi-definitive processing. In addition to flagging data spikes, absolute observations and data offsets should be flagged/reviewed in a similar manner for use when computing baseline transformation matrices. diff --git a/docs/overview.png b/docs/overview.png new file mode 100644 index 0000000000000000000000000000000000000000..37b8612f030e946ca4d0c1943f1ef11a0efe0ec9 Binary files /dev/null and b/docs/overview.png differ