diff --git a/docs/algorithms/Adjusted_usage.md b/docs/algorithms/Adjusted_usage.md index 4db49e5c509c398b8aa61f6675dd28781a47326e..7388add068a673800182bfe890fd687fa9612a89 100644 --- a/docs/algorithms/Adjusted_usage.md +++ b/docs/algorithms/Adjusted_usage.md @@ -8,7 +8,7 @@ the [Adjusted Algorithm](./Adjusted.md). `geomag.py --algorithm sqdist` -### Example +### Command Line Example This example uses a state file to produce X, Y, Z and F channels from raw H, E, Z and F channels using the EDGE channel naming @@ -27,7 +27,6 @@ contained in the statefile. --outchannels X Y Z F \ --output-iaga-stdout - ### Statefile Example Content This is the content of /etc/adjusted/adjbou_state_.json: @@ -41,6 +40,56 @@ This is the content of /etc/adjusted/adjbou_state_.json: "M12": -0.15473074200902157, "M14": -1276.1646811919759, "M31": -0.006725053082782385} +### API Example + +This example uses an AdjustedMatrix object to produce X, Y, Z and F channels +from raw H, E, Z and F channels using the EDGE channel naming +convention. Absolutes were used to compute the transform matrix and pier correction +contained in the object. + +```python +from geomagio.algorithm import AdjustedAlgorithm +from geomagio.adjusted.AdjustedMatrix import AdjustedMatrix +from geomagio.iaga2002 import IAGA2002Factory + +with open("etc/adjusted/BOU201601vmin.min") as f: + raw = IAGA2002Factory().parse_string(f.read()) + +a = adj( + matrix=AdjustedMatrix( + matrix=[ + [ + 0.9834275767090617, + -0.15473074200902157, + 0.027384986324932026, + -1276.164681191976, + ], + [ + 0.16680172992706568, + 0.987916201012128, + -0.0049868332295851525, + -0.8458192581350419, + ], + [ + -0.006725053082782385, + -0.011809351484171948, + 0.9961869012493976, + 905.3800885796844, + ], + [0, 0, 0, 1], + ], + pier_correction=-22, + ) + ) +# definition can also use statefiles +# a = adj(statefile="etc/adjusted/adjbou_state_.json") + +result = adjusted.process(raw) +``` + + + + ### Library Notes > Note: this library internally represents data gaps as NaN, and diff --git a/etc/adjusted/BOU_expected.json b/etc/adjusted/BOU_expected.json new file mode 100644 index 0000000000000000000000000000000000000000..e3f46068de20dadc9188a71c1ebfb62ef2d0e8f3 --- /dev/null +++ b/etc/adjusted/BOU_expected.json @@ -0,0 +1,1126 @@ +{ + "short_causal": [ + [ + [ + 0.9871097009314467, + -0.16004511341190467, + 0, + -37.511849255164805 + ], + [ + 0.16004511341190483, + 0.987109700931447, + 0, + -235.4457568796662 + ], + [ + 0, + 0, + 1, + 577.0756736370693 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.983314127911998, + -0.18191571083518093, + 0, + 39.47155018108392 + ], + [ + 0.18191571083518096, + 0.9833141279119971, + 0, + -687.6882471353883 + ], + [ + 0, + 0, + 1, + 576.7942725719031 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9833141279119977, + -0.18191571083518102, + 0, + 39.471550181090976 + ], + [ + 0.18191571083518107, + 0.9833141279119972, + 0, + -687.6882471353906 + ], + [ + 0, + 0, + 1, + 576.7942725719031 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9851538020327897, + -0.17167407008730542, + 0, + 0.7195450827020984 + ], + [ + 0.17167407008730545, + 0.985153802032789, + 0, + -478.0074382663735 + ], + [ + 0, + 0, + 1, + 577.3916751331093 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9847280751919824, + -0.17409944838710265, + 0, + 9.46491884112513 + ], + [ + 0.17409944838710265, + 0.9847280751919824, + 0, + -527.4922171037747 + ], + [ + 0, + 0, + 1, + 577.5584135956137 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9901118679697742, + -0.14028003744440923, + 0, + -98.03252747455716 + ], + [ + 0.14028003744440917, + 0.9901118679697746, + 0, + 175.26602768706883 + ], + [ + 0, + 0, + 1, + 577.1018879032138 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9901118679697742, + -0.14028003744440923, + 0, + -98.03252747455716 + ], + [ + 0.1402800374444092, + 0.9901118679697746, + 0, + 175.26602768706834 + ], + [ + 0, + 0, + 1, + 577.1018879032138 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9919258591858662, + -0.12681912268416584, + 0, + -134.53546777305382 + ], + [ + 0.12681912268416606, + 0.9919258591858664, + 0, + 455.9929230025889 + ], + [ + 0, + 0, + 1, + 577.279558981481 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9903692399859244, + -0.13845132173331787, + 0, + -102.64034148096148 + ], + [ + 0.1384513217333179, + 0.9903692399859242, + 0, + 213.39740145429747 + ], + [ + 0, + 0, + 1, + 577.8077879580178 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9903692399859242, + -0.13845132173331787, + 0, + -102.6403414809573 + ], + [ + 0.13845132173331787, + 0.9903692399859242, + 0, + 213.39740145429792 + ], + [ + 0, + 0, + 1, + 577.8077879580178 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9911646910710129, + -0.13263692990303616, + 0, + -119.05506248263204 + ], + [ + 0.13263692990303597, + 0.9911646910710129, + 0, + 335.4583285961499 + ], + [ + 0, + 0, + 1, + 578.1774527764944 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9914203103131642, + -0.13071254070879684, + 0, + -123.47878103057063 + ], + [ + 0.13071254070879676, + 0.9914203103131644, + 0, + 375.04180207316716 + ], + [ + 0, + 0, + 1, + 578.0939187358629 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9921315874452739, + -0.12519949358252275, + 0, + -137.60006615600244 + ], + [ + 0.1251994935825226, + 0.9921315874452742, + 0, + 488.3687142412409 + ], + [ + 0, + 0, + 1, + 578.4413328140095 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9886249098255323, + -0.15040208666257698, + 0, + -66.83290795907212 + ], + [ + 0.15040208666257696, + 0.9886249098255329, + 0, + -35.37056652586703 + ], + [ + 0, + 0, + 1, + 578.2320125532879 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ], + "short_acausal": [ + [ + [ + 0.9841980584144114, + -0.17707112077722834, + 0, + 21.299054173690934 + ], + [ + 0.17707112077722773, + 0.9841980584144118, + 0, + -590.0593348189517 + ], + [ + 0, + 0, + 1, + 577.1726804427047 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9862789438593202, + -0.1650873856469475, + 0, + -21.96930164917458 + ], + [ + 0.1650873856469476, + 0.9862789438593202, + 0, + -338.5220209652398 + ], + [ + 0, + 0, + 1, + 577.3291257179811 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9868180721043974, + -0.16183353350946586, + 0, + -32.930569280884264 + ], + [ + 0.16183353350946555, + 0.9868180721043974, + 0, + -272.57414851023987 + ], + [ + 0, + 0, + 1, + 577.5130166196902 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9870306254792979, + -0.16053206647254872, + 0, + -37.12063081593727 + ], + [ + 0.160532066472549, + 0.9870306254792981, + 0, + -245.34347932190397 + ], + [ + 0, + 0, + 1, + 577.4953339759405 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9859614694020957, + -0.1669729943866995, + 0, + -14.427487128573667 + ], + [ + 0.16697299438669955, + 0.9859614694020955, + 0, + -380.4617229224813 + ], + [ + 0, + 0, + 1, + 577.2154615593286 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9871191315525296, + -0.1599869373510866, + 0, + -37.552850655344294 + ], + [ + 0.15998693735108674, + 0.9871191315525294, + 0, + -235.07955540691572 + ], + [ + 0, + 0, + 1, + 577.2569675551619 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9863300145308425, + -0.16478198456016807, + 0, + -21.666067156329618 + ], + [ + 0.1647819845601679, + 0.9863300145308426, + 0, + -335.01264151496815 + ], + [ + 0, + 0, + 1, + 577.2775308585723 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9869588966574858, + -0.16097247065457224, + 0, + -34.208866912869134 + ], + [ + 0.16097247065457207, + 0.9869588966574863, + 0, + -255.07927306724065 + ], + [ + 0, + 0, + 1, + 577.7473165139924 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9874711413684588, + -0.15779969887320738, + 0, + -44.16838139688891 + ], + [ + 0.15779969887320777, + 0.9874711413684584, + 0, + -188.96638187193005 + ], + [ + 0, + 0, + 1, + 577.91470877669 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9860004605425757, + -0.16674259146909692, + 0, + -13.679838869412256 + ], + [ + 0.1667425914690969, + 0.9860004605425751, + 0, + -374.8737406611576 + ], + [ + 0, + 0, + 1, + 577.8888623614705 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9863209698929457, + -0.16483611360815456, + 0, + -19.899159604446613 + ], + [ + 0.1648361136081547, + 0.9863209698929452, + 0, + -335.4865381536899 + ], + [ + 0, + 0, + 1, + 577.8635880922781 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9864801956247771, + -0.16388051635293008, + 0, + -23.363269571082828 + ], + [ + 0.16388051635293008, + 0.9864801956247765, + 0, + -315.5057518385161 + ], + [ + 0, + 0, + 1, + 578.0649084873531 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9875378285181738, + -0.15738181993359696, + 0, + -44.92194329417005 + ], + [ + 0.157381819933597, + 0.9875378285181733, + 0, + -180.8378739649987 + ], + [ + 0, + 0, + 1, + 578.2345073147621 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9873462968377212, + -0.1585789712441082, + 0, + -40.729956312740775 + ], + [ + 0.15857897124410833, + 0.9873462968377208, + 0, + -205.2722199036162 + ], + [ + 0, + 0, + 1, + 577.993559989513 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ], + "inf_weekly": [ + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ], + "inf_one_interval": [[ + [ + 0.982779837283398, + -0.1847803870252956, + 0, + 50.394210236635764 + ], + [ + 0.18478038702529562, + 0.9827798372833981, + 0, + -750.6964090039248 + ], + [ + 0, + 0, + 1, + 577.4055827661293 + ], + [ + 0, + 0, + 0, + 1 + ] + ]] +} \ No newline at end of file diff --git a/etc/adjusted/CMO_expected.json b/etc/adjusted/CMO_expected.json new file mode 100644 index 0000000000000000000000000000000000000000..4b53c32650c9a31e7e10969db32890a2b64d7c34 --- /dev/null +++ b/etc/adjusted/CMO_expected.json @@ -0,0 +1,3390 @@ +{ + "short_causal": [ + [ + [ + 0.9520106076060315, + -0.3060650306807265, + 0, + 99.65088924353279 + ], + [ + 0.3060650306807261, + 0.9520106076060315, + 0, + 257.97535719664864 + ], + [ + 0, + 0, + 1, + -53.21366487278599 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9552016612734489, + -0.29595571678959665, + 0, + 60.26698791955369 + ], + [ + 0.29595571678959687, + 0.9552016612734487, + 0, + 383.44028395677265 + ], + [ + 0, + 0, + 1, + -52.87006023415485 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9511585942240501, + -0.3087026540762643, + 0, + 110.129690010723 + ], + [ + 0.30870265407626424, + 0.9511585942240496, + 0, + 226.10380069007348 + ], + [ + 0, + 0, + 1, + -52.67229878128798 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9515742808541782, + -0.3074189129133971, + 0, + 104.67647657144059 + ], + [ + 0.3074189129133969, + 0.9515742808541776, + 0, + 242.10601732468 + ], + [ + 0, + 0, + 1, + -52.43740175872636 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.948621387123097, + -0.31641343823018786, + 0, + 141.25969063215254 + ], + [ + 0.3164134382301883, + 0.948621387123098, + 0, + 130.54895106153558 + ], + [ + 0, + 0, + 1, + -52.63979785302694 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9486213871230978, + -0.3164134382301881, + 0, + 141.2596906321415 + ], + [ + 0.3164134382301883, + 0.948621387123098, + 0, + 130.54895106153558 + ], + [ + 0, + 0, + 1, + -52.63979785302694 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9486213871230977, + -0.3164134382301883, + 0, + 141.25969063214285 + ], + [ + 0.31641343823018836, + 0.9486213871230981, + 0, + 130.5489510615348 + ], + [ + 0, + 0, + 1, + -52.63979785302694 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9488775268856051, + -0.31564448192777933, + 0, + 138.5736682077212 + ], + [ + 0.3156444819277799, + 0.9488775268856054, + 0, + 140.4491045887207 + ], + [ + 0, + 0, + 1, + -52.69650104874719 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9472688339429288, + -0.3204399417057814, + 0, + 158.3824370711891 + ], + [ + 0.3204399417057814, + 0.9472688339429285, + 0, + 80.74101015130873 + ], + [ + 0, + 0, + 1, + -52.89012982270929 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9479982729343638, + -0.31827546954401664, + 0, + 149.15611760297898 + ], + [ + 0.3182754695440166, + 0.9479982729343636, + 0, + 107.61021805698002 + ], + [ + 0, + 0, + 1, + -53.27139701240504 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.947998272934362, + -0.31827546954401625, + 0, + 149.15611760300106 + ], + [ + 0.3182754695440165, + 0.9479982729343631, + 0, + 107.61021805698138 + ], + [ + 0, + 0, + 1, + -53.27139701240504 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9492837362131247, + -0.31442071840330643, + 0, + 132.79138861861873 + ], + [ + 0.3144207184033068, + 0.9492837362131243, + 0, + 154.88628731934745 + ], + [ + 0, + 0, + 1, + -53.192828164424775 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9501482731754195, + -0.3117984268429974, + 0, + 122.3123260735645 + ], + [ + 0.31179842684299697, + 0.9501482731754196, + 0, + 186.83919994577647 + ], + [ + 0, + 0, + 1, + -53.49385810165857 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9501482731754192, + -0.3117984268429975, + 0, + 122.3123260735684 + ], + [ + 0.3117984268429973, + 0.9501482731754196, + 0, + 186.83919994577232 + ], + [ + 0, + 0, + 1, + -53.49385810165857 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9501482731754188, + -0.31179842684299736, + 0, + 122.31232607357265 + ], + [ + 0.3117984268429972, + 0.9501482731754195, + 0, + 186.8391999457737 + ], + [ + 0, + 0, + 1, + -53.49385810165857 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.950148273175419, + -0.31179842684299747, + 0, + 122.31232607357146 + ], + [ + 0.3117984268429971, + 0.9501482731754194, + 0, + 186.83919994577505 + ], + [ + 0, + 0, + 1, + -53.49385810165857 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9510420200470575, + -0.30906160567888885, + 0, + 110.78631608336227 + ], + [ + 0.309061605678889, + 0.9510420200470572, + 0, + 220.1228506549593 + ], + [ + 0, + 0, + 1, + -53.95450488765145 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9510420200470578, + -0.3090616056788887, + 0, + 110.78631608335698 + ], + [ + 0.30906160567888896, + 0.9510420200470572, + 0, + 220.12285065495976 + ], + [ + 0, + 0, + 1, + -53.95450488765145 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9511834337542936, + -0.3086261093157717, + 0, + 109.63477209051936 + ], + [ + 0.30862610931577167, + 0.9511834337542937, + 0, + 225.47903259111393 + ], + [ + 0, + 0, + 1, + -53.93337886228512 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9522437645884091, + -0.3053388491536775, + 0, + 96.44732813933707 + ], + [ + 0.30533884915367704, + 0.9522437645884101, + 0, + 266.16861124419376 + ], + [ + 0, + 0, + 1, + -53.83314985198863 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9522437645884108, + -0.30533884915367737, + 0, + 96.44732813931674 + ], + [ + 0.3053388491536774, + 0.9522437645884105, + 0, + 266.16861124418904 + ], + [ + 0, + 0, + 1, + -53.83314985198863 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9522474112520676, + -0.3053274762672623, + 0, + 96.20104651341549 + ], + [ + 0.3053274762672624, + 0.9522474112520676, + 0, + 266.29550094623545 + ], + [ + 0, + 0, + 1, + -53.6427500942824 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9492667038170053, + -0.3144721371193278, + 0, + 132.19674477317423 + ], + [ + 0.31447213711932726, + 0.9492667038170053, + 0, + 153.0047793960055 + ], + [ + 0, + 0, + 1, + -53.91983054901134 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9492667038170048, + -0.31447213711932775, + 0, + 132.5637586266592 + ], + [ + 0.3144721371193272, + 0.9492667038170051, + 0, + 152.8317764409154 + ], + [ + 0, + 0, + 1, + -54.11103438825747 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9492667038170055, + -0.31447213711932775, + 0, + 132.56375862665092 + ], + [ + 0.31447213711932753, + 0.9492667038170054, + 0, + 152.83177644091128 + ], + [ + 0, + 0, + 1, + -54.11103438825747 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9492667038170054, + -0.31447213711932787, + 0, + 132.563758626652 + ], + [ + 0.31447213711932775, + 0.9492667038170055, + 0, + 152.83177644090856 + ], + [ + 0, + 0, + 1, + -54.11103438825747 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9506383093674282, + -0.3103011517268955, + 0, + 115.50464948380066 + ], + [ + 0.3103011517268955, + 0.9506383093674282, + 0, + 206.01153307196805 + ], + [ + 0, + 0, + 1, + -54.400000000001455 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9505955192448176, + -0.3104322128769406, + 0, + 115.82708505308234 + ], + [ + 0.3104322128769401, + 0.9505955192448181, + 0, + 203.62370891387909 + ], + [ + 0, + 0, + 1, + -54.340851617960595 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9513703014531312, + -0.3080495893731687, + 0, + 106.4440517325126 + ], + [ + 0.30804958937316823, + 0.9513703014531312, + 0, + 232.5937707310631 + ], + [ + 0, + 0, + 1, + -54.53673610924302 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9513703014531311, + -0.30804958937316834, + 0, + 106.44405173251397 + ], + [ + 0.30804958937316806, + 0.9513703014531314, + 0, + 232.593770731065 + ], + [ + 0, + 0, + 1, + -54.53673610924302 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9516721259049419, + -0.3071158816726485, + 0, + 102.66812181019742 + ], + [ + 0.3071158816726488, + 0.9516721259049421, + 0, + 243.73830390207047 + ], + [ + 0, + 0, + 1, + -54.724043387716776 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9516926525939859, + -0.30705226753538545, + 0, + 102.3414222822089 + ], + [ + 0.3070522675353849, + 0.9516926525939859, + 0, + 244.52300143937606 + ], + [ + 0, + 0, + 1, + -54.793475102073316 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9524913061011091, + -0.304565775821586, + 0, + 92.42316654793483 + ], + [ + 0.30456577582158645, + 0.9524913061011087, + 0, + 275.13536834049125 + ], + [ + 0, + 0, + 1, + -54.62052782371143 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9524913061011095, + -0.3045657758215856, + 0, + 92.42316654792937 + ], + [ + 0.3045657758215862, + 0.952491306101109, + 0, + 275.13536834049404 + ], + [ + 0, + 0, + 1, + -54.62052782371143 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.950296018730672, + -0.3113478388950792, + 0, + 119.12979753766503 + ], + [ + 0.31134783889507983, + 0.9502960187306717, + 0, + 191.44318333401347 + ], + [ + 0, + 0, + 1, + -54.74202678734332 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9500606048354571, + -0.3120654532941546, + 0, + 122.06007435929664 + ], + [ + 0.31206545329415464, + 0.950060604835457, + 0, + 182.27466723645855 + ], + [ + 0, + 0, + 1, + -54.532197240180246 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9500606048354564, + -0.3120654532941546, + 0, + 122.06007435930393 + ], + [ + 0.3120654532941544, + 0.9500606048354565, + 0, + 182.27466723646128 + ], + [ + 0, + 0, + 1, + -54.532197240180246 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9502846023268284, + -0.3113826818892501, + 0, + 119.35310099476098 + ], + [ + 0.31138268188925017, + 0.9502846023268285, + 0, + 191.37261392106782 + ], + [ + 0, + 0, + 1, + -54.82128108711125 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9495509561919879, + -0.3136127892718685, + 0, + 128.4839608908748 + ], + [ + 0.3136127892718684, + 0.9495509561919879, + 0, + 163.37843197593557 + ], + [ + 0, + 0, + 1, + -54.51315932421658 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9491116933242957, + -0.31493966659518874, + 0, + 133.59958201705274 + ], + [ + 0.3149396665951888, + 0.949111693324296, + 0, + 146.82017888401936 + ], + [ + 0, + 0, + 1, + -54.40109499783231 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9493328378160123, + -0.31427243443260483, + 0, + 131.16521523308737 + ], + [ + 0.3142724344326044, + 0.9493328378160117, + 0, + 155.21045447759514 + ], + [ + 0, + 0, + 1, + -54.242767938657366 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9491380180978055, + -0.3148603223675052, + 0, + 133.39158086952148 + ], + [ + 0.3148603223675051, + 0.9491380180978056, + 0, + 148.03594551276845 + ], + [ + 0, + 0, + 1, + -54.07640075887624 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9491380180978056, + -0.3148603223675052, + 0, + 133.39158086952054 + ], + [ + 0.314860322367505, + 0.9491380180978055, + 0, + 148.0359455127698 + ], + [ + 0, + 0, + 1, + -54.07640075887624 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ], + "short_acausal": [ + [ + [ + 0.9497629660016135, + -0.31297013980860444, + 0, + 127.27992032316135 + ], + [ + 0.3129701398086044, + 0.9497629660016137, + 0, + 172.83563567435738 + ], + [ + 0, + 0, + 1, + -52.882802662550375 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9499176319312758, + -0.31250038807348235, + 0, + 125.28544865161753 + ], + [ + 0.3125003880734824, + 0.949917631931276, + 0, + 178.90990418300998 + ], + [ + 0, + 0, + 1, + -52.726459632828885 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9496474055263497, + -0.3133206108398736, + 0, + 128.63794898508766 + ], + [ + 0.3133206108398734, + 0.9496474055263502, + 0, + 169.1933417150359 + ], + [ + 0, + 0, + 1, + -52.518411509905384 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9500870332841831, + -0.31198498230725696, + 0, + 123.16429136202113 + ], + [ + 0.3119849823072568, + 0.950087033284183, + 0, + 185.6097808970522 + ], + [ + 0, + 0, + 1, + -52.56719725446565 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9500647521207781, + -0.3120528269022471, + 0, + 123.63151782636125 + ], + [ + 0.31205282690224684, + 0.9500647521207782, + 0, + 184.54766316289917 + ], + [ + 0, + 0, + 1, + -52.622621598143816 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9528204722536545, + -0.30353442581414564, + 0, + 89.93318566268395 + ], + [ + 0.30353442581414564, + 0.9528204722536542, + 0, + 289.95768722051054 + ], + [ + 0, + 0, + 1, + -52.64799745147313 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9527951273959943, + -0.3036139740039157, + 0, + 90.1011261470215 + ], + [ + 0.30361397400391577, + 0.9527951273959945, + 0, + 288.85233800116447 + ], + [ + 0, + 0, + 1, + -52.85032638671682 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9532026083831409, + -0.3023322466618072, + 0, + 85.09232061099232 + ], + [ + 0.3023322466618072, + 0.9532026083831411, + 0, + 304.65412663810844 + ], + [ + 0, + 0, + 1, + -52.95642710646523 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9532721889835791, + -0.3021127830967359, + 0, + 83.88586536629727 + ], + [ + 0.3021127830967357, + 0.9532721889835792, + 0, + 307.2380541573799 + ], + [ + 0, + 0, + 1, + -53.175193691696414 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9531583362986883, + -0.30247179363424626, + 0, + 85.44875313066764 + ], + [ + 0.3024717936342463, + 0.9531583362986878, + 0, + 302.7268324079667 + ], + [ + 0, + 0, + 1, + -53.21155022037145 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9531663161501063, + -0.3024466461325625, + 0, + 85.34474935065883 + ], + [ + 0.30244664613256256, + 0.9531663161501065, + 0, + 302.84365993328186 + ], + [ + 0, + 0, + 1, + -53.214027264168934 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9531696080468409, + -0.3024362714619902, + 0, + 85.29249999539867 + ], + [ + 0.3024362714619901, + 0.9531696080468413, + 0, + 302.7606526077114 + ], + [ + 0, + 0, + 1, + -53.27758035674312 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9531379171339579, + -0.3025361315967738, + 0, + 85.57110836246713 + ], + [ + 0.3025361315967738, + 0.9531379171339578, + 0, + 301.41590904344093 + ], + [ + 0, + 0, + 1, + -53.47187569504326 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530126019638929, + -0.30293065295214333, + 0, + 87.10557261117486 + ], + [ + 0.3029306529521439, + 0.9530126019638925, + 0, + 296.49443603148313 + ], + [ + 0, + 0, + 1, + -53.47821307034197 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9529475879803535, + -0.30313510941893185, + 0, + 87.65727234441096 + ], + [ + 0.30313510941893257, + 0.9529475879803533, + 0, + 293.6550546636824 + ], + [ + 0, + 0, + 1, + -53.8206144977806 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.952918435331757, + -0.30322673959411245, + 0, + 87.52228582077669 + ], + [ + 0.3032267395941123, + 0.9529184353317572, + 0, + 292.4136580267427 + ], + [ + 0, + 0, + 1, + -53.988184917863634 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9528366760357795, + -0.3034835560637305, + 0, + 88.73296005428116 + ], + [ + 0.30348355606373056, + 0.9528366760357795, + 0, + 289.2267871858141 + ], + [ + 0, + 0, + 1, + -53.96823178920712 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9527754800340136, + -0.30367562405625353, + 0, + 89.82948181771125 + ], + [ + 0.30367562405625353, + 0.9527754800340135, + 0, + 286.7824725119135 + ], + [ + 0, + 0, + 1, + -53.86009415809723 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9526741446925282, + -0.3039933782738695, + 0, + 91.18199715875774 + ], + [ + 0.3039933782738696, + 0.9526741446925282, + 0, + 282.8336636698898 + ], + [ + 0, + 0, + 1, + -53.82540698150436 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.952527717919876, + -0.3044518789469904, + 0, + 92.89729892370873 + ], + [ + 0.3044518789469901, + 0.9525277179198758, + 0, + 277.1337664286167 + ], + [ + 0, + 0, + 1, + -53.81150289619046 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9524014055392418, + -0.30484678565941703, + 0, + 94.40373288138663 + ], + [ + 0.3048467856594178, + 0.9524014055392415, + 0, + 272.4361692360764 + ], + [ + 0, + 0, + 1, + -53.63332212936763 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.952277987263523, + -0.3052321001686008, + 0, + 95.69882730442794 + ], + [ + 0.30523210016860114, + 0.9522779872635226, + 0, + 267.24406234759635 + ], + [ + 0, + 0, + 1, + -53.8369047322245 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9519239491141167, + -0.30633444974894, + 0, + 99.89443042843077 + ], + [ + 0.3063344497489405, + 0.9519239491141163, + 0, + 253.46354175240361 + ], + [ + 0, + 0, + 1, + -54.005861928134614 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9519614228223408, + -0.30621797703280734, + 0, + 99.56570113241516 + ], + [ + 0.30621797703280756, + 0.9519614228223406, + 0, + 255.07744901650892 + ], + [ + 0, + 0, + 1, + -54.14545553629375 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9517214848895255, + -0.30696288896164325, + 0, + 102.42360942886475 + ], + [ + 0.3069628889616435, + 0.9517214848895251, + 0, + 246.32383929907027 + ], + [ + 0, + 0, + 1, + -54.34041606617888 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9512203900522184, + -0.30851218703465266, + 0, + 108.31761306477459 + ], + [ + 0.3085121870346525, + 0.9512203900522185, + 0, + 227.6719209251085 + ], + [ + 0, + 0, + 1, + -54.34347214750673 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9512692367507553, + -0.3083615397737448, + 0, + 107.66000912738522 + ], + [ + 0.30836153977374414, + 0.9512692367507553, + 0, + 228.94612723171636 + ], + [ + 0, + 0, + 1, + -54.47392013977816 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9513776206666531, + -0.30802698403655976, + 0, + 106.33599241151092 + ], + [ + 0.30802698403656, + 0.9513776206666528, + 0, + 233.32878147858722 + ], + [ + 0, + 0, + 1, + -54.56242455331851 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9513087306466879, + -0.30823967784078043, + 0, + 107.0744493752633 + ], + [ + 0.3082396778407808, + 0.9513087306466878, + 0, + 230.320266886755 + ], + [ + 0, + 0, + 1, + -54.603179194884305 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9506105786417777, + -0.3103860946858668, + 0, + 115.57435498862584 + ], + [ + 0.31038609468586664, + 0.950610578641778, + 0, + 203.19598999479484 + ], + [ + 0, + 0, + 1, + -54.71562075578477 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9503143484018964, + -0.3112918874970556, + 0, + 119.07786030309022 + ], + [ + 0.31129188749705583, + 0.9503143484018963, + 0, + 191.94577906153995 + ], + [ + 0, + 0, + 1, + -54.71183454741115 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9500356637477465, + -0.31214137439208334, + 0, + 122.45181958079965 + ], + [ + 0.3121413743920831, + 0.9500356637477464, + 0, + 181.38251130176764 + ], + [ + 0, + 0, + 1, + -54.74134747486604 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9501348095023199, + -0.3118394519171522, + 0, + 121.22777483788327 + ], + [ + 0.311839451917152, + 0.9501348095023201, + 0, + 185.14893825365792 + ], + [ + 0, + 0, + 1, + -54.7224753059539 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9500994880574112, + -0.3119470512651868, + 0, + 121.76005294606801 + ], + [ + 0.3119470512651864, + 0.9500994880574112, + 0, + 183.91376679160405 + ], + [ + 0, + 0, + 1, + -54.77036961035862 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9502415293653819, + -0.31151410219978953, + 0, + 119.91439869816548 + ], + [ + 0.3115141021997899, + 0.9502415293653819, + 0, + 189.37589568272057 + ], + [ + 0, + 0, + 1, + -54.69794169418869 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9502174875676661, + -0.31158742965753733, + 0, + 120.29658734348071 + ], + [ + 0.3115874296575364, + 0.9502174875676661, + 0, + 188.5647908438023 + ], + [ + 0, + 0, + 1, + -54.66880722665258 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9496120358838342, + -0.31342779280810334, + 0, + 127.73969844873508 + ], + [ + 0.31342779280810346, + 0.9496120358838344, + 0, + 165.7270142032457 + ], + [ + 0, + 0, + 1, + -54.65389059373349 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9492310322384901, + -0.3145797950209939, + 0, + 132.48373825101822 + ], + [ + 0.314579795020994, + 0.9492310322384906, + 0, + 151.4686278906733 + ], + [ + 0, + 0, + 1, + -54.6139591944754 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.949700845852346, + -0.3131585914314641, + 0, + 126.58766294189229 + ], + [ + 0.31315859143146396, + 0.9497008458523459, + 0, + 168.9858523891425 + ], + [ + 0, + 0, + 1, + -54.43790057295874 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9496672039772364, + -0.3132605970913976, + 0, + 126.91271474785472 + ], + [ + 0.313260597091398, + 0.9496672039772364, + 0, + 167.6538314057921 + ], + [ + 0, + 0, + 1, + -54.339337779171494 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9497520015380477, + -0.3130034114422281, + 0, + 126.00038000915222 + ], + [ + 0.3130034114422282, + 0.9497520015380476, + 0, + 170.98464429976246 + ], + [ + 0, + 0, + 1, + -54.217873286785725 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9496915924226942, + -0.31318665246406663, + 0, + 126.60352150474561 + ], + [ + 0.3131866524640669, + 0.9496915924226942, + 0, + 169.01884096286346 + ], + [ + 0, + 0, + 1, + -53.833458756227905 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9497605517985698, + -0.3129774660375366, + 0, + 126.06566315391913 + ], + [ + 0.31297746603753684, + 0.9497605517985694, + 0, + 171.77221482185792 + ], + [ + 0, + 0, + 1, + -53.42655859217465 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ], + "inf_weekly": [ + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ], + "inf_one_interval": [ + [ + [ + 0.9530416740423117, + -0.30283917768120405, + 0, + 86.51389127230165 + ], + [ + 0.3028391776812039, + 0.9530416740423118, + 0, + 297.5848898679969 + ], + [ + 0, + 0, + 1, + -53.71111111111132 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + ] +} \ No newline at end of file diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150242140.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150242140.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..e9221ae181e6dccb1c040610c79e2000a729fe24 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150242140.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150312045.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150312045.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7e9743e99d9d87831c35c4963dfd48f7f5d47a89 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150312045.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150382118.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150382118.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..2b424cbffe3f699d2cd550af7fe015184a813ddc Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150382118.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150452047.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150452047.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..6e04c369c4342e9aa4d38a7fcaa978bf255644b8 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150452047.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150522238.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150522238.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..11e1ee2d6c58ba00c91f41176af07171ad3b9d02 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150522238.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150592055.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150592055.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..2b309ed9bb2fc50b5f9e7a8f17ba2765dc730b44 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150592055.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150802144.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150802144.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..1ac0109771cee477605e36ce0ba589f174a2adfa Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150802144.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150871947.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150871947.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..9e2d23551f99ac30eb0aff54142d825ffdd139e8 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150871947.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20150942001.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20150942001.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..dcd5ddb137697c94303b84927590cb7f0f4c9d34 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20150942001.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151022138.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151022138.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..f841e626ee818de49f97d5d2beacc60b04932dad Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151022138.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151081856.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151081856.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..ac8edf5e1798b2a39e97c7f47d1ce5663c7b4939 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151081856.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151151919.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151151919.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..d0f2f1af89e1c51e1880d9e7311235b6aca9182e Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151151919.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151371803.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151371803.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..3ad463298420adb86047fcaa93c4d04fc82a310f Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151371803.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151572030.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151572030.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..352b7ae77612d310c67238bc428ffdcd618d9f09 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151572030.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151642030.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151642030.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..5442be0f2072dd950ed62625075eee7e635969b0 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151642030.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151720045.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151720045.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..b6dac6d9e4b0b7cb9b3327681cbbde6ca6b6ebda Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151720045.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151782044.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151782044.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..dca6870c64d52b0b58433e91a617dfde891862b3 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151782044.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151852045.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151852045.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..bbe550bf8cd9d18d27ef528f5955f46d9fc3535e Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151852045.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20151921943.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20151921943.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..3b4d3da3d825e4027376c945eb70f34ab6bc388e Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20151921943.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152092206.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152092206.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..80c5d0b8fa383a236506bbed79ba16b9b2acb328 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152092206.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152142145.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152142145.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..6d186975721fe5a107551571e6f1d29ca39a06dc Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152142145.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152210109.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152210109.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7bf2410db112bd9a7719e39e989a7055c989a686 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152210109.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152271939.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152271939.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7cbc062630410c97683b0bb3f38c383c1d82e586 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152271939.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152350029.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152350029.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..0c1edbfb7d937163217bfb07ba181726f4a51de8 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152350029.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152412009.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152412009.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..b917b73dd0d522ff0877eced39816822c6421287 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152412009.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152481957.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152481957.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..4e317c8785f0d9e38e2e71753fe0fa343d5f7cdc Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152481957.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152551948.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152551948.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..3098c657e5fc09e31f7d785b4153f1ef78c8a6c7 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152551948.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152640443.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152640443.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7602b9745e59a33bde2fc32c831aaee7f43d1f66 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152640443.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152691948.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152691948.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7199388a3ee0bd4f8d13d84b7efa2fc241aacf32 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152691948.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152761954.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152761954.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..832f4c7ee8aaf50b2a180ad92495ee1e0c8b855d Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152761954.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152840043.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152840043.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..83a50d0b760482b91c561043519acb06573cbd97 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152840043.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152910229.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152910229.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7460546f42f234fa86e1559dfd1981671eb4852c Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152910229.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20152971940.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20152971940.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..5784ac3e3f2ecb353fae79b449de03052e58c54b Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20152971940.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153042027.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153042027.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7eb824c58a7da46fa47d3398d9298da7204084ab Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153042027.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153112110.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153112110.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..1d7190e13c952ecdde7117b808e0086f67768c5b Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153112110.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153182110.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153182110.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..ab80b2945818aac2e163685c39498ed62d3743df Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153182110.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153270200.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153270200.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..115cfcfd237f783c01ba45fe54379d62975d0888 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153270200.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153312150.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153312150.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..22c12be201a7112ada79084967a6526ff9f490fd Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153312150.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153392115.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153392115.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..fe5c971293d77dc33a0a43e20257b27c21bdf920 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153392115.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153462126.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153462126.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..76add8f1fc5a2eab2879d5861c87790814bd6774 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153462126.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153552019.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153552019.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..7878fd89e5b29714d386b3a024822db04b5ecc3d Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153552019.xlsm differ diff --git a/etc/adjusted/Caldata/CMO/2015/CMO20153592215.xlsm b/etc/adjusted/Caldata/CMO/2015/CMO20153592215.xlsm new file mode 100755 index 0000000000000000000000000000000000000000..81fc2ed6e1c386388abc9cc8f3cac09eb2643026 Binary files /dev/null and b/etc/adjusted/Caldata/CMO/2015/CMO20153592215.xlsm differ diff --git a/etc/adjusted/adjbou_state_.json b/etc/adjusted/adjbou_state_.json index 365af0fa1f48d98474ce7642cc270227bc4c92c1..7492efd366c258ee9392814efb53c9ed2451f204 100644 --- a/etc/adjusted/adjbou_state_.json +++ b/etc/adjusted/adjbou_state_.json @@ -1 +1,24 @@ -{"M32": -0.011809351484171948, "M41": -0.0, "M44": 1.0, "PC": -22, "M24": -0.84581925813504188, "M43": 0.0, "M34": 905.38008857968441, "M33": 0.99618690124939757, "M21": 0.16680172992706568, "M22": 0.98791620101212796, "M23": -0.0049868332295851525, "M11": 0.98342757670906167, "M42": -0.0, "M13": 0.027384986324932026, "M12": -0.15473074200902157, "M14": -1276.1646811919759, "M31": -0.006725053082782385} \ No newline at end of file +{ + "matrix":[ + [ + 0.9834275767090617, + -0.15473074200902157, + 0.027384986324932026, + -1276.164681191976 + ], + [ + 0.16680172992706568, + 0.987916201012128, + -0.0049868332295851525, + -0.8458192581350419 + ], + [ + -0.006725053082782385, + -0.011809351484171948, + 0.9961869012493976, + 905.3800885796844 + ], + [0, 0, 0, 1] + ], + "pier_correction": -22 +} \ No newline at end of file diff --git a/etc/adjusted/adjbou_state_HE_.json b/etc/adjusted/adjbou_state_HE_.json index 91ed3625c0bbb0ecd40e770c3ac7202797783df8..f9c0e3d9d3ba643259f9619554ecf9e2894d3f90 100644 --- a/etc/adjusted/adjbou_state_HE_.json +++ b/etc/adjusted/adjbou_state_HE_.json @@ -1 +1,8 @@ -{"PC": -22.0, "M11": -1.0, "M12": -0.0, "M13": -0.0, "M21": -0.0, "M22": -1.0, "M23": -0.0, "M31": 0.0, "M32": 0.0, "M33": 1.0} \ No newline at end of file +{ + "matrix": [ + [-1, 0, 0], + [0, -1, 0], + [0, 0, 1] + ], + "pier_correction": -22.0 +} \ No newline at end of file diff --git a/etc/adjusted/synthetic.json b/etc/adjusted/synthetic.json new file mode 100644 index 0000000000000000000000000000000000000000..c3dd5594f10c4ed6cc19c9a9256c02e72e81634f --- /dev/null +++ b/etc/adjusted/synthetic.json @@ -0,0 +1,398 @@ +{ + "variables": { + "h_ord": [ + 1.79289322, + 1.83983806, + 1.91024109, + 2.51489726, + 1.78952971, + 2.81157403, + 2.36417497, + 2.43864844, + 3.25364394, + 2.27479228, + 3.47800226, + 2.95694665, + 3.03482926, + 3.84541634, + 2.92632734, + 4.01635411, + 3.59456862, + 3.69312428, + 4.27043672, + 3.82250402 + ], + "e_ord": [ + 4.46592583, + 3.99414122, + 4.86042918, + 4.08398446, + 4.06789278, + 4.89701546, + 3.29221074, + 4.91313376, + 3.76430406, + 3.77979418, + 4.76896396, + 2.98259167, + 4.69899703, + 3.54330923, + 3.56461527, + 4.43474537, + 2.93993354, + 4.24945532, + 3.43005774, + 3.46078181 + ], + "z_ord": [ + 2.24118095, + 2.17339265, + 2.88988358, + 1.74783897, + 3.18842428, + 2.36825251, + 2.39116112, + 3.56453372, + 1.85923781, + 3.7224498, + 2.71235632, + 2.74095046, + 3.96752743, + 2.26503334, + 4.0367477, + 3.13361064, + 3.18331114, + 4.12716967, + 3.00224898, + 4.03024472 + ], + "x_abs": [ + -1, + -0.899979996, + -0.799959992, + -0.699939988, + -0.599919984, + -0.49989998, + -0.399879976, + -0.299859972, + -0.199839968, + -0.099819964, + 0.000200040008, + 0.100220044, + 0.200240048, + 0.300260052, + 0.400280056, + 0.50030006, + 0.600320064, + 0.700340068, + 0.800360072, + 0.900380076 + ], + "y_abs": [ + 0, + -0.35248241, + 0.18456572, + 0.22223661, + -0.64867776, + 0.86607699, + -0.7390734, + 0.29000341, + 0.30840017, + -0.80892729, + 0.99997154, + -0.80006479, + 0.29431378, + 0.30363751, + -0.74702288, + 0.8657967, + -0.64130022, + 0.211858, + 0.19298425, + -0.35563497 + ], + "z_abs": [ + 0, + -0.25649982, + 0.57096366, + -0.67874509, + 0.46830885, + 0.0032657, + -0.54209456, + 0.90883553, + -0.93002867, + 0.57937261, + 0.00754126, + -0.59148311, + 0.93449629, + -0.904239, + 0.53078497, + 0.00979431, + -0.47785966, + 0.68164505, + -0.56760976, + 0.25067805 + ] + }, + "results": { + "NoConstraints": [ + [ + 0.5364919027495736, + -0.20707814501736121, + 0.33725290893445425, + -1.7929218549826373 + ], + [ + 0.5325045930544003, + 0.847092478025647, + -0.32696549213267934, + -4.004987214867749 + ], + [ + -0.25884768127170443, + 0.4215661361680674, + 0.6706148784369698, + -2.921566136168065 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "ZRotationShear": [ + [ + 0.5587910280835219, + -0.1593988978820583, + 0, + -0.9594702768899068 + ], + [ + 0.5108856705906101, + 0.8008676180137191, + 0, + -4.813015546664324 + ], + [ + 0, + 0, + 0.7036074465845426, + -2.273788534881388 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "ZRotationHscale": [ + [ + 0.6395521012137755, + -0.3193786953272828, + 0, + -0.6145604575448345 + ], + [ + 0.3193786953272828, + 0.6395521012137755, + 0, + -3.550295624100921 + ], + [ + 0, + 0, + 0.7036074465845424, + -2.2737885348813887 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "ZRotationHscaleZbaseline": [ + [ + 0.053926953353156136, + 0.03582167951143748, + 0, + 0 + ], + [ + -0.03582167951143748, + 0.053926953353156136, + 0, + 0 + ], + [ + 0, + 0, + 1, + -3.2332029498017487 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "RotationTranslation3D": [ + [ + 0.8100811050998874, + -0.3030939677547838, + 0.5018990435045756, + -2.8734904692082317 + ], + [ + 0.4979757275740692, + 0.8075341522117618, + -0.3160834822617412, + -3.7722406528522203 + ], + [ + -0.3094976218119015, + 0.5059867979723319, + 0.8051015975456041, + -3.511977037527143 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "Rescale3D": [ + [ + 0.10754195315849033, + 0, + 0, + 0 + ], + [ + 0, + 0.012787625579547376, + 0, + 0 + ], + [ + 0, + 0, + 0.032798154583522594, + 0 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "TranslateOrigins": [ + [ + 1, + 0, + 0, + -3.0518410718922895 + ], + [ + 0, + 1, + 0, + -3.8667699341149695 + ], + [ + 0, + 0, + 1, + -3.2332029498017487 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "ShearYZ": [ + [ + 1, + 0, + 0, + 0 + ], + [ + -0.6692050417082805, + 1, + 0, + 0 + ], + [ + 0.35315324212931615, + -0.06394207042397665, + 1, + 0 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "RotationTranslationXY": [ + [ + 0.8946494196067308, + -0.44676886193795917, + 0, + -0.9794537295011263 + ], + [ + 0.4467688619379593, + 0.894649419606731, + 0, + -4.958977628364134 + ], + [ + 0, + 0, + 1, + -3.233202949801749 + ], + [ + 0, + 0, + 0, + 1 + ] + ], + "QRFactorization": [ + [ + 0.7380347209075413, + -0.2626732024800775, + 0, + -1.1628964087154696 + ], + [ + 0.6747627366229761, + 1.1147956698369132, + 0, + -6.5703958608516055 + ], + [ + 0, + 0, + 1, + -3.233202949801749 + ], + [ + 0, + 0, + 0, + 1 + ] + ] + } +} \ No newline at end of file diff --git a/geomagio/adjusted/AdjustedMatrix.py b/geomagio/adjusted/AdjustedMatrix.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b8dcc649c14a3008389c0438b49edf5400c0eb --- /dev/null +++ b/geomagio/adjusted/AdjustedMatrix.py @@ -0,0 +1,30 @@ +from obspy import UTCDateTime +from pydantic import BaseModel +from typing import ( + Any, + List, + Optional, +) + +from .. import pydantic_utcdatetime +from .Metric import Metric + + +class AdjustedMatrix(BaseModel): + """Attributes pertaining to adjusted(affine) matrices, applied by the AdjustedAlgorithm + + Attributes + ---------- + matrix: affine matrix generated by Affine's calculate method + pier_correction: pier correction generated by Affine's calculate method + starttime: beginning of interval that matrix is valid for + endtime: end of interval that matrix is valid for + NOTE: valid intervals are only generated when bad data is encountered. + Matrix is non-constrained otherwise + """ + + matrix: Any + pier_correction: float + metrics: Optional[List[Metric]] = None + starttime: Optional[UTCDateTime] = None + endtime: Optional[UTCDateTime] = None diff --git a/geomagio/adjusted/Affine.py b/geomagio/adjusted/Affine.py new file mode 100644 index 0000000000000000000000000000000000000000..3189ef60b95fb241da3550700cdee756744eb2e0 --- /dev/null +++ b/geomagio/adjusted/Affine.py @@ -0,0 +1,413 @@ +from functools import reduce +import numpy as np +from obspy import UTCDateTime +from pydantic import BaseModel +from typing import List, Optional, Tuple + +from .AdjustedMatrix import AdjustedMatrix +from .. import ChannelConverter +from .. import pydantic_utcdatetime +from .Metric import Metric +from ..residual import Reading +from .Transform import Transform, TranslateOrigins, RotationTranslationXY + + +def weighted_quartile(data: List[float], weights: List[float], quant: float) -> float: + """Get weighted quartile to determine statistically good/bad data + + Attributes + ---------- + data: filtered array of observations + weights: array of vector distances/metrics + quant: statistical percentile of input data + """ + # sort data and weights + ind_sorted = np.argsort(data) + sorted_data = data[ind_sorted] + sorted_weights = weights[ind_sorted] + # compute auxiliary arrays + Sn = np.cumsum(sorted_weights) + Pn = (Sn - 0.5 * sorted_weights) / Sn[-1] + # interpolate to weighted quantile + return np.interp(quant, Pn, sorted_data) + + +def filter_iqr( + series: List[float], threshold: int = 6, weights: List[int] = None +) -> List[int]: + """ + Identify "good" elements in series by calculating potentially weighted + 25%, 50% (median), and 75% quantiles of series, the number of 25%-50% + quantile ranges below, or 50%-75% quantile ranges above each value of + series falls from the median, and finally, setting elements of good to + True that fall within these multiples of quantile ranges. + + NOTE: NumPy has a percentile function, but it does not yet handle + weights. This algorithm was adapted from the PyPI + package wquantiles (https://pypi.org/project/wquantiles/) + + Inputs: + series: array of observations to filter + + Options: + threshold: threshold in fractional number of 25%-50% (50%-75%) + quantile ranges below (above) the median each element of + series may fall and still be considered "good" + Default set to 6. + weights: weights to assign to each element of series. Default set to 1. + + Output: + good: Boolean array where True values correspond to "good" data + + """ + + if weights is None: + weights = np.ones_like(series) + + # initialize good as all True for weights greater than 0 + good = (weights > 0).astype(bool) + if np.size(good) <= 1: + # if a singleton is passed, assume it is "good" + return good + + good_old = ~good + while not (good_old == good).all(): + good_old = good + + wq25 = weighted_quartile(series[good], weights[good], 0.25) + wq50 = weighted_quartile(series[good], weights[good], 0.50) + wq75 = weighted_quartile(series[good], weights[good], 0.75) + + # NOTE: it is necessary to include good on the RHS here + # to prevent oscillation between two equally likely + # "optimal" solutions; this is a common problem with + # expectation maximization algorithms + good = ( + good + & (series >= (wq50 - threshold * (wq50 - wq25))) + & (series <= (wq50 + threshold * (wq75 - wq50))) + ) + + return good + + +class Affine(BaseModel): + """Creates AdjustedMatrix objects from readings + + Attributes + ---------- + observatory: 3-letter observatory code + starttime: beginning time for matrix creation + endtime: end time for matrix creation + acausal: when True, utilizes readings from after set endtime + update_interval: window of time a matrix is representative of + transforms: list of methods for matrix calculations + """ + + observatory: str = None + starttime: UTCDateTime = UTCDateTime() - (86400 * 7) + endtime: UTCDateTime = UTCDateTime() + acausal: bool = False + update_interval: Optional[int] = 86400 * 7 + transforms: List[Transform] = [ + RotationTranslationXY(memory=(86400 * 100)), + TranslateOrigins(memory=(86400 * 10)), + ] + + class Config: + arbitrary_types_allowed = True + + def calculate(self, readings: List[Reading]) -> List[AdjustedMatrix]: + """Calculates affine matrices for a range of times + + Attributes + ---------- + readings: list of readings containing absolutes + + Outputs + ------- + Ms: list of AdjustedMatrix objects created from calculations + """ + # default set to create one matrix between starttime and endtime + update_interval = self.update_interval or (self.endtime - self.starttime) + all_readings = [r for r in readings if r.valid] + Ms = [] + time = self.starttime + epoch_start = None + epoch_end = None + # search for "bad" H values + epochs = [r.time for r in all_readings if r.get_absolute("H").absolute == 0] + while time < self.endtime: + # update epochs for current time + epoch_start, epoch_end = self.get_epochs( + epoch_start=epoch_start, epoch_end=epoch_end, epochs=epochs, time=time + ) + # utilize readings that occur after or before a bad reading + readings = [ + r + for r in all_readings + if (epoch_start is None or r.time > epoch_start) + or (epoch_end is None or r.time < epoch_end) + ] + M = self.calculate_matrix(time, readings) + # if readings are trimmed by bad data, mark the vakid interval + M.starttime = epoch_start + M.endtime = epoch_end + time += update_interval + + Ms.append(M) + + return Ms + + def get_epochs( + self, + epoch_start: float, + epoch_end: float, + epochs: List[float], + time: UTCDateTime, + ) -> Tuple[float, float]: + """Updates valid start/end time for a given interval + + Attributes + ---------- + epoch_start: float value signifying start of last valid interval + epoch_end: float value signifying end of last valid interval + epochs: list of floats signifying bad data times + time: current time epoch is being evaluated at + + Outputs + ------- + epoch_start: float value signifying start of current valid interval + epoch_end: float value signifying end of current valid interval + """ + for e in epochs: + if e > time: + if epoch_end is None or e < epoch_end: + epoch_end = e + if e < time: + if epoch_start is None or e > epoch_start: + epoch_start = e + return epoch_start, epoch_end + + def valid_reading(self, reading: Reading) -> bool: + """ensures that readings used in calculations have been marked as valid + + Attributes + ---------- + readings: list containing valid and invalid readings + """ + if ( + reading.get_absolute("D").valid == True + and reading.get_absolute("H").valid == True + and reading.get_absolute("Z").valid == True + ): + return True + + def get_times(self, readings: List[UTCDateTime]) -> np.array: + """Returns times from H absolutes""" + return np.array([reading.get_absolute("H").endtime for reading in readings]) + + def get_ordinates( + self, readings: List[Reading] + ) -> Tuple[List[float], List[float], List[float]]: + """Calculates ordinates from absolutes and baselines""" + h_abs, d_abs, z_abs = self.get_absolutes(readings) + h_bas, d_bas, z_bas = self.get_baselines(readings) + + # recreate ordinate variometer measurements from absolutes and baselines + h_ord = h_abs - h_bas + d_ord = d_abs - d_bas + z_ord = z_abs - z_bas + + # WebAbsolutes defines/generates h differently than USGS residual + # method spreadsheets. The following should ensure that ordinate + # values are converted back to their original raw measurements: + e_o = h_abs * d_ord * 60 / 3437.7468 + # TODO: is this handled in residual package? + if self.observatory in ["DED", "CMO"]: + h_o = np.sqrt(h_ord ** 2 - e_o ** 2) + else: + h_o = h_ord + z_o = z_ord + return (h_o, e_o, z_o) + + def get_baselines( + self, readings: List[Reading] + ) -> Tuple[List[float], List[float], List[float]]: + """Get H, D and Z baselines""" + h_bas = np.array([reading.get_absolute("H").baseline for reading in readings]) + d_bas = np.array([reading.get_absolute("D").baseline for reading in readings]) + z_bas = np.array([reading.get_absolute("Z").baseline for reading in readings]) + return (h_bas, d_bas, z_bas) + + def get_absolutes( + self, readings: List[Reading] + ) -> Tuple[List[float], List[float], List[float]]: + """Get H, D and Z absolutes""" + h_abs = np.array([reading.get_absolute("H").absolute for reading in readings]) + d_abs = np.array([reading.get_absolute("D").absolute for reading in readings]) + z_abs = np.array([reading.get_absolute("Z").absolute for reading in readings]) + + return (h_abs, d_abs, z_abs) + + def get_absolutes_xyz( + self, readings: List[Reading] + ) -> Tuple[List[float], List[float], List[float]]: + """Get X, Y and Z absolutes from H, D and Z baselines""" + h_abs, d_abs, z_abs = self.get_absolutes(readings) + + # convert from cylindrical to Cartesian coordinates + x_a = h_abs * np.cos(d_abs * np.pi / 180) + y_a = h_abs * np.sin(d_abs * np.pi / 180) + z_a = z_abs + return (x_a, y_a, z_a) + + def calculate_matrix( + self, time: UTCDateTime, readings: List[Reading] + ) -> AdjustedMatrix: + """Calculates affine matrix for a given time + + Attributes + ---------- + time: time within calculation interval + readings: list of valid readings + + Outputs + ------- + AdjustedMatrix object containing result + """ + absolutes = self.get_absolutes_xyz(readings) + baselines = self.get_baselines(readings) + ordinates = self.get_ordinates(readings) + times = self.get_times(readings) + Ms = [] + weights = [] + inputs = ordinates + + for transform in self.transforms: + weights = self.get_weights( + time=time, + times=times, + transform=transform, + ) + # return NaNs if no valid observations + if np.sum(weights) == 0: + return AdjustedMatrix( + matrix=np.nan * np.ones((4, 4)), + pier_correction=np.nan, + ) + # zero out statistically 'bad' baselines + weights = self.weight_baselines(baselines=baselines, weights=weights) + + M = transform.calculate( + ordinates=inputs, absolutes=absolutes, weights=weights + ) + + # apply latest M matrix to inputs to get intermediate inputs + inputs = np.dot( + M, np.vstack([inputs[0], inputs[1], inputs[2], np.ones_like(inputs[0])]) + )[0:3] + Ms.append(M) + + # compose affine transform matrices using reverse ordered matrices + M_composed = reduce(np.dot, np.flipud(Ms)) + pier_correction = np.average( + [reading.pier_correction for reading in readings], weights=weights + ) + metrics = self.get_metrics( + absolutes=absolutes, ordinates=ordinates, matrix=M_composed + ) + return AdjustedMatrix( + matrix=M_composed, + pier_correction=pier_correction, + metrics=metrics, + ) + + def get_metrics( + self, absolutes: List[float], ordinates: List[float], matrix: List[float] + ) -> Tuple[List[float], List[float], float, float]: + """Computes mean absolute error and standard deviation for X, Y, Z, and dF between expected and predicted values. + + Attributes + ---------- + absolutes: X, Y and Z absolutes + ordinates: H, E and Z ordinates + matrix: composed matrix + + Outputs + ------- + metrics: list of Metric objects + """ + ordinates = np.vstack((ordinates, np.ones_like(ordinates[0]))) + predicted = matrix @ ordinates + + channels = ["X", "Y", "Z"] + metrics = [] + for i in range(len(channels)): + metric = Metric(element=channels[i]) + metric.calculate(expected=absolutes[i], predicted=predicted[i]) + metrics.append(metric) + + expected_f = ChannelConverter.get_computed_f_using_squares( + absolutes[0], absolutes[1], absolutes[2] + ) + predicted_f = ChannelConverter.get_computed_f_using_squares( + predicted[0], predicted[1], predicted[2] + ) + df = ChannelConverter.get_deltaf(expected_f, predicted_f) + metrics.append(self.get_metrics_df(predicted=predicted, expected=absolutes)) + return metrics + + def get_metrics_df(self, predicted, expected): + """Computes mean absolute error and standard deviation for dF between expected and predicted values""" + expected_f = ChannelConverter.get_computed_f_using_squares( + expected[0], expected[1], expected[2] + ) + predicted_f = ChannelConverter.get_computed_f_using_squares( + predicted[0], predicted[1], predicted[2] + ) + df = ChannelConverter.get_deltaf(expected_f, predicted_f) + return Metric( + element="dF", + mae=abs(np.nanmean(df)), + std=np.nanstd(df), + ) + + def get_weights( + self, + time: UTCDateTime, + times: List[UTCDateTime], + transform: Transform, + ) -> np.array: + """ + + Attributes + ---------- + time: time within calculation interval + times: times of valid readings + transform: matrix calculation method + + Outputs + ------- + weights: array of weights to apply to absolutes/ordinates within calculations + """ + + weights = transform.get_weights(time=time.timestamp, times=times) + # set weights for future observations to zero if not acausal + if not self.acausal: + weights[times > time.timestamp] = 0.0 + return weights + + def weight_baselines( + self, + baselines: List[float], + weights: List[float], + threshold=3, + ) -> List[float]: + """Filters "bad" weights generated by unreliable readings""" + good = ( + filter_iqr(baselines[0], threshold=threshold, weights=weights) + & filter_iqr(baselines[1], threshold=threshold, weights=weights) + & filter_iqr(baselines[2], threshold=threshold, weights=weights) + ) + return weights * good diff --git a/geomagio/adjusted/Metric.py b/geomagio/adjusted/Metric.py new file mode 100644 index 0000000000000000000000000000000000000000..ee946bb5c88ce17e6d60817b40056592f77dce77 --- /dev/null +++ b/geomagio/adjusted/Metric.py @@ -0,0 +1,22 @@ +import numpy as np +from pydantic import BaseModel + + +class Metric(BaseModel): + """Mean absolute error and standard deviation for a given element + + Attributes + ---------- + element: Channel that metrics are representative of + mae: mean absolute error + std: standard deviation + """ + + element: str + mae: float = None + std: float = None + + def calculate(self, expected, predicted): + """Calculates mean absolute error and standard deviation between expected and predicted data""" + self.mae = abs(np.nanmean(expected - predicted)) + self.std = np.nanstd(expected - predicted) diff --git a/geomagio/adjusted/SpreadsheetSummaryFactory.py b/geomagio/adjusted/SpreadsheetSummaryFactory.py new file mode 100644 index 0000000000000000000000000000000000000000..dbcd1e5b9e5d2bc8d0014558840c5d4bcc3e7aeb --- /dev/null +++ b/geomagio/adjusted/SpreadsheetSummaryFactory.py @@ -0,0 +1,143 @@ +import os + +from obspy import UTCDateTime +import openpyxl +from typing import List + +from ..residual import Absolute, Angle, Reading +from ..residual.SpreadsheetAbsolutesFactory import parse_relative_time + + +class SpreadsheetSummaryFactory(object): + """Read absolutes from summary spreadsheets""" + + def __init__( + self, base_directory: str = r"/Volumes/geomag/pub/Caldata/Checked Baseline Data" + ): + self.base_directory = base_directory + + def get_readings( + self, observatory: str, starttime: UTCDateTime, endtime: UTCDateTime + ) -> List[Reading]: + """Gathers readings from factory's base directory + + Attributes + ---------- + observatory: 3-letter observatory code + starttime: beginning date of readings + endtime: end date of readings + """ + readings = [] + start_filename = f"{observatory}{starttime.datetime:%Y%j%H%M}.xlsm" + end_filename = f"{observatory}{endtime.datetime:%Y%j%H%M}.xlsm" + for year in range(starttime.year, endtime.year + 1): + observatory_directory = os.path.join( + self.base_directory, observatory, f"{year}" + ) + for (dirpath, _, filenames) in os.walk(observatory_directory): + filenames.sort() + for filename in filenames: + if ".xlsm" not in filename: + continue + if start_filename <= filename < end_filename: + rs = self.parse_spreadsheet( + os.path.join(dirpath, filename), + ) + for r in rs: + readings.append(r) + return readings + + def parse_spreadsheet(self, path: str) -> List[Reading]: + sheet = openpyxl.load_workbook(path, data_only=True)["Sheet1"] + readings = self._parse_readings(sheet, path) + return readings + + def _parse_metadata(self, sheet: openpyxl.worksheet, observatory: str) -> dict: + """gather metadata from spreadsheet + + Attributes + ---------- + sheet: excel sheet containing residual summary values + observatory: 3-letter observatory code + """ + date = sheet["I1"].value + date = f"{date.year}{date.month:02}{date.day:02}" + return { + "observatory": observatory, + "pier_correction": sheet["C5"].value, + "instrument": sheet["B3"].value, + "date": date, + "observer": sheet["I10"].value, + } + + def _parse_readings(self, sheet: openpyxl.worksheet, path: str) -> List[Reading]: + """get list of readings from spreadsheet + + Attributes + ---------- + sheet: excel sheet containing residual summary values + path: spreadsheet's filepath + + Outputs + ------- + List of valid readings from spreadsheet. + If all readings are valid, 4 readings are returned + """ + metadata = self._parse_metadata(sheet, path.split("/")[-1][0:3]) + date = sheet["I1"].value + base_date = f"{date.year}{date.month:02}{date.day:02}" + readings = [] + for d_n in range(10, 14): + h_n = d_n + 14 + v_n = d_n + 28 + absolutes = [ + Absolute( + element="D", + absolute=Angle.from_dms( + degrees=sheet[f"C{d_n}"].value, minutes=sheet[f"D{d_n}"].value + ), + baseline=sheet[f"H{d_n}"].value / 60, + starttime=parse_relative_time( + base_date, "{0:04d}".format(sheet[f"B{v_n}"].value) + ), + endtime=parse_relative_time( + base_date, "{0:04d}".format(sheet[f"B{d_n}"].value) + ), + ), + Absolute( + element="H", + absolute=sheet[f"D{h_n}"].value, + baseline=sheet[f"H{h_n}"].value, + starttime=parse_relative_time( + base_date, "{0:04d}".format(sheet[f"B{v_n}"].value) + ), + endtime=parse_relative_time( + base_date, "{0:04d}".format(sheet[f"B{h_n}"].value) + ), + ), + Absolute( + element="Z", + absolute=sheet[f"D{v_n}"].value, + baseline=sheet[f"H{v_n}"].value, + starttime=parse_relative_time( + base_date, "{0:04d}".format(sheet[f"B{v_n}"].value) + ), + endtime=parse_relative_time( + base_date, "{0:04d}".format(sheet[f"B{v_n}"].value) + ), + ), + ] + valid = [ + sheet[f"J{d_n}"].value, + sheet[f"J{h_n}"].value, + sheet[f"J{d_n}"].value, + ] + if valid == [None, None, None]: + readings.append( + Reading( + metadata=metadata, + absolutes=absolutes, + pier_correction=metadata["pier_correction"], + ), + ) + return readings diff --git a/geomagio/adjusted/Transform.py b/geomagio/adjusted/Transform.py new file mode 100644 index 0000000000000000000000000000000000000000..aeb33131a0d47b2d8eee290c1af4e4a505d98821 --- /dev/null +++ b/geomagio/adjusted/Transform.py @@ -0,0 +1,665 @@ +import numpy as np +from obspy import UTCDateTime +from pydantic import BaseModel +import scipy.linalg as spl +from typing import List, Tuple, Optional + + +class Transform(BaseModel): + """Method for generating an affine matrix. + + Attributes + ---------- + memory: Controls impacts of measurements from the past. + Defaults to infinite(equal weighting) + """ + + memory: Optional[float] = None + + def get_weights(self, times: UTCDateTime, time: int = None) -> List[float]: + """ + Calculate time-dependent weights according to exponential decay. + + Inputs: + times: array of times, or any time-like index whose relative values represent spacing between events + Output: + weights: array of vector distances/metrics + """ + + # convert to array of floats + times = np.asarray(times).astype(float) + + if time is None: + time = float(max(times)) + + # if memory is actually infinite, return equal weights + if np.isinf(self.memory): + return np.ones(times.shape) + + # initialize weights + weights = np.zeros(times.shape) + + # calculate exponential decay time-dependent weights + weights[times <= time] = np.exp((times[times <= time] - time) / self.memory) + weights[times >= time] = np.exp((time - times[times >= time]) / self.memory) + + return weights + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + """Type skeleton inherited by any instance of Transform + + Attributes + ---------- + ordinates: H, E and Z ordinates + absolutes: X, Y and Z absolutes(NOTE: absolutes must be rotated from original H, E and Z values) + weights: time weights to apply during calculations of matrices + """ + return + + +class LeastSq(Transform): + """Intance of Transform. Applies least squares to generate matrices""" + + def get_stacked_absolutes(self, absolutes): + """Formats absolutes for least squares method + + Attributes + ---------- + absolutes: Rotated X, Y, and Z absolutes + + Output + ------ + X, Y and Z absolutes placed end to end and transposed + """ + return np.vstack([absolutes[0], absolutes[1], absolutes[2]]).T.ravel() + + def get_weighted_values( + self, + values: Tuple[List[float], List[float], List[float]], + weights: Optional[List[float]], + ) -> Tuple[List[float], List[float], List[float]]: + """Application of weights for least squares methods, which calls for square rooting of weights + + Attributes + ---------- + values: absolutes or ordinates + + Outputs + ------- + tuple of weights applied to each element of values + + """ + if weights is None: + return values + weights = np.sqrt(weights) + weights = np.vstack((weights, weights, weights)).T.ravel() + return values * weights + + +class SVD(Transform): + """Instance of Transform. Applies singular value decomposition to generate matrices""" + + def get_stacked_values(self, values, weighted_values, ndims=3) -> np.array: + """Supports intermediate mathematical steps by differencing and shaping values for SVD + + Attributes + ---------- + values: absolutes or ordinates + weighted_values: absolutes or ordinates with weights applied + ndims: number of rows desired in return array(case specific). Default set to 3 dimensions(XYZ/HEZ) + + Outputs + ------- + Stacked and differenced values from their weighted counterparts + """ + return np.vstack([[values[i] - weighted_values[i]] for i in range(ndims)]) + + def get_weighted_values( + self, + values: Tuple[List[float], List[float], List[float]], + weights: Optional[List[float]], + ) -> Tuple[List[float], List[float], List[float]]: + """Application of weights for SVD methods, which call for weighted averages""" + if weights is None: + weights = np.ones_like(values[0]) + return ( + np.average(values[0], weights=weights), + np.average(values[1], weights=weights), + np.average(values[2], weights=weights), + ) + + +class NoConstraints(LeastSq): + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + """Calculates affine with no constraints using least squares.""" + # LHS, or dependent variables + # + # [A[0,0], A[1,0], A[2,0], A[0,1], A[1,1], A[2,1], ...] + abs_stacked = self.get_stacked_absolutes(absolutes) + # return generate_affine_0(ord_hez, abs_xyz, weights) + # RHS, or independent variables + # (reduces degrees of freedom by 4: + # - 4 for the last row of zeros and a one) + # [ + # [o[0,0], 0, 0, o[0,1], 0, 0, ...], + # [0, o[1,0], 0, 0, o[1,1], 0, ...], + # [0, 0, o[2,0], 0, 0, o[2,1], ...], + # ... + # ] + ord_stacked = np.zeros((12, len(ordinates[0]) * 3)) + ord_stacked[0, 0::3] = ordinates[0] + ord_stacked[1, 0::3] = ordinates[1] + ord_stacked[2, 0::3] = ordinates[2] + ord_stacked[3, 0::3] = 1.0 + ord_stacked[4, 1::3] = ordinates[0] + ord_stacked[5, 1::3] = ordinates[1] + ord_stacked[6, 1::3] = ordinates[2] + ord_stacked[7, 1::3] = 1.0 + ord_stacked[8, 2::3] = ordinates[0] + ord_stacked[9, 2::3] = ordinates[1] + ord_stacked[10, 2::3] = ordinates[2] + ord_stacked[11, 2::3] = 1.0 + + ord_stacked = self.get_weighted_values(ord_stacked, weights) + abs_stacked = self.get_weighted_values(abs_stacked, weights) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + return np.array( + [ + [M_r[0], M_r[1], M_r[2], M_r[3]], + [M_r[4], M_r[5], M_r[6], M_r[7]], + [M_r[8], M_r[9], M_r[10], M_r[11]], + [0.0, 0.0, 0.0, 1.0], + ] + ) + + +class ZRotationShear(LeastSq): + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + """Calculates affine using least squares, constrained to rotate about the Z axis.""" + # LHS, or dependent variables + # + abs_stacked = self.get_stacked_absolutes(absolutes) + # return generate_affine_1(ord_hez, abs_xyz, weights) + # RHS, or independent variables + # (reduces degrees of freedom by 8: + # - 2 for making x,y independent of z; + # - 2 for making z independent of x,y + # - 4 for the last row of zeros and a one) + ord_stacked = np.zeros((8, len(ordinates[0]) * 3)) + ord_stacked[0, 0::3] = ordinates[0] + ord_stacked[1, 0::3] = ordinates[1] + ord_stacked[2, 0::3] = 1.0 + ord_stacked[3, 1::3] = ordinates[0] + ord_stacked[4, 1::3] = ordinates[1] + ord_stacked[5, 1::3] = 1.0 + ord_stacked[6, 2::3] = ordinates[2] + ord_stacked[7, 2::3] = 1.0 + + ord_stacked = self.get_weighted_values(ord_stacked, weights) + abs_stacked = self.get_weighted_values(abs_stacked, weights) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + return [ + [M_r[0], M_r[1], 0.0, M_r[2]], + [M_r[3], M_r[4], 0.0, M_r[5]], + [0.0, 0.0, M_r[6], M_r[7]], + [0.0, 0.0, 0.0, 1.0], + ] + + +class ZRotationHscale(LeastSq): + """Calculates affine using least squares, constrained to rotate about the Z axis + and apply uniform horizontal scaling.""" + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + # LHS, or dependent variables + # + abs_stacked = self.get_stacked_absolutes(absolutes) + # RHS, or independent variables + # (reduces degrees of freedom by 10: + # - 2 for making x,y independent of z; + # - 2 for making z independent of x,y + # - 2 for not allowing shear in x,y; and + # - 4 for the last row of zeros and a one) + ord_stacked = np.zeros((6, len(ordinates[0]) * 3)) + ord_stacked[0, 0::3] = ordinates[0] + ord_stacked[0, 1::3] = ordinates[1] + ord_stacked[1, 0::3] = ordinates[1] + ord_stacked[1, 1::3] = -ordinates[0] + ord_stacked[2, 0::3] = 1.0 + ord_stacked[3, 1::3] = 1.0 + ord_stacked[4, 2::3] = ordinates[2] + ord_stacked[5, 2::3] = 1.0 + + ord_stacked = self.get_weighted_values(ord_stacked, weights) + abs_stacked = self.get_weighted_values(abs_stacked, weights) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + return [ + [M_r[0], M_r[1], 0.0, M_r[2]], + [-M_r[1], M_r[0], 0.0, M_r[3]], + [0.0, 0.0, M_r[4], M_r[5]], + [0.0, 0.0, 0.0, 1.0], + ] + + +class ZRotationHscaleZbaseline(LeastSq): + """Calculates affine using least squares, constrained to rotate about the Z axis, + apply a uniform horizontal scaling, and apply a baseline shift for the Z axis.""" + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + # RHS, or independent variables + # (reduces degrees of freedom by 13: + # - 2 for making x,y independent of z; + # - 2 for making z independent of x,y; + # - 2 for not allowing shear in x,y; + # - 2 for not allowing translation in x,y; + # - 1 for not allowing scaling in z; and + # - 4 for the last row of zeros and a one) + ord_stacked = np.zeros((3, len(ordinates[0]) * 3)) + ord_stacked[0, 0::3] = ordinates[0] + ord_stacked[0, 1::3] = ordinates[1] + ord_stacked[1, 0::3] = ordinates[1] + ord_stacked[1, 1::3] = -ordinates[0] + ord_stacked[2, 2::3] = 1.0 + + # subtract z_o from z_a to force simple z translation + abs_stacked = self.get_stacked_absolutes(absolutes) + abs_stacked[2::3] = absolutes[2] - ordinates[2] + + abs_stacked = self.get_weighted_values(values=abs_stacked, weights=weights) + ord_stacked = self.get_weighted_values(values=ord_stacked, weights=weights) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + return [ + [M_r[0], M_r[1], 0.0, 0.0], + [-M_r[1], M_r[0], 0.0, 0.0], + [0.0, 0.0, 1.0, M_r[2]], + [0.0, 0.0, 0.0, 1.0], + ] + + +class RotationTranslation3D(SVD): + """Calculates affine using using singular value decomposition, + constrained to 3D scaled rotation and translation(no shear)""" + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + weighted_ordinates = self.get_weighted_values(values=ordinates, weights=weights) + weighted_absolutes = self.get_weighted_values(values=absolutes, weights=weights) + # generate weighted "covariance" matrix + H = np.dot( + self.get_stacked_values( + values=ordinates, + weighted_values=weighted_ordinates, + ndims=3, + ), + np.dot( + np.diag(weights), + self.get_stacked_values( + values=absolutes, + weighted_values=weighted_absolutes, + ndims=3, + ).T, + ), + ) + # Singular value decomposition, then rotation matrix from L&R eigenvectors + # (the determinant guarantees a rotation, and not a reflection) + U, S, Vh = np.linalg.svd(H) + if np.sum(S) < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + R = np.dot(Vh.T, np.dot(np.diag([1, 1, np.linalg.det(np.dot(Vh.T, U.T))]), U.T)) + + # now get translation using weighted centroids and R + T = np.array( + [weighted_absolutes[0], weighted_absolutes[1], weighted_absolutes[2]] + ) - np.dot( + R, [weighted_ordinates[0], weighted_ordinates[1], weighted_ordinates[2]] + ) + + return [ + [R[0, 0], R[0, 1], R[0, 2], T[0]], + [R[1, 0], R[1, 1], R[1, 2], T[1]], + [R[2, 0], R[2, 1], R[2, 2], T[2]], + [0.0, 0.0, 0.0, 1.0], + ] + + +class Rescale3D(LeastSq): + """Calculates affine using using least squares, constrained to re-scale each axis""" + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + abs_stacked = self.get_stacked_absolutes(absolutes=absolutes) + # RHS, or independent variables + # (reduces degrees of freedom by 13: + # - 2 for making x independent of y,z; + # - 2 for making y,z independent of x; + # - 1 for making y independent of z; + # - 1 for making z independent of y; + # - 3 for not translating xyz + # - 4 for the last row of zeros and a one) + ord_stacked = np.zeros((3, len(ordinates[0]) * 3)) + ord_stacked[0, 0::3] = ordinates[0] + ord_stacked[1, 1::3] = ordinates[1] + ord_stacked[2, 2::3] = ordinates[2] + + abs_stacked = self.get_weighted_values(values=abs_stacked, weights=weights) + ord_stacked = self.get_weighted_values(values=ord_stacked, weights=weights) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + return [ + [M_r[0], 0.0, 0.0, 0.0], + [0.0, M_r[1], 0.0, 0.0], + [0.0, 0.0, M_r[2], 0.0], + [0.0, 0.0, 0.0, 1.0], + ] + + +class TranslateOrigins(LeastSq): + """Calculates affine using using least squares, constrained to tanslate origins""" + + def get_weighted_values( + self, + values: Tuple[List[float], List[float], List[float]], + weights: Optional[List[float]], + ): + """Weights are applied after matrix creation steps, + requiring weights to be stacked similar to ordinates and absolutes""" + if weights is not None: + weights = np.sqrt(weights) + weights = np.vstack((weights, weights, weights)).T.ravel() + else: + weights = 1 + return values * weights + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + + abs_stacked = self.get_stacked_absolutes(absolutes) + ord_stacked = np.zeros((3, len(ordinates[0]) * 3)) + + ord_stacked[0, 0::3] = 1.0 + ord_stacked[1, 1::3] = 1.0 + ord_stacked[2, 2::3] = 1.0 + + # subtract ords from abs to force simple translation + abs_stacked[0::3] = absolutes[0] - ordinates[0] + abs_stacked[1::3] = absolutes[1] - ordinates[1] + abs_stacked[2::3] = absolutes[2] - ordinates[2] + + # apply weights + ord_stacked = self.get_weighted_values(values=ord_stacked, weights=weights) + abs_stacked = self.get_weighted_values(values=abs_stacked, weights=weights) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + return [ + [1.0, 0.0, 0.0, M_r[0]], + [0.0, 1.0, 0.0, M_r[1]], + [0.0, 0.0, 1.0, M_r[2]], + [0.0, 0.0, 0.0, 1.0], + ] + + +class ShearYZ(LeastSq): + """Calculates affine using least squares, constrained to shear y and z, but not x.""" + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + # RHS, or independent variables + # (reduces degrees of freedom by 13: + # - 2 for making x independent of y,z; + # - 1 for making y independent of z; + # - 3 for not scaling each axis + # - 4 for the last row of zeros and a one) + ord_stacked = np.zeros((3, len(ordinates[0]) * 3)) + ord_stacked[0, 0::3] = 1.0 + ord_stacked[1, 0::3] = ordinates[0] + ord_stacked[1, 1::3] = 1.0 + ord_stacked[2, 0::3] = ordinates[0] + ord_stacked[2, 1::3] = ordinates[1] + ord_stacked[2, 2::3] = 1.0 + abs_stacked = self.get_stacked_absolutes(absolutes=absolutes) + ord_stacked = self.get_weighted_values(values=ord_stacked, weights=weights) + abs_stacked = self.get_weighted_values(values=abs_stacked, weights=weights) + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 3: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + return [ + [1.0, 0.0, 0.0, 0.0], + [M_r[0], 1.0, 0.0, 0.0], + [M_r[1], M_r[2], 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ] + + +class RotationTranslationXY(SVD): + """Calculates affine using singular value decomposition, + constrained to rotation and translation in XY(no scale or shear), + and only translation in Z""" + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + if weights is None: + weights = np.ones_like(ordinates[0]) + weighted_ordinates = self.get_weighted_values(values=ordinates, weights=weights) + weighted_absolutes = self.get_weighted_values(values=absolutes, weights=weights) + # return generate_affine_8(ord_hez, abs_xyz, weights) + # generate weighted "covariance" matrix + H = np.dot( + self.get_stacked_values( + values=ordinates, + weighted_values=weighted_ordinates, + ndims=2, + ), + np.dot( + np.diag(weights), + self.get_stacked_values( + values=absolutes, + weighted_values=weighted_absolutes, + ndims=2, + ).T, + ), + ) + + # Singular value decomposition, then rotation matrix from L&R eigenvectors + # (the determinant guarantees a rotation, and not a reflection) + U, S, Vh = np.linalg.svd(H) + if np.sum(S) < 2: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + R = np.dot(Vh.T, np.dot(np.diag([1, np.linalg.det(np.dot(Vh.T, U.T))]), U.T)) + + # now get translation using weighted centroids and R + T = np.array([weighted_absolutes[0], weighted_absolutes[1]]) - np.dot( + R, [weighted_ordinates[0], weighted_ordinates[1]] + ) + + return [ + [R[0, 0], R[0, 1], 0.0, T[0]], + [R[1, 0], R[1, 1], 0.0, T[1]], + [ + 0.0, + 0.0, + 1.0, + np.array(weighted_absolutes[2]) - np.array(weighted_ordinates[2]), + ], + [0.0, 0.0, 0.0, 1.0], + ] + + +class QRFactorization(SVD): + """Calculates affine using singular value decomposition with QR factorization""" + + def get_weighted_values_lstsq( + self, + values: Tuple[List[float], List[float], List[float]], + weights: Optional[List[float]], + ): + """ Applies least squares weights in two dimensions(X and Y)""" + if weights is None: + return values + weights = np.sqrt(weights) + return np.array( + [ + values[0] * weights, + values[1] * weights, + ] + ) + + def calculate( + self, + ordinates: Tuple[List[float], List[float], List[float]], + absolutes: Tuple[List[float], List[float], List[float]], + weights: List[float], + ) -> np.array: + + if weights is None: + weights = np.ones_like(ordinates[0]) + + weighted_absolutes = self.get_weighted_values(values=absolutes, weights=weights) + weighted_ordinates = self.get_weighted_values(values=ordinates, weights=weights) + + # return generate_affine_9(ord_hez, abs_xyz, weights) + # LHS, or dependent variables + abs_stacked = self.get_stacked_values( + values=absolutes, + weighted_values=weighted_absolutes, + ndims=2, + ) + + # RHS, or independent variables + ord_stacked = self.get_stacked_values( + values=ordinates, + weighted_values=weighted_ordinates, + ndims=2, + ) + + abs_stacked = self.get_weighted_values_lstsq( + values=abs_stacked, + weights=weights, + ) + ord_stacked = self.get_weighted_values_lstsq( + values=ord_stacked, + weights=weights, + ) + + # regression matrix M that minimizes L2 norm + M_r, res, rank, sigma = spl.lstsq(ord_stacked.T, abs_stacked.T) + if rank < 2: + print("Poorly conditioned or singular matrix, returning NaNs") + return np.nan * np.ones((4, 4)) + + # QR fatorization + # NOTE: forcing the diagonal elements of Q to be positive + # ensures that the determinant is 1, not -1, and is + # therefore a rotation, not a reflection + Q, R = np.linalg.qr(M_r.T) + neg = np.diag(Q) < 0 + Q[:, neg] = -1 * Q[:, neg] + R[neg, :] = -1 * R[neg, :] + + # isolate scales from shear + S = np.diag(np.diag(R)) + H = np.dot(np.linalg.inv(S), R) + + # combine shear and rotation + QH = np.dot(Q, H) + + # now get translation using weighted centroids and R + T = np.array([weighted_absolutes[0], weighted_absolutes[1]]) - np.dot( + QH, [weighted_ordinates[0], weighted_ordinates[1]] + ) + + return [ + [QH[0, 0], QH[0, 1], 0.0, T[0]], + [QH[1, 0], QH[1, 1], 0.0, T[1]], + [ + 0.0, + 0.0, + 1.0, + np.array(weighted_absolutes[2]) - np.array(weighted_ordinates[2]), + ], + [0.0, 0.0, 0.0, 1.0], + ] diff --git a/geomagio/algorithm/AdjustedAlgorithm.py b/geomagio/algorithm/AdjustedAlgorithm.py index 601bad2444246efeee5b0f81f4551f245e3a9406..2a5b647e3f205721359cfdaf92dc2f3910395d82 100644 --- a/geomagio/algorithm/AdjustedAlgorithm.py +++ b/geomagio/algorithm/AdjustedAlgorithm.py @@ -1,23 +1,22 @@ -"""Algorithm that converts from one geomagnetic coordinate system to a - related geographic coordinate system, by using transformations generated - from absolute, baseline measurements. -""" -from __future__ import absolute_import +import sys -from .Algorithm import Algorithm import json import numpy as np from obspy.core import Stream, Stats -import sys + +from ..adjusted.AdjustedMatrix import AdjustedMatrix +from .Algorithm import Algorithm class AdjustedAlgorithm(Algorithm): - """Adjusted Data Algorithm""" + """Algorithm that converts from one geomagnetic coordinate system to a + related geographic coordinate system, by using transformations generated + from absolute, baseline measurements. + """ def __init__( self, - matrix=None, - pier_correction=None, + matrix: AdjustedMatrix = None, statefile=None, data_type=None, location=None, @@ -33,7 +32,6 @@ class AdjustedAlgorithm(Algorithm): ) # state variables self.matrix = matrix - self.pier_correction = pier_correction self.statefile = statefile self.data_type = data_type self.location = location @@ -45,12 +43,12 @@ class AdjustedAlgorithm(Algorithm): """Load algorithm state from a file. File name is self.statefile. """ - # Adjusted matrix defaults to identity matrix - matrix_size = len([c for c in self.get_input_channels() if c != "F"]) + 1 - self.matrix = np.eye(matrix_size) - self.pier_correction = 0 + pier_correction = 0 if self.statefile is None: return + # Adjusted matrix defaults to identity matrix + matrix_size = len([c for c in self.get_input_channels() if c != "F"]) + 1 + matrix = np.eye(matrix_size) data = None try: with open(self.statefile, "r") as f: @@ -60,10 +58,9 @@ class AdjustedAlgorithm(Algorithm): sys.stderr.write("I/O error {0}".format(err)) if data is None or data == "": return - for row in range(matrix_size): - for col in range(matrix_size): - self.matrix[row, col] = np.float64(data[f"M{row+1}{col+1}"]) - self.pier_correction = np.float64(data["PC"]) + self.matrix = AdjustedMatrix( + matrix=np.array(data["matrix"]), pier_correction=data["pier_correction"] + ) def save_state(self): """Save algorithm state to a file. @@ -71,12 +68,11 @@ class AdjustedAlgorithm(Algorithm): """ if self.statefile is None: return - data = {"PC": self.pier_correction} - length = len(self.matrix[0, :]) - for i in range(0, length): - for j in range(0, length): - key = "M" + str(i + 1) + str(j + 1) - data[key] = self.matrix[i, j] + data = { + "matrix": list(self.matrix.matrix), + "pier_correction": self.matrix.pier_correction, + } + with open(self.statefile, "w") as f: f.write(json.dumps(data)) @@ -134,7 +130,7 @@ class AdjustedAlgorithm(Algorithm): ] + [np.ones_like(stream[0].data)] ) - adjusted = np.matmul(self.matrix, raws) + adjusted = np.matmul(self.matrix.matrix, raws) out = Stream( [ self.create_trace( @@ -147,7 +143,7 @@ class AdjustedAlgorithm(Algorithm): ) if "F" in inchannels and "F" in outchannels: f = stream.select(channel="F")[0] - out += self.create_trace("F", f.stats, f.data + self.pier_correction) + out += self.create_trace("F", f.stats, f.data + self.matrix.pier_correction) return out def can_produce_data(self, starttime, endtime, stream): diff --git a/geomagio/processing/adjusted.py b/geomagio/processing/adjusted.py new file mode 100644 index 0000000000000000000000000000000000000000..03eab2ab98b23444f76b79aa8811e9c2cd281954 --- /dev/null +++ b/geomagio/processing/adjusted.py @@ -0,0 +1,64 @@ +import json +from obspy import UTCDateTime +from typing import Optional, Literal +import typer + +from ..adjusted.Affine import Affine +from ..adjusted.SpreadsheetSummaryFactory import SpreadsheetSummaryFactory +from ..residual import WebAbsolutesFactory + + +def main(): + typer.run(generate_matrix) + + +def generate_matrix( + observatory: str, + starttime: UTCDateTime, + endtime: UTCDateTime, + readings_starttime: UTCDateTime, + readings_endtime: UTCDateTime, + output_file: str, + input_factory: str = "webabsolutes", + acausal: bool = False, + spreadsheet_directory: Optional[ + str + ] = r"/Volumes/geomag/pub/Caldata/Checked Baseline Data", + webabsolutes_url: Optional[ + str + ] = "https://geomag.usgs.gov/baselines/observation.json.php", +): + if input_factory == "spreadsheet": + readings = SpreadsheetSummaryFactory( + base_directory=spreadsheet_directory + ).get_readings( + observatory=observatory, + starttime=readings_starttime, + endtime=readings_endtime, + ) + elif input_factory == "webabsolutes": + readings = WebAbsolutesFactory(url=webabsolutes_url).get_readings( + observatory=observatory, + starttime=readings_starttime, + endtime=readings_endtime, + ) + else: + readings = [] + + result = Affine( + observatory=observatory, + starttime=starttime, + endtime=endtime, + acausal=acausal, + ).calculate(readings=readings)[0] + + matrix = [ + list(result.matrix[0]), + list(result.matrix[1]), + list(result.matrix[2]), + list(result.matrix[3]), + ] + result.matrix = matrix + + with open(output_file, "w") as file: + json.dump(result.dict(), file) diff --git a/setup.py b/setup.py index a92451cdf5762c924da69c7c343f1b73a92efab8..8f1ff47f4d8290489c96edf7283b8e9c5dc3f873 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setuptools.setup( "console_scripts": [ "magproc-prepfiles=geomagio.processing.magproc:main", "filter-realtime=geomagio.processing.obsrio:main", + "generate-matrix=geomagio.processing.adjusted:main", ], }, ) diff --git a/test/adjusted_test/adjusted_test.py b/test/adjusted_test/adjusted_test.py new file mode 100644 index 0000000000000000000000000000000000000000..adcd74112dcf2f80618f7b0aeaaedc72e91dc23c --- /dev/null +++ b/test/adjusted_test/adjusted_test.py @@ -0,0 +1,503 @@ +import json +import numpy as np +from numpy.testing import assert_equal, assert_array_almost_equal, assert_array_equal +from obspy.core import UTCDateTime + +from geomagio.adjusted.SpreadsheetSummaryFactory import SpreadsheetSummaryFactory +from geomagio.adjusted.Affine import Affine +from geomagio.adjusted.Transform import ( + NoConstraints, + ZRotationShear, + ZRotationHscale, + ZRotationHscaleZbaseline, + RotationTranslation3D, + Rescale3D, + TranslateOrigins, + ShearYZ, + RotationTranslationXY, + QRFactorization, +) + +from geomagio.residual.WebAbsolutesFactory import WebAbsolutesFactory + + +def get_spreadsheet_directory_readings(path, observatory, starttime, endtime): + ssf = SpreadsheetSummaryFactory(base_directory=path) + readings = ssf.get_readings( + observatory=observatory, starttime=starttime, endtime=endtime + ) + return readings + + +def test_CMO_summaries(): + starttime = UTCDateTime("2015-04-01") + endtime = UTCDateTime("2015-06-15") + readings = get_spreadsheet_directory_readings( + path="etc/adjusted/Caldata", + observatory="CMO", + starttime=starttime, + endtime=endtime, + ) + for reading in readings: + assert_equal(reading.metadata["observatory"], "CMO") + assert_equal(reading.metadata["instrument"], 200803) + assert_equal(reading.pier_correction, 10.5) + assert_equal(len(readings), 26) + + assert readings[0].time > starttime + assert readings[-1].time < endtime + + +def get_sythetic_variables(): + with open("etc/adjusted/synthetic.json") as file: + data = json.load(file) + variables = data["variables"] + ordinates = np.array([variables["h_ord"], variables["e_ord"], variables["z_ord"]]) + absolutes = np.array([variables["x_abs"], variables["y_abs"], variables["z_abs"]]) + weights = np.arange(0, len(ordinates[0])) + return ordinates, absolutes, weights + + +def get_expected_synthetic_result(key): + with open("etc/adjusted/synthetic.json") as file: + expected = json.load(file) + return expected["results"][key] + + +def test_NoConstraints_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + NoConstraints().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("NoConstraints"), + decimal=3, + ) + + +def test_ZRotationShear_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + ZRotationShear().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("ZRotationShear"), + decimal=3, + ) + + +def test_ZRotationHscale_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + ZRotationHscale().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("ZRotationHscale"), + decimal=3, + ) + + +def test_ZRotationHscaleZbaseline_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + ZRotationHscaleZbaseline().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("ZRotationHscaleZbaseline"), + decimal=3, + ) + + +def test_RotationTranslation3D_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + RotationTranslation3D().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("RotationTranslation3D"), + decimal=3, + ) + + +def test_Rescale3D_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + Rescale3D().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("Rescale3D"), + decimal=3, + ) + + +def test_TranslateOrigins_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + TranslateOrigins().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("TranslateOrigins"), + decimal=3, + ) + + +def test_ShearYZ_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + ShearYZ().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("ShearYZ"), + decimal=3, + ) + + +def test_RotationTranslationXY_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + RotationTranslationXY().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("RotationTranslationXY"), + decimal=3, + ) + + +def test_QRFactorization_synthetic(): + ordinates, absolutes, weights = get_sythetic_variables() + assert_array_almost_equal( + QRFactorization().calculate( + ordinates=ordinates, + absolutes=absolutes, + weights=weights, + ), + get_expected_synthetic_result("QRFactorization"), + decimal=3, + ) + + +def format_result(result) -> dict: + Ms = [] + for M in result: + m = [] + for row in M: + m.append(list(row)) + Ms.append(m) + return Ms + + +def get_excpected_matrices(observatory, key): + with open(f"etc/adjusted/{observatory}_expected.json", "r") as file: + expected = json.load(file) + return expected[key] + + +def get_readings_BOU201911202001(): + readings = WebAbsolutesFactory().get_readings( + observatory="BOU", + starttime=UTCDateTime("2019-10-01T00:00:00Z"), + endtime=UTCDateTime("2020-02-29T23:59:00Z"), + ) + return readings + + +def test_BOU201911202001_short_causal(): + readings = get_readings_BOU201911202001() + + starttime = UTCDateTime("2019-11-01T00:00:00Z") + endtime = UTCDateTime("2020-01-31T23:59:00Z") + + update_interval = 86400 * 7 + + result = Affine( + observatory="BOU", + starttime=starttime, + endtime=endtime, + update_interval=update_interval, + ).calculate(readings=readings) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("BOU", "short_causal") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), ((endtime - starttime) // update_interval) + 1) + + +def test_BOU201911202001_short_acausal(): + readings = get_readings_BOU201911202001() + + starttime = UTCDateTime("2019-11-01T00:00:00Z") + endtime = UTCDateTime("2020-01-31T23:59:00Z") + + update_interval = 86400 * 7 + + result = Affine( + observatory="BOU", + starttime=starttime, + endtime=endtime, + update_interval=update_interval, + acausal=True, + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("BOU", "short_acausal") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), ((endtime - starttime) // update_interval) + 1) + + +def test_BOU201911202001_infinite_weekly(): + readings = get_readings_BOU201911202001() + + starttime = UTCDateTime("2019-11-01T00:00:00Z") + endtime = UTCDateTime("2020-01-31T23:59:00Z") + + update_interval = 86400 * 7 + + result = Affine( + observatory="BOU", + starttime=starttime, + endtime=endtime, + update_interval=update_interval, + acausal=True, + transforms=[ + RotationTranslationXY(memory=np.inf), + TranslateOrigins(memory=np.inf), + ], + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("BOU", "inf_weekly") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), ((endtime - starttime) // update_interval) + 1) + + +def test_BOU201911202001_infinite_one_interval(): + readings = get_readings_BOU201911202001() + result = Affine( + observatory="BOU", + starttime=UTCDateTime("2019-11-01T00:00:00Z"), + endtime=UTCDateTime("2020-01-31T23:59:00Z"), + acausal=True, + transforms=[ + RotationTranslationXY(memory=np.inf), + TranslateOrigins(memory=np.inf), + ], + update_interval=None, + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("BOU", "inf_one_interval") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), 1) + + +def test_BOU201911202001_invalid_readings(): + readings = [] + result = Affine( + observatory="BOU", + starttime=UTCDateTime("2019-11-01T00:00:00Z"), + endtime=UTCDateTime("2020-01-31T23:59:00Z"), + acausal=True, + transforms=[ + RotationTranslationXY(memory=np.inf), + TranslateOrigins(memory=np.inf), + ], + update_interval=None, + ).calculate(readings=readings,)[0] + assert_array_equal(result.matrix, np.nan * np.ones((4, 4))) + assert_equal(result.pier_correction, np.nan) + + +def test_CMO2015_causal(): + readings = get_spreadsheet_directory_readings( + observatory="CMO", + starttime=UTCDateTime("2015-01-01T00:00:00Z"), + endtime=UTCDateTime("2015-12-31T23:59:00Z"), + path="etc/adjusted/Caldata/", + ) + assert len(readings) == 146 + + starttime = UTCDateTime("2015-02-01T00:00:00Z") + endtime = UTCDateTime("2015-11-27T23:59:00Z") + + update_interval = 86400 * 7 + + result = Affine( + observatory="CMO", + starttime=starttime, + endtime=endtime, + update_interval=update_interval, + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("CMO", "short_causal") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), ((endtime - starttime) // update_interval) + 1) + + +def test_CMO2015_acausal(): + readings = get_spreadsheet_directory_readings( + observatory="CMO", + starttime=UTCDateTime("2015-01-01T00:00:00Z"), + endtime=UTCDateTime("2015-12-31T23:59:00Z"), + path="etc/adjusted/Caldata/", + ) + assert len(readings) == 146 + + starttime = UTCDateTime("2015-02-01T00:00:00Z") + endtime = UTCDateTime("2015-11-27T23:59:00Z") + + update_interval = 86400 * 7 + + result = Affine( + observatory="CMO", + starttime=starttime, + endtime=endtime, + update_interval=update_interval, + acausal=True, + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("CMO", "short_acausal") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), ((endtime - starttime) // update_interval) + 1) + + +def test_CMO2015_infinite_weekly(): + readings = get_spreadsheet_directory_readings( + observatory="CMO", + starttime=UTCDateTime("2015-01-01T00:00:00Z"), + endtime=UTCDateTime("2015-12-31T23:59:00Z"), + path="etc/adjusted/Caldata/", + ) + assert len(readings) == 146 + + starttime = UTCDateTime("2015-02-01T00:00:00Z") + endtime = UTCDateTime("2015-11-27T23:59:00Z") + + update_interval = 86400 * 7 + + result = Affine( + observatory="CMO", + starttime=starttime, + endtime=endtime, + transforms=[ + RotationTranslationXY(memory=np.inf), + TranslateOrigins(memory=np.inf), + ], + update_interval=update_interval, + acausal=True, + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("CMO", "inf_weekly") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + assert_equal(len(matrices), ((endtime - starttime) // update_interval) + 1) + + +def test_CMO2015_infinite_one_interval(): + readings = get_spreadsheet_directory_readings( + observatory="CMO", + starttime=UTCDateTime("2015-01-01T00:00:00Z"), + endtime=UTCDateTime("2015-12-31T23:59:00Z"), + path="etc/adjusted/Caldata/", + ) + + assert len(readings) == 146 + + result = Affine( + observatory="CMO", + starttime=UTCDateTime("2015-02-01T00:00:00Z"), + endtime=UTCDateTime("2015-11-27T23:59:00Z"), + transforms=[ + RotationTranslationXY(memory=np.inf), + TranslateOrigins(memory=np.inf), + ], + acausal=True, + update_interval=None, + ).calculate( + readings=readings, + ) + + matrices = format_result([adjusted_matrix.matrix for adjusted_matrix in result]) + expected_matrices = get_excpected_matrices("CMO", "inf_one_interval") + for i in range(len(matrices)): + assert_array_almost_equal( + matrices[i], + expected_matrices[i], + decimal=3, + err_msg=f"Matrix {i} not equal", + ) + + assert_equal(len(matrices), 1) diff --git a/test/algorithm_test/AdjustedAlgorithm_test.py b/test/algorithm_test/AdjustedAlgorithm_test.py index f37f93f78893e6f13b22e50923724bca0b4ee260..d44ae09e6d06cd2feca7447c7d50992fceaf0134 100644 --- a/test/algorithm_test/AdjustedAlgorithm_test.py +++ b/test/algorithm_test/AdjustedAlgorithm_test.py @@ -1,3 +1,4 @@ +from geomagio.adjusted.AdjustedMatrix import AdjustedMatrix from geomagio.algorithm import AdjustedAlgorithm as adj import geomagio.iaga2002 as i2 from numpy.testing import assert_almost_equal, assert_equal @@ -8,19 +9,53 @@ def test_construct(): # load adjusted data transform matrix and pier correction a = adj(statefile="etc/adjusted/adjbou_state_.json") - assert_almost_equal(actual=a.matrix[0, 0], desired=9.83427577e-01, decimal=6) + assert_almost_equal(actual=a.matrix.matrix[0, 0], desired=9.83427577e-01, decimal=6) - assert_equal(actual=a.pier_correction, desired=-22) + assert_equal(actual=a.matrix.pier_correction, desired=-22) -def test_process_XYZF(): +def assert_streams_almost_equal(adjusted, expected, channels): + for channel in channels: + assert_almost_equal( + actual=adjusted.select(channel=channel)[0].data, + desired=expected.select(channel=channel)[0].data, + decimal=2, + ) + + +def test_process_XYZF_AdjustedMatrix(): """algorithm_test.AdjustedAlgorithm_test.test_process() Check adjusted data processing versus files generated from original script """ - # load adjusted data transform matrix and pier correction - a = adj(statefile="etc/adjusted/adjbou_state_.json") + # Initiate algorithm with AdjustedMatrix object + a = adj( + matrix=AdjustedMatrix( + matrix=[ + [ + 0.9834275767090617, + -0.15473074200902157, + 0.027384986324932026, + -1276.164681191976, + ], + [ + 0.16680172992706568, + 0.987916201012128, + -0.0049868332295851525, + -0.8458192581350419, + ], + [ + -0.006725053082782385, + -0.011809351484171948, + 0.9961869012493976, + 905.3800885796844, + ], + [0, 0, 0, 1], + ], + pier_correction=-22, + ) + ) # load boulder Jan 16 files from /etc/ directory with open("etc/adjusted/BOU201601vmin.min") as f: @@ -31,34 +66,77 @@ def test_process_XYZF(): # process hezf (raw) channels with loaded transform adjusted = a.process(raw) - # compare channels from adjusted and expected streams - assert_almost_equal( - actual=adjusted.select(channel="X")[0].data, - desired=expected.select(channel="X")[0].data, - decimal=2, + assert_streams_almost_equal( + adjusted=adjusted, expected=expected, channels=["X", "Y", "Z", "F"] ) - assert_almost_equal( - actual=adjusted.select(channel="Y")[0].data, - desired=expected.select(channel="Y")[0].data, - decimal=2, + + +def test_process_reverse_polarity_AdjustedMatrix(): + """algorithm_test.AdjustedAlgorithm_test.test_process() + + Check adjusted data processing versus files generated from + original script. Tests reverse polarity martix. + """ + # Initiate algorithm with AdjustedMatrix object + a = adj( + matrix=AdjustedMatrix( + matrix=[ + [-1, 0, 0], + [0, -1, 0], + [0, 0, 1], + ], + pier_correction=-22, + ), + inchannels=["H", "E"], + outchannels=["H", "E"], ) - assert_almost_equal( - actual=adjusted.select(channel="Z")[0].data, - desired=expected.select(channel="Z")[0].data, - decimal=2, + + # load boulder May 20 files from /etc/ directory + with open("etc/adjusted/BOU202005vmin.min") as f: + raw = i2.IAGA2002Factory().parse_string(f.read()) + with open("etc/adjusted/BOU202005adj.min") as f: + expected = i2.IAGA2002Factory().parse_string(f.read()) + + # process he(raw) channels with loaded transform + adjusted = a.process(raw) + + assert_streams_almost_equal( + adjusted=adjusted, expected=expected, channels=["H", "E"] ) - assert_almost_equal( - actual=adjusted.select(channel="F")[0].data, - desired=expected.select(channel="F")[0].data, - decimal=2, + + +def test_process_XYZF_statefile(): + """algorithm_test.AdjustedAlgorithm_test.test_process() + + Check adjusted data processing versus files generated from + original script + + Uses statefile to generate AdjustedMatrix + """ + # load adjusted data transform matrix and pier correction + a = adj(statefile="etc/adjusted/adjbou_state_.json") + + # load boulder Jan 16 files from /etc/ directory + with open("etc/adjusted/BOU201601vmin.min") as f: + raw = i2.IAGA2002Factory().parse_string(f.read()) + with open("etc/adjusted/BOU201601adj.min") as f: + expected = i2.IAGA2002Factory().parse_string(f.read()) + + # process hezf (raw) channels with loaded transform + adjusted = a.process(raw) + + assert_streams_almost_equal( + adjusted=adjusted, expected=expected, channels=["X", "Y", "Z", "F"] ) -def test_process_reverse_polarity(): +def test_process_reverse_polarity_statefile(): """algorithm_test.AdjustedAlgorithm_test.test_process() Check adjusted data processing versus files generated from original script. Tests reverse polarity martix. + + Uses statefile to generate AdjustedMatrix """ # load adjusted data transform matrix and pier correction a = adj( @@ -76,14 +154,6 @@ def test_process_reverse_polarity(): # process he(raw) channels with loaded transform adjusted = a.process(raw) - # compare channels from adjusted and expected streams - assert_almost_equal( - actual=adjusted.select(channel="H")[0].data, - desired=expected.select(channel="H")[0].data, - decimal=2, - ) - assert_almost_equal( - actual=adjusted.select(channel="E")[0].data, - desired=expected.select(channel="E")[0].data, - decimal=2, + assert_streams_almost_equal( + adjusted=adjusted, expected=expected, channels=["H", "E"] )