| Previous changeset 0:55bbea0cdc60 (2025-01-13) |
|
Commit message:
planemo upload for repository https://github.com/Helmholtz-UFZ/galaxy-tools/blob/main/tools/saqc/ commit b674325a07b6e964e25cd65967149018dc2671fe |
|
modified:
macros.xml saqc.xml test-data/out.csv |
|
added:
json_to_saqc_config.py test-data/empty_output.csv test-data/test1/config.csv test-data/test1/data.csv test-data/test1/out.csv test-data/test1/saqcTest1.png test-data/test2/config.csv test-data/test2/data.csv test-data/test2/maint.csv test-data/test2/out.csv test_macros.xml |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a json_to_saqc_config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/json_to_saqc_config.py Sat Aug 16 11:43:23 2025 +0000 |
| [ |
| @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +import json +import math +import sys + + +def format_saqc_value_repr(value: any) -> str: + """ + Konvertiert einen Python-Wert in seine korrekte String-Darstellung für die SaQC-Konfiguration. + Behandelt None, Bools, Floats (inkl. inf/nan) und Strings zentral. + """ + if value is None: + return "None" + if isinstance(value, bool): + return str(value) + if isinstance(value, float): + if math.isinf(value): + return "float('inf')" if value > 0 else "float('-inf')" + if math.isnan(value): + return "float('nan')" + return repr(value) + if isinstance(value, int): + return str(value) + if isinstance(value, str): + val_lower = value.lower() + if val_lower == "inf": + return "float('inf')" + if val_lower == "-inf": + return "float('-inf')" + if val_lower == "nan": + return "float('nan')" + escaped_v = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped_v}"' + sys.stderr.write( + f"Warning: Unhandled type {type(value)}. Converting to string representation: '{str(value)}'.\n" + ) + return repr(value) + + +print("varname; function") + +try: + infile = sys.argv[1] + with open(infile) as fh: + params_from_galaxy = json.load(fh) +except Exception as e: + sys.stderr.write( + f"Error opening or reading JSON file {infile}: {type(e).__name__} - {e}\n" + ) + sys.exit(1) + +EMPTY_STRING_IS_NONE_PARAMS = { + "xscope", + "yscope", + "max_gap", + "min_periods", + "min_residuals", + "min_offset", +} + +for r_method_set in params_from_galaxy.get("methods_repeat", []): + method_str_for_error = "unknown_method_in_repeat" + field_str_for_error = "unknown_field_in_repeat" + try: + method_cond_params = r_method_set.get("module_cond", {}).get("method_cond", {}) + if not method_cond_params: + sys.stderr.write( + f"Warning: Skipping a methods_repeat entry due to missing/empty method_cond: {r_method_set}\n" + ) + continue + + params_to_process = method_cond_params.copy() + + method = params_to_process.pop("method_select", "unknown_method") + method_str_for_error = method + + raw_field_val = None + if "field" in params_to_process: + raw_field_val = params_to_process.pop("field") + elif "field_repeat" in params_to_process: + field_repeat_data = params_to_process.pop("field_repeat", []) + if isinstance(field_repeat_data, list) and len(field_repeat_data) > 0: + first_field_item = field_repeat_data[0] + if isinstance(first_field_item, dict): + raw_field_val = first_field_item.get("field") + + if raw_field_val is None or str(raw_field_val).strip() == "": + field_str = "undefined_field" + field_str_for_error = "undefined_field (extraction failed or empty)" + sys.stderr.write( + f"Warning: Field name could not be determined for method '{method}'. Using '{field_str}'.\n" + ) + else: + field_str = str(raw_field_val) if not isinstance(raw_field_val, list) else ",".join(map(str, raw_field_val)) + field_str_for_error = field_str + + saqc_args_dict = {} + for param_key, param_value_json in params_to_process.items(): + if param_key.endswith("_select_type"): + continue + + actual_param_name_for_saqc = param_key + current_value_for_saqc = param_value_json + + if isinstance(param_value_json, dict) and param_key.endswith("_cond"): + actual_param_name_for_saqc = param_key[:-5] + value_found = False + for inner_k, inner_v in param_value_json.items(): + if not inner_k.endswith("_select_type"): + current_value_for_saqc = inner_v + value_found = True + break + if not value_found: + current_value_for_saqc = None + + if current_value_for_saqc == "__none__": + saqc_args_dict[actual_param_name_for_saqc] = None + elif isinstance(current_value_for_saqc, str) and not current_value_for_saqc and actual_param_name_for_saqc in EMPTY_STRING_IS_NONE_PARAMS: + saqc_args_dict[actual_param_name_for_saqc] = None + else: + saqc_args_dict[actual_param_name_for_saqc] = current_value_for_saqc + param_strings_for_saqc_call = [ + f"{k_saqc}={format_saqc_value_repr(v_saqc)}" + for k_saqc, v_saqc in sorted(saqc_args_dict.items()) + ] + + print( + f"{field_str}; {method}({', '.join(param_strings_for_saqc_call)})", + flush=True, + ) + + except Exception as e: + sys.stderr.write( + f"FATAL Error processing a method entry: {type(e).__name__} - {e}\n" + ) + sys.stderr.write( + f"Offending entry: {r_method_set}\n" + ) + sys.stderr.write( + f"Method context: {method_str_for_error}, Field context: {field_str_for_error}\n" + ) + import traceback + traceback.print_exc(file=sys.stderr) + print( + f"{field_str_for_error}; ERROR_PROCESSING_METHOD({method_str_for_error})", + flush=True) + continue |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a macros.xml --- a/macros.xml Mon Jan 13 16:27:12 2025 +0000 +++ b/macros.xml Sat Aug 16 11:43:23 2025 +0000 |
| b |
| @@ -1,12 +1,11 @@ <macros> - <xml name="requirements"> <requirements> <requirement type="package" version="@TOOL_VERSION@">saqc</requirement> </requirements> </xml> -<token name="@TOOL_VERSION@">2.4.1</token> +<token name="@TOOL_VERSION@">2.6.0</token> <token name="@VERSION_SUFFIX@">0</token> <xml name="citations"> @@ -15,4 +14,115 @@ </citations> </xml> +<xml name="field"> + <param argument="field" + label="Field" + optional="false" + value="" + type="text" + help="Variable to process"/> +</xml> + +<xml name="field_multiple"> + <repeat name="field_repeat" title="field(s)" min="1"> + <param argument="field" + label="Field" + optional="false" + value="" + type="text" + help="Variable to process"/> + </repeat> +</xml> + +<xml name="flag"> + <param argument="flag" + label="Flag" + optional="true" + value="255.0" + type="float" + help="The flag value the function uses to mark observations"/> +</xml> + +<xml name="saqc_tests"> + <tests> + <expand macro="config_tests"/> + <test> + <param name="data" value="test1/data.csv" ftype="csv"/> + <repeat name="methods_repeat"> + <conditional name="module_cond"> + <param name="module_select" value="outliers"/> + <conditional name="method_cond"> + <param name="method_select" value="flagRange"/> + <param name="field" value="SM2"/> + <param name="max" value="60.0"/> + <param name="min" value="10.0"/> + + <param name="flag" value="255.0"/> + </conditional> + </conditional> + </repeat> + <repeat name="methods_repeat"> + <conditional name="module_cond"> + <param name="module_select" value="outliers"/> + <conditional name="method_cond"> + + <param name="method_select" value="flagZScore"/> + + <repeat name="field_repeat"> + <param name="field" value="SM2"/> + </repeat> + <param name="center" value="false"/> + <conditional name="window_cond"> + <param name="window_select_type" value="timedelta"/> + <param name="window" value="30d"/> + </conditional> + <param name="thresh" value="3.5"/> + <param name="method" value="modified"/> + + <param name="flag" value="255.0"/> + </conditional> + </conditional> + </repeat> + <repeat name="methods_repeat"> + <conditional name="module_cond"> + <param name="module_select" value="tools"/> + <conditional name="method_cond"> + <param name="method_select" value="plot"/> + <param name="field" value="SM2"/> + <param name="path" value="test"/> + <conditional name="history_cond"> + <param name="history_select_type" value="valid" /> + <param name="history" value="valid" /> + </conditional> + <param name="mode" value="oneplot"/> + <param name="max_gap" value="" /> + <param name="xscope" value="" /> + <param name="yscope" value="" /> + <param name="dfilter" value="inf" /> + </conditional> + </conditional> + </repeat> + + <output name="output" value="test1/out.csv" ftype="csv"/> + <output name="config_out" ftype="txt"> + <assert_contents> + <has_n_lines n="4"/> + <has_n_columns n="2" sep=";"/> + <has_text_matching expression="flagRange.*max=60.0.*min=10.0"/> + <has_text_matching expression='flagZScore.*method="modified".*thresh=3.5.*window="30d"'/> + <has_text_matching expression='plot.*dfilter=float\('inf'\).*history="valid".*path="test"'/> + </assert_contents> + </output> + + <output_collection name="plots" type="list"> + <element name="test" ftype="png"> + <assert_contents> + <has_text text="PNG"/> + <has_size value="120k" delta="10k"/> + </assert_contents> + </element> + </output_collection> + </test> + </tests> +</xml> </macros> \ No newline at end of file |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a saqc.xml --- a/saqc.xml Mon Jan 13 16:27:12 2025 +0000 +++ b/saqc.xml Sat Aug 16 11:43:23 2025 +0000 |
| [ |
| b'@@ -1,57 +1,1224 @@\n <tool name="SaQC" id="saqc" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="22.01">\n- <description>quality control for time series</description>\n- <creator>\n- <organization name="Helmholtz Centre for Environmental Research - UFZ, Research Data Management"\n- url ="https://www.ufz.de/index.php?en=45348"/>\n- </creator>\n+ <description>quality control pipelines for environmental sensor data</description>\n <macros>\n <import>macros.xml</import>\n+ <import>test_macros.xml</import>\n </macros>\n <expand macro="requirements"/>\n <version_command><![CDATA[python -c \'import saqc; print(saqc.__version__)\']]></version_command>\n- <command detect_errors="exit_code"><![CDATA[\n- ln -s \'$config\' config.csv &&\n+ <command><![CDATA[#if str($run_test_mode) == "true":\n+ \'$__tool_directory__\'/json_to_saqc_config.py \'$param_conf\' > config.csv\n+#else\n+ \'$__tool_directory__\'/json_to_saqc_config.py \'$param_conf\' > config.csv &&\n #for $i, $d in enumerate($data)\n- ln -s \'$d\' \'${i}.csv\' &&\n+ ##maybe link to element_identifier\n+ ln -s \'$d\' \'${i}.csv\' &&\n #end for\n- saqc -c config.csv\n+ saqc --config config.csv\n #for $i, $d in enumerate($data)\n- -d \'${i}.csv\'\n+ --data \'${i}.csv\'\n #end for\n- -o output.csv\n- ]]></command>\n+ --outfile output.csv\n+#end if]]></command>\n+ <configfiles>\n+ <inputs name="param_conf"/>\n+ </configfiles>\n <inputs>\n- <param label="Input Table(s)"\n- argument="--data"\n- type="data"\n- format="csv"\n- multiple="true"\n- optional="false"\n- />\n- <param label="Configuration"\n- argument="--config"\n- type="data"\n- format="txt"\n- optional="false"/>\n+ <param argument="--data" type="data" label="Input table(s)" format="csv" multiple="true"/>\n+ <param name="run_test_mode" type="hidden" value="false" label=""/>\n+ <repeat name="methods_repeat" title="Methods (add multiple QC steps)">\n+ <conditional name="module_cond" label="SaQC Module">\n+ <param name="module_select" type="select" label="Select SaQC module">\n+ <option value="breaks">breaks: Detecting breaks in data</option>\n+ <option value="changepoints">changepoints: changepoints</option>\n+ <option value="constants">constants: constants</option>\n+ <option value="curvefit">curvefit: curvefit</option>\n+ <option value="drift">drift: drift</option>\n+ <option value="flagtools">flagtools: flagtools</option>\n+ <option value="generic">generic: generic</option>\n+ <option value="interpolation">interpolation: interpolation</option>\n+ <option value="noise">noise: noise</option>\n+ <option value="outliers">outliers: outliers</option>\n+ <option value="pattern">pattern: pattern</option>\n+ <option value="resampling">resampling: resampling</option>\n+ <option value="residuals">residuals: residuals</option>\n+ <option value="rolling">rolling: rolling</option>\n+ <option value="scores">scores: scores</option>\n+ <option value="tools">tools: tools</option>\n+ <option value="transformation">transformation: transformation</option>\n+ </param>\n+ <when value="breaks">\n+ <conditional name="method_cond" label="Method">\n+ <param name="method_select" type="select" label="Method">\n+ <option value="flagIsolated">flagIsolated: Find and flag temporal isolated groups of data</option>\n+ <option value="flagJumps">flagJumps: Flag jumps and drops in data</option>\n+ <option value="flagMissing">flagMissing: Flag NaNs in data</option>\n+ </param>\n+ <when value="flagIsolated">\n+ <param argument="field" type="text" value="" label="Field" help="The name of the variable to process.">\n+ <validator type="empty_field"/>\n+ </param>\n+ <pa'..b'" type="boolean" label="Wheather or not to include the mask defining bounds to the mask" help="Wheather or not to include the mask defining bounds to the mask" checked="true" truevalue="closed" falsevalue=""/>\n+ </when>\n+ </conditional>\n+ </when>\n+ <when value="transformation">\n+ <conditional name="method_cond" label="Method">\n+ <param name="method_select" type="select" label="Method">\n+ <option value="transform">transform: Transform data by applying a custom function on data chunks of variable size. Existing flags are preserved</option>\n+ </param>\n+ <when value="transform">\n+ <param argument="field" type="text" value="" label="Field" help="The name of the variable to process.">\n+ <validator type="empty_field"/>\n+ </param>\n+ <param argument="func" type="text" value="" label="Transformation function" help="Transformation function">\n+ <validator type="empty_field"/>\n+ </param>\n+ <conditional name="freq_cond">\n+ <param name="freq_select_type" type="select" value="none" label="Size of the data window Input Mode" help="The transformation is applied on each window individually * ``None``: Apply transformation on the entire data set at once * ``int`` : Apply transformation on successive data chunks of the given length. Must be grater than 0. * Offset String : Apply transformation on successive data chunks of the given temporal extension">\n+ <option value="number">Frequency as Value (float)</option>\n+ <option value="offset">Frequency as Offset string</option>\n+ <option value="none">None (use default)</option>\n+ </param>\n+ <when value="number">\n+ <param argument="freq" type="float" value="" label="Size of the data window (Frequency as Value (float))"/>\n+ </when>\n+ <when value="offset">\n+ <param argument="freq" type="text" value="" label="Size of the data window (Frequency as Offset string)">\n+ <validator type="empty_field"/>\n+ </param>\n+ </when>\n+ <when value="none">\n+ <param name="freq" type="hidden" value="__none__" label=""/>\n+ </when>\n+ </conditional>\n+ </when>\n+ </conditional>\n+ </when>\n+ </conditional>\n+ </repeat>\n </inputs>\n <outputs>\n- <data name="output"\n- format="csv"\n- from_work_dir="output.csv"\n- hidden="false"/>\n+ <data name="output" format="csv" label="${tool.name} on ${on_string}: Processed Data" from_work_dir="output.csv" hidden="false"/>\n+ <collection name="plots" type="list" label="${tool.name} on ${on_string}: Plots (if any generated)">\n+ <discover_datasets pattern="(?P<name>.*)\\.png" ext="png" visible="true"/>\n+ </collection>\n+ <data name="config_out" format="txt" label="${tool.name} on ${on_string}: Generated SaQC Configuration" from_work_dir="config.csv" hidden="false"/>\n </outputs>\n- <tests>\n- <test>\n- <!-- https://rdm-software.pages.ufz.de/saqc/cookbooks/MultivariateFlagging.html -->\n- <param name="data" value="data.csv,maint.csv" ftype="csv"/>\n- <param name="config" value="config.txt" ftype="txt"/>\n- <output name="output" value="out.csv" ftype="csv"/>\n- </test>\n- </tests>\n- <help><![CDATA[\n- This tool runs the SaQC quality control pipeline specified by a config file on the input data.\n- Just upload your data and a config file, and the tool will run the SaQC pipeline on the data.\n-]]></help>\n+ <expand macro="saqc_tests"/>\n+ <help><![CDATA[This tool provides access to SaQC functions for quality control of time series data. Select a module and method, then configure its parameters.]]></help>\n <expand macro="citations"/>\n </tool>\n \n' |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/empty_output.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/empty_output.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| @@ -0,0 +1,1 @@ + |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/out.csv --- a/test-data/out.csv Mon Jan 13 16:27:12 2025 +0000 +++ b/test-data/out.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| b'@@ -1,6 +1,5 @@\n-,kNN_scores,kNN_scores,level_raw,level_raw,level_z,level_z,maint,maint,sac254_corr,sac254_corr,sac254_raw,sac254_raw,sac_z,sac_z,water_temp_raw,water_temp_raw,water_z,water_z\n+,sac254_raw,sac254_raw,level_raw,level_raw,water_temp_raw,water_temp_raw,maint,maint,sac254_corr,sac254_corr,level_z,level_z,water_z,water_z,sac_z,sac_z,kNN_scores,kNN_scores\n ,data,flags,data,flags,data,flags,data,flags,data,flags,data,flags,data,flags,data,flags,data,flags\n-Timestamp,,,,,,,,,,,,,,,,,,\n 2016-01-10 11:15:00,nan,nan,nan,nan,nan,nan,2016-01-10 12:15:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n 2016-01-12 14:40:00,nan,nan,nan,nan,nan,nan,2016-01-12 15:30:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n 2016-02-10 13:40:00,nan,nan,nan,nan,nan,nan,2016-02-10 14:40:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n@@ -20,2651 +19,2651 @@\n 2016-09-20 07:00:00,nan,nan,nan,nan,nan,nan,2016-09-20 09:15:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n 2016-10-25 08:15:00,nan,nan,nan,nan,nan,nan,2016-10-25 09:50:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n 2016-11-07 09:00:00,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED,nan,nan,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED\n-2016-11-07 09:15:00,3.2809596322332033,UNFLAGGED,95.22133333333333,UNFLAGGED,-1.4857440608231318,UNFLAGGED,nan,nan,21.80953333333333,UNFLAGGED,21.80953333333333,UNFLAGGED,-0.8337933247126337,UNFLAGGED,7.99,UNFLAGGED,1.4745966518654343,UNFLAGGED\n-2016-11-07 09:30:00,2.603586562136605,UNFLAGGED,95.21350000000001,UNFLAGGED,-1.4867717510350733,UNFLAGGED,nan,nan,22.688286666666666,UNFLAGGED,22.688286666666666,UNFLAGGED,-0.20314027557653885,UNFLAGGED,7.99,UNFLAGGED,1.4745966518654343,UNFLAGGED\n-2016-11-07 09:45:00,4.431650241803424,UNFLAGGED,95.22550000000001,UNFLAGGED,-1.4851974170933737,UNFLAGGED,nan,nan,21.223173333333335,UNFLAGGED,21.223173333333335,UNFLAGGED,-1.2546050677403737,UNFLAGGED,8.007333333333333,UNFLAGGED,1.486742333592796,UNFLAGGED\n-2016-11-07 10:00:00,3.4641053861793925,UNFLAGGED,95.2275,UNFLAGGED,-1.484935028103091,UNFLAGGED,nan,nan,22.084333333333333,UNFLAGGED,22.084333333333333,UNFLAGGED,-0.6365781889870983,UNFLAGGED,8.001333333333333,UNFLAGGED,1.482538059148709,UNFLAGGED\n-2016-11-07 10:15:00,3.189688691081255,UNFLAGGED,95.26866666666668,UNFLAGGED,-1.4795341880530937,UNFLAGGED,nan,nan,21.79454,UNFLAGGED,21.79454,UNFLAGGED,-0.8445535585856713,UNFLAGGED,7.974,UNFLAGGED,1.4633852533478695,UNFLAGGED\n-2016-11-07 10:30:00,5.008621710395384,UNFLAGGED,95.29883333333333,UNFLAGGED,-1.4755764874496555,UNFLAGGED,nan,nan,20.745493333333332,UNFLAGGED,20.745493333333332,UNFLAGGED,-1.5974206646594913,UNFLAGGED,7.961333333333333,UNFLAGGED,1.4545095628547973,UNFLAGGED\n-2016-11-07 10:45:00,3.171039233420425,UNFLAGGED,95.27866666666667,UNFLAGGED,-1.4782222431016787,UNFLAGGED,nan,nan,21.78129333333333,UNFLAGGED,21.78129333333333,UNFLAGGED,-0.8540602658803524,UNFLAGGED,7.968666666666667,UNFLAGGED,1.459648120508681,UNFLAGGED\n-2016-11-07 11:00:00,2.708662482265309,UNFLAGGED,95.2165,UNFLAGGED,-1.4863781675496504,UNFLAGGED,nan,nan,22.6398,UNFLAGGED,22.6398,UNFLAGGED,-0.23793759925756192,UNFLAGGED,7.9959999999999996,UNFLAGGED,1.4788009263095205,UNFLAGGED\n-2016-11-07 11:15:00,2.624185502656696,UNFLAGGED,95.15116666666667,UNFLAGGED,-1.494949541232236,UNFLAGGED,nan,nan,22.783839999999998,UNFLAGGED,22.783839999999998,UNFLAGGED,-0.13456471661598537,UNFLAGGED,8.008666666666667,UNFLAGGED,1.487676616802593,UNFLAGGED\n-2016-11-07 11:30:00,2.9929298128150275,UNFLAGGED,95.07966666666665,UNFLAGGED,-1.5043299476348646,UNFLAGGED,nan,nan,23.163793333333334,UNFLAGGED,23.163793333333334,UNFLAGGED,0.1381155897985083,UNFLAGGED,8.01,UNFLAGGED,1.48861090001239,UNFLAGGED\n-2016-11-07 11:45:00,3.752634516279193,UNFLAGGED,95.018,UNFLAGGED,-1.5124202748352633,UNFLAGGED,nan,nan,23.389173333333332,UNFLAGGED,23.389173333333332,UNFLAGGED,0.2998635784805327,UNFLAGGED,8.018666666666666,UNFLAGGED,1.4946837408760707,UNFLAGGED\n-2016-11-07 12:00:00,6.033690150'..b'UNFLAGGED,-0.0033863900083442628,UNFLAGGED,3.2094459165601243,UNFLAGGED\n+2016-12-04 18:15:00,23.691826666666664,UNFLAGGED,106.6226,UNFLAGGED,2.4113333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.10435982690422928,UNFLAGGED,-0.8920413224184737,UNFLAGGED,-0.05006288760348915,UNFLAGGED,3.3462437691954965,UNFLAGGED\n+2016-12-04 18:30:00,23.870706666666667,UNFLAGGED,106.5374,UNFLAGGED,2.41,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.06926656863647129,UNFLAGGED,-0.893327364832533,UNFLAGGED,0.16199667614063953,UNFLAGGED,2.670953422798556,UNFLAGGED\n+2016-12-04 18:45:00,24.067453333333336,UNFLAGGED,106.5062,UNFLAGGED,2.4186666666666667,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.056415516313067696,UNFLAGGED,-0.8849680891411461,UNFLAGGED,0.3952369059116857,UNFLAGGED,2.8110520075789296,UNFLAGGED\n+2016-12-04 19:00:00,24.871546666666667,UNFLAGGED,106.49173333333333,UNFLAGGED,2.42,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.05045680188105722,UNFLAGGED,-0.8836820467270867,UNFLAGGED,1.3484775223932681,UNFLAGGED,4.135793107790369,UNFLAGGED\n+2016-12-04 19:15:00,25.753866666666667,UNFLAGGED,106.48306666666666,UNFLAGGED,2.428666666666667,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.0468870651245546,UNFLAGGED,-0.8753227710356993,UNFLAGGED,2.3944546818449477,UNFLAGGED,8.32811978188483,UNFLAGGED\n+2016-12-04 19:30:00,24.198533333333334,UNFLAGGED,106.4846,UNFLAGGED,2.421333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.0475186339353236,UNFLAGGED,-0.8823960043130273,UNFLAGGED,0.5506302848002106,UNFLAGGED,3.098174011330855,UNFLAGGED\n+2016-12-04 19:45:00,24.15899333333333,UNFLAGGED,106.498,UNFLAGGED,2.4113333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.05303799615114643,UNFLAGGED,-0.8920413224184737,UNFLAGGED,0.5037562063652642,UNFLAGGED,2.960259596791867,UNFLAGGED\n+2016-12-04 20:00:00,24.709206666666667,UNFLAGGED,106.487,UNFLAGGED,2.401333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.048507176421737006,UNFLAGGED,-0.9016866405239204,UNFLAGGED,1.1560258811912256,UNFLAGGED,3.564664862193379,UNFLAGGED\n+2016-12-04 20:15:00,23.766033333333333,UNFLAGGED,106.43473333333334,UNFLAGGED,2.4,UNFLAGGED,nan,nan,nan,UNFLAGGED,0.02697891782868899,UNFLAGGED,-0.9029726829379798,UNFLAGGED,0.03790800551055091,UNFLAGGED,3.0535580197692864,UNFLAGGED\n+2016-12-04 20:30:00,23.0237,UNFLAGGED,106.3152,UNFLAGGED,2.3913333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,-0.022255989897517196,UNFLAGGED,-0.9113319586293668,UNFLAGGED,-0.8421170549735076,UNFLAGGED,5.557099423920803,UNFLAGGED\n+2016-12-04 20:45:00,23.851326666666665,UNFLAGGED,106.17666666666666,UNFLAGGED,2.381333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,-0.0793168589745167,UNFLAGGED,-0.9209772767348136,UNFLAGGED,0.13902197608830552,UNFLAGGED,2.8384557312572616,UNFLAGGED\n+2016-12-04 21:00:00,24.59596,UNFLAGGED,106.03406666666666,UNFLAGGED,2.3626666666666667,UNFLAGGED,nan,nan,nan,UNFLAGGED,-0.13805275837571768,UNFLAGGED,-0.9389818705316467,UNFLAGGED,1.0217736521616563,UNFLAGGED,4.228704607480928,UNFLAGGED\n+2016-12-04 21:15:00,24.040879999999998,UNFLAGGED,105.91706666666667,UNFLAGGED,2.3513333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,-0.18624420458847968,UNFLAGGED,-0.9499132310511529,UNFLAGGED,0.3637346168134354,UNFLAGGED,3.127373228777544,UNFLAGGED\n+2016-12-04 21:30:00,24.2467,UNFLAGGED,105.87166666666667,UNFLAGGED,2.341333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,-0.2049441332898439,UNFLAGGED,-0.9595585491565997,UNFLAGGED,0.6077311475033751,UNFLAGGED,3.8452553483406025,UNFLAGGED\n+2016-12-04 21:45:00,23.86366,UNFLAGGED,105.86093333333334,UNFLAGGED,2.3313333333333333,UNFLAGGED,nan,nan,nan,UNFLAGGED,-0.2093651149652035,UNFLAGGED,-0.969203867262046,UNFLAGGED,0.15364295823375346,UNFLAGGED,3.4434139012772293,UNFLAGGED\n 2016-12-04 22:00:00,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED,nan,nan,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED,nan,UNFLAGGED\n 2016-12-13 15:05:00,nan,nan,nan,nan,nan,nan,2016-12-13 16:50:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n 2017-01-10 08:05:00,nan,nan,nan,nan,nan,nan,2017-01-10 09:15:00,UNFLAGGED,nan,nan,nan,nan,nan,nan,nan,nan,nan,nan\n' |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test1/config.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test1/config.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| @@ -0,0 +1,5 @@ +varname;test +#------;-------------------------- +SM2 ;flagRange(min=10, max=60) +SM2 ;flagZScore(window="30d", thresh=3.5, method="modified", center=False) +SM2 ;plot() \ No newline at end of file |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test1/data.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test1/data.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| b'@@ -0,0 +1,14694 @@\n+Date,Battery,SM1,SM2\n+2016-04-01 00:05:48,3573,32.685,29.3157\n+2016-04-01 00:20:42,3572,32.7428,29.3157\n+2016-04-01 00:35:37,3572,32.6186,29.3679\n+2016-04-01 00:50:32,3572,32.736999999999995,29.3679\n+2016-04-01 01:05:26,3572,32.736999999999995,29.3131\n+2016-04-01 01:20:21,3571,32.6186,29.3157\n+2016-04-01 01:35:16,3571,32.736999999999995,29.3157\n+2016-04-01 01:50:11,3571,32.736999999999995,29.4727\n+2016-04-01 02:05:05,3571,32.6186,29.5252\n+2016-04-01 02:20:00,3571,32.911,29.5252\n+2016-04-01 02:34:55,3570,32.85,29.5226\n+2016-04-01 02:49:49,3570,32.911,29.4727\n+2016-04-01 03:04:44,3570,32.9081,29.5252\n+2016-04-01 03:19:39,3569,32.8993,29.5252\n+2016-04-01 03:34:33,3569,32.9023,29.4176\n+2016-04-01 03:49:28,3568,32.7861,29.5252\n+2016-04-01 04:04:23,3568,32.9023,29.5252\n+2016-04-01 04:19:17,3568,33.013000000000005,29.4727\n+2016-04-01 04:34:12,3568,32.9023,29.5252\n+2016-04-01 04:49:07,3567,32.8993,29.5252\n+2016-04-01 05:04:01,3567,32.9023,29.5252\n+2016-04-01 05:18:56,3566,32.9023,29.4727\n+2016-04-01 05:33:50,3567,32.7861,29.5252\n+2016-04-01 05:48:45,3566,32.8993,29.5252\n+2016-04-01 06:03:39,3566,32.8993,29.4727\n+2016-04-01 06:18:34,3566,32.8935,29.5252\n+2016-04-01 06:33:28,3565,32.7774,29.4727\n+2016-04-01 06:48:22,3565,32.8935,29.4727\n+2016-04-01 07:03:17,3565,32.8935,29.5252\n+2016-04-01 07:18:11,3565,32.8935,29.4727\n+2016-04-01 07:33:06,3565,32.8935,29.5252\n+2016-04-01 07:48:00,3565,32.8935,29.5252\n+2016-04-01 08:02:55,3565,32.8935,29.47\n+2016-04-01 08:17:49,3566,32.8935,29.5252\n+2016-04-01 08:32:43,3567,32.8964,29.5252\n+2016-04-01 08:47:38,3568,32.8935,29.5252\n+2016-04-01 09:02:32,3570,32.8354,29.5252\n+2016-04-01 09:17:27,3572,32.8935,29.4727\n+2016-04-01 09:32:22,3575,32.8935,29.5252\n+2016-04-01 09:47:17,3578,32.8354,29.4203\n+2016-04-01 10:02:12,3581,32.8935,29.4176\n+2016-04-01 10:17:07,3583,32.8354,29.47\n+2016-04-01 10:32:02,3587,32.8964,29.4176\n+2016-04-01 10:46:57,3591,32.8325,29.4203\n+2016-04-01 11:01:52,3594,32.7774,29.5252\n+2016-04-01 11:16:47,3596,32.7803,29.5252\n+2016-04-01 11:31:42,3598,32.8935,29.4727\n+2016-04-01 11:46:37,3598,32.7745,29.5252\n+2016-04-01 12:01:32,3598,32.8935,29.5252\n+2016-04-01 12:16:27,3598,32.8354,29.5252\n+2016-04-01 12:31:22,3599,32.9023,29.4727\n+2016-04-01 12:46:17,3599,32.7861,29.4727\n+2016-04-01 13:01:12,3599,32.7861,29.47\n+2016-04-01 13:16:08,3599,32.7949,29.4203\n+2016-04-01 13:31:03,3599,32.9081,29.4176\n+2016-04-01 13:45:58,3599,32.7891,29.4203\n+2016-04-01 14:00:53,3599,32.8036,29.47\n+2016-04-01 14:15:48,3599,32.7428,29.4203\n+2016-04-01 14:30:43,3599,32.685,29.5252\n+2016-04-01 14:45:38,3599,32.6937,29.4727\n+2016-04-01 15:00:34,3599,32.636,29.4203\n+2016-04-01 15:15:29,3599,32.636,29.4203\n+2016-04-01 15:30:24,3599,32.636,29.4203\n+2016-04-01 15:45:19,3599,32.636,29.47\n+2016-04-01 16:00:15,3599,32.6389,29.4176\n+2016-04-01 16:15:10,3599,32.7515,29.4203\n+2016-04-01 16:30:05,3599,32.6937,29.4203\n+2016-04-01 16:45:01,3599,32.636,29.4727\n+2016-04-01 16:59:56,3599,32.7631,29.4727\n+2016-04-01 17:14:51,3598,32.6447,29.4727\n+2016-04-01 17:29:46,3598,32.6476,29.4727\n+2016-04-01 17:44:41,3598,32.7024,29.47\n+2016-04-01 17:59:36,3598,32.7024,29.4727\n+2016-04-01 18:14:31,3597,32.5958,29.4203\n+2016-04-01 18:29:26,3596,32.6534,29.4176\n+2016-04-01 18:44:21,3595,32.6534,29.4727\n+2016-04-01 18:59:16,3593,32.7689,29.5252\n+2016-04-01 19:14:11,3591,32.7111,29.47\n+2016-04-01 19:29:06,3589,32.7689,29.4727\n+2016-04-01 19:44:01,3587,32.6563,29.47\n+2016-04-01 19:58:55,3585,32.7111,29.4727\n+2016-04-01 20:13:50,3583,32.6534,29.4727\n+2016-04-01 20:28:45,3581,32.714,29.4727\n+2016-04-01 20:43:39,3578,32.7689,29.5252\n+2016-04-01 20:58:34,3577,32.7718,29.4203\n+2016-04-01 21:13:28,3574,32.8269,29.4805\n+2016-04-01 21:28:23,3573,32.7631,29.5252\n+2016-04-01 21:43:17,3572,32.7053,29.5252\n+2016-04-01 21:58:12,3570,32.7631,29.533\n+2016-04-01 22:13:06,3569,32.6447,29.5304\n+2016-04-01 22:28:00,3567,32.7631,29.4805\n+2016-04-01 22:42:55,3566,32.7053,29.5304\n+2016-04-01 22:57:50,3566,32.6937,29.533\n+2016-04-01 23:12:44,3566,'..b'31 08:06:45,3592,11.0395,14.4877\n+2016-08-31 08:16:47,3592,11.0395,14.4578\n+2016-08-31 08:26:49,3592,11.0684,14.5202\n+2016-08-31 08:36:52,3593,11.0395,14.5202\n+2016-08-31 08:46:54,3594,11.0684,14.5189\n+2016-08-31 08:56:57,3595,11.0395,14.4864\n+2016-08-31 09:06:59,3596,11.0385,14.4851\n+2016-08-31 09:17:02,3597,11.0395,14.5215\n+2016-08-31 09:27:04,3598,11.0684,14.4838\n+2016-08-31 09:37:07,3598,11.0395,14.5189\n+2016-08-31 09:47:09,3598,11.0395,14.4864\n+2016-08-31 09:57:12,3598,11.0684,14.4851\n+2016-08-31 10:07:14,3599,11.0395,14.5163\n+2016-08-31 10:17:17,3599,11.0395,14.4552\n+2016-08-31 10:27:19,3599,11.0395,14.4526\n+2016-08-31 10:37:22,3599,10.953,14.4838\n+2016-08-31 10:47:24,3599,10.9808,14.515\n+2016-08-31 10:57:27,3599,10.9808,14.4838\n+2016-08-31 11:07:29,3599,10.9808,14.4526\n+2016-08-31 11:17:32,3599,10.9808,14.4539\n+2016-08-31 11:27:35,3599,10.9808,14.4552\n+2016-08-31 11:37:37,3599,10.9818,14.4838\n+2016-08-31 11:47:40,3599,10.953,14.4877\n+2016-08-31 11:57:43,3599,10.9808,14.4552\n+2016-08-31 12:07:46,3599,10.9818,14.4864\n+2016-08-31 12:17:49,3599,10.9818,14.4526\n+2016-08-31 12:27:52,3599,10.9808,14.4864\n+2016-08-31 12:37:54,3599,10.952,14.4851\n+2016-08-31 12:47:57,3599,10.9242,14.4825\n+2016-08-31 12:58:00,3599,10.8954,14.4501\n+2016-08-31 13:08:03,3599,10.8954,14.4539\n+2016-08-31 13:18:05,3599,10.8954,14.4851\n+2016-08-31 13:28:08,3599,10.8944,14.4526\n+2016-08-31 13:38:11,3599,10.8983,14.4526\n+2016-08-31 13:48:14,3599,10.8686,14.48\n+2016-08-31 13:58:17,3599,10.8983,14.3891\n+2016-08-31 14:08:19,3599,10.8408,14.3567\n+2016-08-31 14:18:22,3599,10.8686,14.3865\n+2016-08-31 14:28:25,3599,10.8686,14.3891\n+2016-08-31 14:38:28,3599,10.8686,14.4189\n+2016-08-31 14:48:31,3599,10.8973,14.3891\n+2016-08-31 14:58:34,3599,10.8973,14.4176\n+2016-08-31 15:08:39,3599,10.8973,14.3865\n+2016-08-31 15:18:42,3599,10.8695,14.3891\n+2016-08-31 15:28:45,3599,10.8695,14.3865\n+2016-08-31 15:38:48,3599,10.9002,14.358\n+2016-08-31 15:48:51,3599,10.9012,14.3878\n+2016-08-31 15:58:54,3599,10.8715,14.3567\n+2016-08-31 16:08:57,3599,10.8724,14.3554\n+2016-08-31 16:19:00,3599,10.8724,14.4176\n+2016-08-31 16:29:03,3599,10.8753,14.3865\n+2016-08-31 16:39:06,3599,10.8753,14.3554\n+2016-08-31 16:49:09,3599,10.8734,14.3878\n+2016-08-31 16:59:12,3599,10.9041,14.3865\n+2016-08-31 17:09:15,3599,10.8753,14.3554\n+2016-08-31 17:19:18,3599,10.8753,14.3865\n+2016-08-31 17:29:21,3599,10.9031,14.3878\n+2016-08-31 17:39:24,3599,10.8753,14.3891\n+2016-08-31 17:49:27,3599,10.9041,14.3865\n+2016-08-31 17:59:30,3599,10.9041,14.3865\n+2016-08-31 18:09:33,3599,10.9031,14.3865\n+2016-08-31 18:19:36,3599,10.8744,14.3865\n+2016-08-31 18:29:39,3599,10.9061,14.3567\n+2016-08-31 18:39:42,3599,10.8773,14.3592\n+2016-08-31 18:49:45,3599,10.907,14.3554\n+2016-08-31 18:59:48,3599,10.8485,14.4189\n+2016-08-31 19:09:51,3599,10.8773,14.3256\n+2016-08-31 19:19:53,3599,10.9051,14.3554\n+2016-08-31 19:29:56,3599,10.8773,14.3891\n+2016-08-31 19:39:59,3599,10.9061,14.4202\n+2016-08-31 19:50:02,3599,10.8763,14.4202\n+2016-08-31 20:00:05,3599,10.9051,14.3878\n+2016-08-31 20:10:08,3599,10.8773,14.3567\n+2016-08-31 20:20:11,3599,10.9061,14.4202\n+2016-08-31 20:30:13,3599,10.8773,14.3878\n+2016-08-31 20:40:16,3599,10.8773,14.3878\n+2016-08-31 20:50:19,3599,10.907,14.3878\n+2016-08-31 21:00:22,3599,10.8773,14.3865\n+2016-08-31 21:10:25,3599,10.8773,14.3891\n+2016-08-31 21:20:27,3599,10.8763,14.3865\n+2016-08-31 21:30:30,3599,10.8495,14.3567\n+2016-08-31 21:40:33,3599,10.9061,14.3865\n+2016-08-31 21:50:36,3599,10.8782,14.3865\n+2016-08-31 22:00:38,3599,10.9061,14.3852\n+2016-08-31 22:10:41,3599,10.9051,14.3903\n+2016-08-31 22:20:44,3599,10.9061,14.3541\n+2016-08-31 22:30:46,3599,10.907,14.3865\n+2016-08-31 22:40:49,3599,10.9051,14.3865\n+2016-08-31 22:50:52,3599,10.9061,14.3891\n+2016-08-31 23:00:54,3599,10.9061,14.3852\n+2016-08-31 23:10:57,3599,10.9051,14.3891\n+2016-08-31 23:20:59,3599,10.8773,14.3541\n+2016-08-31 23:31:02,3599,10.9061,14.3865\n+2016-08-31 23:41:04,3599,10.9061,14.3878\n+2016-08-31 23:51:07,3599,10.8773,14.3865\n\\ No newline at end of file\n' |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test1/out.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test1/out.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| b'@@ -0,0 +1,14696 @@\n+,Battery,Battery,SM1,SM1,SM2,SM2\n+,data,flags,data,flags,data,flags\n+Date,,,,,,\n+2016-04-01 00:05:48,3573,UNFLAGGED,32.685,UNFLAGGED,29.3157,UNFLAGGED\n+2016-04-01 00:20:42,3572,UNFLAGGED,32.7428,UNFLAGGED,29.3157,UNFLAGGED\n+2016-04-01 00:35:37,3572,UNFLAGGED,32.6186,UNFLAGGED,29.3679,BAD\n+2016-04-01 00:50:32,3572,UNFLAGGED,32.736999999999995,UNFLAGGED,29.3679,UNFLAGGED\n+2016-04-01 01:05:26,3572,UNFLAGGED,32.736999999999995,UNFLAGGED,29.3131,UNFLAGGED\n+2016-04-01 01:20:21,3571,UNFLAGGED,32.6186,UNFLAGGED,29.3157,UNFLAGGED\n+2016-04-01 01:35:16,3571,UNFLAGGED,32.736999999999995,UNFLAGGED,29.3157,UNFLAGGED\n+2016-04-01 01:50:11,3571,UNFLAGGED,32.736999999999995,UNFLAGGED,29.4727,BAD\n+2016-04-01 02:05:05,3571,UNFLAGGED,32.6186,UNFLAGGED,29.5252,BAD\n+2016-04-01 02:20:00,3571,UNFLAGGED,32.911,UNFLAGGED,29.5252,BAD\n+2016-04-01 02:34:55,3570,UNFLAGGED,32.85,UNFLAGGED,29.5226,BAD\n+2016-04-01 02:49:49,3570,UNFLAGGED,32.911,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 03:04:44,3570,UNFLAGGED,32.9081,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 03:19:39,3569,UNFLAGGED,32.8993,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 03:34:33,3569,UNFLAGGED,32.9023,UNFLAGGED,29.4176,UNFLAGGED\n+2016-04-01 03:49:28,3568,UNFLAGGED,32.7861,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 04:04:23,3568,UNFLAGGED,32.9023,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 04:19:17,3568,UNFLAGGED,33.013000000000005,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 04:34:12,3568,UNFLAGGED,32.9023,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 04:49:07,3567,UNFLAGGED,32.8993,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 05:04:01,3567,UNFLAGGED,32.9023,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 05:18:56,3566,UNFLAGGED,32.9023,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 05:33:50,3567,UNFLAGGED,32.7861,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 05:48:45,3566,UNFLAGGED,32.8993,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 06:03:39,3566,UNFLAGGED,32.8993,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 06:18:34,3566,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 06:33:28,3565,UNFLAGGED,32.7774,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 06:48:22,3565,UNFLAGGED,32.8935,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 07:03:17,3565,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 07:18:11,3565,UNFLAGGED,32.8935,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 07:33:06,3565,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 07:48:00,3565,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 08:02:55,3565,UNFLAGGED,32.8935,UNFLAGGED,29.47,UNFLAGGED\n+2016-04-01 08:17:49,3566,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 08:32:43,3567,UNFLAGGED,32.8964,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 08:47:38,3568,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 09:02:32,3570,UNFLAGGED,32.8354,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 09:17:27,3572,UNFLAGGED,32.8935,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 09:32:22,3575,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 09:47:17,3578,UNFLAGGED,32.8354,UNFLAGGED,29.4203,BAD\n+2016-04-01 10:02:12,3581,UNFLAGGED,32.8935,UNFLAGGED,29.4176,BAD\n+2016-04-01 10:17:07,3583,UNFLAGGED,32.8354,UNFLAGGED,29.47,UNFLAGGED\n+2016-04-01 10:32:02,3587,UNFLAGGED,32.8964,UNFLAGGED,29.4176,UNFLAGGED\n+2016-04-01 10:46:57,3591,UNFLAGGED,32.8325,UNFLAGGED,29.4203,UNFLAGGED\n+2016-04-01 11:01:52,3594,UNFLAGGED,32.7774,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 11:16:47,3596,UNFLAGGED,32.7803,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 11:31:42,3598,UNFLAGGED,32.8935,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 11:46:37,3598,UNFLAGGED,32.7745,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 12:01:32,3598,UNFLAGGED,32.8935,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 12:16:27,3598,UNFLAGGED,32.8354,UNFLAGGED,29.5252,UNFLAGGED\n+2016-04-01 12:31:22,3599,UNFLAGGED,32.9023,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 12:46:17,3599,UNFLAGGED,32.7861,UNFLAGGED,29.4727,UNFLAGGED\n+2016-04-01 13:01:12,3599,UNFLAGGED,32.7861,UNFLAGGED,29.47,UNFLAGGED\n+2016-04-01 13:16:08,3599,UNFLAGGED,32.7949,UNFLAGGED,29.4203,UNFLAGGED\n+2016-04'..b'LAGGED,10.8686,UNFLAGGED,14.4189,UNFLAGGED\n+2016-08-31 14:48:31,3599,UNFLAGGED,10.8973,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 14:58:34,3599,UNFLAGGED,10.8973,UNFLAGGED,14.4176,UNFLAGGED\n+2016-08-31 15:08:39,3599,UNFLAGGED,10.8973,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 15:18:42,3599,UNFLAGGED,10.8695,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 15:28:45,3599,UNFLAGGED,10.8695,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 15:38:48,3599,UNFLAGGED,10.9002,UNFLAGGED,14.358,UNFLAGGED\n+2016-08-31 15:48:51,3599,UNFLAGGED,10.9012,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 15:58:54,3599,UNFLAGGED,10.8715,UNFLAGGED,14.3567,UNFLAGGED\n+2016-08-31 16:08:57,3599,UNFLAGGED,10.8724,UNFLAGGED,14.3554,UNFLAGGED\n+2016-08-31 16:19:00,3599,UNFLAGGED,10.8724,UNFLAGGED,14.4176,UNFLAGGED\n+2016-08-31 16:29:03,3599,UNFLAGGED,10.8753,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 16:39:06,3599,UNFLAGGED,10.8753,UNFLAGGED,14.3554,UNFLAGGED\n+2016-08-31 16:49:09,3599,UNFLAGGED,10.8734,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 16:59:12,3599,UNFLAGGED,10.9041,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 17:09:15,3599,UNFLAGGED,10.8753,UNFLAGGED,14.3554,UNFLAGGED\n+2016-08-31 17:19:18,3599,UNFLAGGED,10.8753,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 17:29:21,3599,UNFLAGGED,10.9031,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 17:39:24,3599,UNFLAGGED,10.8753,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 17:49:27,3599,UNFLAGGED,10.9041,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 17:59:30,3599,UNFLAGGED,10.9041,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 18:09:33,3599,UNFLAGGED,10.9031,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 18:19:36,3599,UNFLAGGED,10.8744,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 18:29:39,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3567,UNFLAGGED\n+2016-08-31 18:39:42,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3592,UNFLAGGED\n+2016-08-31 18:49:45,3599,UNFLAGGED,10.907,UNFLAGGED,14.3554,UNFLAGGED\n+2016-08-31 18:59:48,3599,UNFLAGGED,10.8485,UNFLAGGED,14.4189,UNFLAGGED\n+2016-08-31 19:09:51,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3256,UNFLAGGED\n+2016-08-31 19:19:53,3599,UNFLAGGED,10.9051,UNFLAGGED,14.3554,UNFLAGGED\n+2016-08-31 19:29:56,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 19:39:59,3599,UNFLAGGED,10.9061,UNFLAGGED,14.4202,UNFLAGGED\n+2016-08-31 19:50:02,3599,UNFLAGGED,10.8763,UNFLAGGED,14.4202,UNFLAGGED\n+2016-08-31 20:00:05,3599,UNFLAGGED,10.9051,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 20:10:08,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3567,UNFLAGGED\n+2016-08-31 20:20:11,3599,UNFLAGGED,10.9061,UNFLAGGED,14.4202,UNFLAGGED\n+2016-08-31 20:30:13,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 20:40:16,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 20:50:19,3599,UNFLAGGED,10.907,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 21:00:22,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 21:10:25,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 21:20:27,3599,UNFLAGGED,10.8763,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 21:30:30,3599,UNFLAGGED,10.8495,UNFLAGGED,14.3567,UNFLAGGED\n+2016-08-31 21:40:33,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 21:50:36,3599,UNFLAGGED,10.8782,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 22:00:38,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3852,UNFLAGGED\n+2016-08-31 22:10:41,3599,UNFLAGGED,10.9051,UNFLAGGED,14.3903,UNFLAGGED\n+2016-08-31 22:20:44,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3541,UNFLAGGED\n+2016-08-31 22:30:46,3599,UNFLAGGED,10.907,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 22:40:49,3599,UNFLAGGED,10.9051,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 22:50:52,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 23:00:54,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3852,UNFLAGGED\n+2016-08-31 23:10:57,3599,UNFLAGGED,10.9051,UNFLAGGED,14.3891,UNFLAGGED\n+2016-08-31 23:20:59,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3541,UNFLAGGED\n+2016-08-31 23:31:02,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3865,UNFLAGGED\n+2016-08-31 23:41:04,3599,UNFLAGGED,10.9061,UNFLAGGED,14.3878,UNFLAGGED\n+2016-08-31 23:51:07,3599,UNFLAGGED,10.8773,UNFLAGGED,14.3865,UNFLAGGED\n' |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test1/saqcTest1.png |
| b |
| Binary file test-data/test1/saqcTest1.png has changed |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test2/config.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test2/config.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| @@ -0,0 +1,7 @@ +varname;test +SM2;align(freq="15Min", method="nshift") +'.*';flagRange(min=10, max=60) +SM2;flagMissing() +SM2;flagRange(min=10, max=60) +SM2;flagZScore(window="30d", thresh=3.5, method='modified', center=False) +Dummy;flag(func=(isflagged(SM1) | isflagged(SM2))) |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test2/data.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test2/data.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| b'@@ -0,0 +1,3155 @@\n+Timestamp,sac254_raw,level_raw,water_temp_raw\n+2016-12-01 06:47:00,23.8925,104.147,2.68\n+2016-12-01 07:02:00,22.3412,104.175,2.68\n+2016-12-01 07:17:00,23.5587,104.192,2.7\n+2016-12-01 07:32:00,23.5575,104.23,2.71\n+2016-12-01 07:47:00,24.8387,104.245,2.72\n+2016-12-01 08:02:00,23.2125,104.255,2.75\n+2016-12-01 08:17:00,23.845,104.31,2.77\n+2016-12-01 08:32:00,23.2087,104.39,2.8\n+2016-12-01 08:47:00,24.0362,104.495,2.82\n+2016-12-01 09:02:00,23.5062,104.577,2.84\n+2016-12-01 09:17:00,24.3425,104.777,2.88\n+2016-12-01 09:32:00,24.7375,104.87,2.89\n+2016-12-01 09:47:00,22.3887,104.967,2.9\n+2016-12-01 10:02:00,24.29,105.015,2.93\n+2016-12-01 10:17:00,24.5437,105.022,2.96\n+2016-12-01 10:32:00,24.085,104.917,2.98\n+2016-12-01 10:47:00,23.1162,104.802,3.01\n+2016-12-01 11:02:00,22.8312,104.707,3.03\n+2016-12-01 11:17:00,23.7562,104.727,3.04\n+2016-12-01 11:32:00,21.6137,104.722,3.06\n+2016-12-01 11:47:00,22.9737,104.805,3.09\n+2016-12-01 12:02:00,23.8925,104.947,3.12\n+2016-12-01 12:17:00,25.1375,105.042,3.14\n+2016-12-01 12:32:00,23.0212,105.12,3.14\n+2016-12-01 12:47:00,22.295,105.217,3.16\n+2016-12-01 13:02:00,23.2188,105.245,3.17\n+2016-12-01 13:17:00,23.66,105.253,3.2\n+2016-12-01 13:32:00,22.8325,105.152,3.21\n+2016-12-01 13:47:00,23.8913,105.11,3.22\n+2016-12-01 14:02:00,24.1912,105.065,3.24\n+2016-12-01 14:17:00,21.9088,105.128,3.25\n+2016-12-01 14:32:00,21.22,105.212,3.27\n+2016-12-01 14:47:00,22.8312,105.335,3.3\n+2016-12-01 15:02:00,23.0238,105.475,3.31\n+2016-12-01 15:17:00,22.5362,105.637,3.31\n+2016-12-01 15:32:00,22.1588,105.765,3.32\n+2016-12-01 15:47:00,22.3875,105.93,3.36\n+2016-12-01 16:02:00,24.6875,106.02,3.38\n+2016-12-01 16:17:00,23.41,106.042,3.39\n+2016-12-01 16:32:00,21.9175,106.092,3.41\n+2016-12-01 16:47:00,21.7188,106.085,3.44\n+2016-12-01 17:02:00,24.2438,106.24,3.47\n+2016-12-01 17:17:00,22.9287,106.365,3.49\n+2016-12-01 17:32:00,23.8438,106.585,3.51\n+2016-12-01 17:47:00,23.7975,106.802,3.54\n+2016-12-01 18:02:00,23.9425,107.08,3.55\n+2016-12-01 18:17:00,23.8463,107.3,3.56\n+2016-12-01 18:32:00,24.24,107.522,3.6\n+2016-12-01 18:47:00,22.3475,107.768,3.62\n+2016-12-01 19:02:00,23.7525,107.96,3.65\n+2016-12-01 19:17:00,22.1562,108.027,3.66\n+2016-12-01 19:32:00,22.7825,108.052,3.69\n+2016-12-01 19:47:00,22.5338,108.057,3.71\n+2016-12-01 20:02:00,23.3063,108.112,3.75\n+2016-12-01 20:17:00,25.0925,108.262,3.77\n+2016-12-01 20:32:00,23.1675,108.447,3.79\n+2016-12-01 20:47:00,23.2675,108.64,3.81\n+2016-12-01 21:02:00,23.5112,108.895,3.82\n+2016-12-01 21:17:00,23.89,109.01,3.84\n+2016-12-01 21:32:00,22.9762,109.152,3.85\n+2016-12-01 21:47:00,22.8312,109.24,3.87\n+2016-12-01 22:02:00,22.295,109.225,3.88\n+2016-12-01 22:17:00,22.8775,109.162,3.89\n+2016-12-01 22:32:00,22.9337,109.092,3.89\n+2016-12-01 22:47:00,22.3925,108.91,3.9\n+2016-12-01 23:02:00,23.6575,108.872,3.93\n+2016-12-01 23:17:00,24.495,108.94,3.95\n+2016-12-01 23:32:00,23.7525,109.007,3.96\n+2016-12-01 23:47:00,23.41,109.048,3.96\n+2016-12-02 00:02:00,22.88,109.107,3.97\n+2016-12-02 00:17:00,23.3138,109.097,3.96\n+2016-12-02 00:32:00,23.4637,109.12,3.97\n+2016-12-02 00:47:00,21.865,109.195,3.99\n+2016-12-02 01:02:00,23.5575,109.102,4.0\n+2016-12-02 01:17:00,24.495,108.965,4.02\n+2016-12-02 01:32:00,23.6562,108.75,4.03\n+2016-12-02 01:47:00,21.765,108.63,4.03\n+2016-12-02 02:02:00,24.1375,108.455,4.05\n+2016-12-02 02:17:00,25.0425,108.302,4.06\n+2016-12-02 02:32:00,22.25,108.425,4.08\n+2016-12-02 02:47:00,22.2475,108.6,4.11\n+2016-12-02 03:02:00,23.75,108.78,4.15\n+2016-12-02 03:17:00,22.6387,108.878,4.19\n+2016-12-02 03:32:00,23.7038,108.912,4.2\n+2016-12-02 03:47:00,23.9425,108.807,4.21\n+2016-12-02 04:02:00,24.1437,108.605,4.18\n+2016-12-02 04:17:00,23.995,108.302,4.18\n+2016-12-02 04:32:00,23.9925,108.09,4.19\n+2016-12-02 04:47:00,22.3475,107.74,4.22\n+2016-12-02 05:02:00,23.1663,107.482,4.25\n+2016-12-02 05:17:00,22.93,107.212,4.22\n+2016-12-02 05:32:00,25.1912,106.945,4.2\n+2016-12-02 05:47:00,22.2988,106.705,4.18\n+2016-12-02 06:02:00,24.7462,106.442,4.14\n+2016-12-02 06:17:00,22.9788,106.19,4.11\n+2016-12-02'..b',2.22\n+2017-01-02 03:02:00,26.6612,84.66,2.21\n+2017-01-02 03:17:00,25.0975,84.7375,2.22\n+2017-01-02 03:32:00,26.1263,84.825,2.24\n+2017-01-02 03:47:00,25.6862,84.8925,2.24\n+2017-01-02 04:02:00,25.68,84.955,2.23\n+2017-01-02 04:17:00,27.705,85.01,2.22\n+2017-01-02 04:32:00,27.3937,85.065,2.21\n+2017-01-02 04:47:00,26.2663,85.1225,2.2\n+2017-01-02 05:02:00,26.3138,85.16,2.2\n+2017-01-02 05:17:00,27.1537,85.1775,2.2\n+2017-01-02 05:32:00,27.3512,85.2225,2.18\n+2017-01-02 05:47:00,26.5025,85.235,2.17\n+2017-01-02 06:02:00,26.4562,85.245,2.16\n+2017-01-02 06:17:00,25.195,85.2625,2.15\n+2017-01-02 06:32:00,24.24,85.2725,2.12\n+2017-01-02 06:47:00,26.6625,85.285,2.07\n+2017-01-02 07:02:00,27.2475,85.3075,2.07\n+2017-01-02 07:17:00,25.785,85.325,2.06\n+2017-01-02 07:32:00,26.5638,85.3325,2.02\n+2017-01-02 07:47:00,26.5963,85.32,1.99\n+2017-01-02 08:02:00,26.3475,85.29,1.97\n+2017-01-02 08:17:00,27.34,85.2575,1.98\n+2017-01-02 08:32:00,26.9463,85.215,1.97\n+2017-01-02 08:47:00,26.6512,85.1675,1.94\n+2017-01-02 09:02:00,26.1188,85.1625,1.91\n+2017-01-02 09:17:00,25.8712,85.1775,1.91\n+2017-01-02 09:32:00,25.5825,85.21,1.98\n+2017-01-02 09:47:00,26.8012,85.3025,2.11\n+2017-01-02 10:02:00,25.05,85.4575,2.18\n+2017-01-02 10:17:00,28.205,85.555,2.21\n+2017-01-02 10:32:00,28.6912,85.6325,2.21\n+2017-01-02 10:47:00,27.3825,85.6825,2.23\n+2017-01-02 11:02:00,26.4925,85.6875,2.22\n+2017-01-02 11:17:00,26.0587,85.665,2.17\n+2017-01-02 11:32:00,26.345,85.6225,2.1\n+2017-01-02 11:47:00,25.4688,85.5625,2.06\n+2017-01-02 12:02:00,24.9875,85.4775,2.05\n+2017-01-02 12:17:00,26.2475,85.395,2.06\n+2017-01-02 12:32:00,27.0387,85.3425,2.08\n+2017-01-02 12:47:00,26.9912,85.35,2.09\n+2017-01-02 13:02:00,26.8425,85.38,2.09\n+2017-01-02 13:17:00,27.1362,85.415,2.1\n+2017-01-02 13:32:00,26.4512,85.45,2.11\n+2017-01-02 13:47:00,26.6,85.4625,2.13\n+2017-01-02 14:02:00,26.6963,85.4325,2.15\n+2017-01-02 14:17:00,26.3975,85.4125,2.15\n+2017-01-02 14:32:00,25.915,85.3775,2.15\n+2017-01-02 14:47:00,27.0375,85.325,2.15\n+2017-01-02 15:02:00,27.135,85.2875,2.16\n+2017-01-02 15:17:00,26.9912,85.2325,2.16\n+2017-01-02 15:32:00,27.0413,85.21,2.13\n+2017-01-02 15:47:00,25.9637,85.205,2.11\n+2017-01-02 16:02:00,24.9863,85.245,2.09\n+2017-01-02 16:17:00,27.7875,85.275,2.08\n+2017-01-02 16:32:00,26.1113,85.3125,2.09\n+2017-01-02 16:47:00,26.11,85.3375,2.09\n+2017-01-02 17:02:00,25.8662,85.3525,2.06\n+2017-01-02 17:17:00,26.8,85.365,2.04\n+2017-01-02 17:32:00,25.6675,85.3625,2.03\n+2017-01-02 17:47:00,26.445,85.3425,2.01\n+2017-01-02 18:02:00,26.6,85.325,2.01\n+2017-01-02 18:17:00,26.3,85.3125,1.99\n+2017-01-02 18:32:00,27.3387,85.265,1.97\n+2017-01-02 18:47:00,26.5987,85.245,1.95\n+2017-01-02 19:02:00,27.4938,85.265,1.95\n+2017-01-02 19:17:00,27.8875,85.3025,1.94\n+2017-01-02 19:32:00,26.0625,85.3425,1.93\n+2017-01-02 19:47:00,26.6987,85.375,1.92\n+2017-01-02 20:02:00,26.745,85.4175,1.91\n+2017-01-02 20:17:00,27.4412,85.4325,1.9\n+2017-01-02 20:32:00,25.1862,85.41,1.91\n+2017-01-02 20:47:00,27.235,85.38,1.9\n+2017-01-02 21:02:00,28.735,85.375,1.88\n+2017-01-02 21:17:00,27.7875,85.3325,1.86\n+2017-01-02 21:32:00,27.5413,85.32,1.85\n+2017-01-02 21:47:00,27.29,85.3275,1.86\n+2017-01-02 22:02:00,26.0625,85.3675,1.85\n+2017-01-02 22:17:00,26.015,85.4025,1.84\n+2017-01-02 22:32:00,28.6338,85.4125,1.83\n+2017-01-02 22:47:00,27.2875,85.4375,1.84\n+2017-01-02 23:02:00,26.8475,85.44,1.84\n+2017-01-02 23:17:00,26.1575,85.4375,1.87\n+2017-01-02 23:32:00,28.4875,85.405,1.86\n+2017-01-02 23:47:00,27.8375,85.37,1.85\n+2017-01-03 00:02:00,26.3475,85.3475,1.85\n+2017-01-03 00:17:00,28.2425,85.305,1.86\n+2017-01-03 00:32:00,27.3875,85.2675,1.86\n+2017-01-03 00:47:00,25.52,85.2625,1.84\n+2017-01-03 01:02:00,26.445,85.2875,1.85\n+2017-01-03 01:17:00,26.25,85.3225,1.85\n+2017-01-03 01:32:00,24.635,85.3575,1.86\n+2017-01-03 01:47:00,26.2487,85.345,1.85\n+2017-01-03 02:02:00,26.6462,85.33,1.84\n+2017-01-03 02:17:00,24.5863,85.3125,1.85\n+2017-01-03 02:32:00,26.3475,85.305,1.86\n+2017-01-03 02:47:00,25.4713,85.2575,1.84\n+2017-01-03 03:02:00,27.235,85.2425,1.86\n\\ No newline at end of file\n' |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test2/maint.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test2/maint.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| @@ -0,0 +1,37 @@ +Timestamp,maint +2016-01-10 11:15:00,2016-01-10 12:15:00 +2016-01-12 14:40:00,2016-01-12 15:30:00 +2016-02-10 13:40:00,2016-02-10 14:40:00 +2016-02-24 16:40:00,2016-02-24 17:30:00 +2016-03-08 09:30:00,2016-03-08 10:30:00 +2016-03-15 12:00:00,2016-03-15 16:30:00 +2016-03-21 11:00:00,2016-03-21 14:00:00 +2016-04-05 07:00:00,2016-04-05 07:55:00 +2016-05-02 15:30:00,2016-05-02 16:30:00 +2016-05-03 15:25:00,2016-05-03 16:35:00 +2016-05-31 07:00:00,2016-05-31 08:25:00 +2016-06-02 13:00:00,2016-06-02 15:50:00 +2016-06-28 07:30:00,2016-06-28 08:45:00 +2016-06-28 09:30:00,2016-06-28 10:30:00 +2016-07-25 08:45:00,2016-07-25 10:00:00 +2016-08-22 08:15:00,2016-08-22 09:20:00 +2016-09-20 07:00:00,2016-09-20 09:15:00 +2016-10-25 08:15:00,2016-10-25 09:50:00 +2016-11-08 09:40:00,2016-11-08 11:50:00 +2016-11-14 09:35:00,2016-11-14 10:40:00 +2016-12-13 15:05:00,2016-12-13 16:50:00 +2017-01-10 08:05:00,2017-01-10 09:15:00 +2017-02-06 09:20:00,2017-02-06 11:00:00 +2017-03-07 08:40:00,2017-03-07 09:40:00 +2017-04-05 07:50:00,2017-04-05 09:20:00 +2017-04-12 07:30:00,2017-04-12 09:45:00 +2017-05-02 13:20:00,2017-05-02 13:30:00 +2017-05-30 07:20:00,2017-05-30 08:30:00 +2017-06-14 08:25:00,2017-06-14 09:30:00 +2017-06-27 07:30:00,2017-06-27 08:40:00 +2017-07-24 08:00:00,2017-07-24 09:00:00 +2017-09-20 14:00:00,2017-09-20 14:35:00 +2017-10-17 08:55:00,2017-10-17 10:20:00 +2017-11-14 15:30:00,2017-11-14 16:20:00 +2017-11-27 09:10:00,2017-11-27 10:10:00 +2017-12-12 14:10:00,2017-12-12 14:50:00 \ No newline at end of file |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test-data/test2/out.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test2/out.csv Sat Aug 16 11:43:23 2025 +0000 |
| b |
| b'@@ -0,0 +1,3193 @@\n+,level_raw,level_raw,maint,maint,sac254_raw,sac254_raw,water_temp_raw,water_temp_raw\n+,data,flags,data,flags,data,flags,data,flags\n+Timestamp,,,,,,,,\n+2016-01-10 11:15:00,nan,nan,2016-01-10 12:15:00,UNFLAGGED,nan,nan,nan,nan\n+2016-01-12 14:40:00,nan,nan,2016-01-12 15:30:00,UNFLAGGED,nan,nan,nan,nan\n+2016-02-10 13:40:00,nan,nan,2016-02-10 14:40:00,UNFLAGGED,nan,nan,nan,nan\n+2016-02-24 16:40:00,nan,nan,2016-02-24 17:30:00,UNFLAGGED,nan,nan,nan,nan\n+2016-03-08 09:30:00,nan,nan,2016-03-08 10:30:00,UNFLAGGED,nan,nan,nan,nan\n+2016-03-15 12:00:00,nan,nan,2016-03-15 16:30:00,UNFLAGGED,nan,nan,nan,nan\n+2016-03-21 11:00:00,nan,nan,2016-03-21 14:00:00,UNFLAGGED,nan,nan,nan,nan\n+2016-04-05 07:00:00,nan,nan,2016-04-05 07:55:00,UNFLAGGED,nan,nan,nan,nan\n+2016-05-02 15:30:00,nan,nan,2016-05-02 16:30:00,UNFLAGGED,nan,nan,nan,nan\n+2016-05-03 15:25:00,nan,nan,2016-05-03 16:35:00,UNFLAGGED,nan,nan,nan,nan\n+2016-05-31 07:00:00,nan,nan,2016-05-31 08:25:00,UNFLAGGED,nan,nan,nan,nan\n+2016-06-02 13:00:00,nan,nan,2016-06-02 15:50:00,UNFLAGGED,nan,nan,nan,nan\n+2016-06-28 07:30:00,nan,nan,2016-06-28 08:45:00,UNFLAGGED,nan,nan,nan,nan\n+2016-06-28 09:30:00,nan,nan,2016-06-28 10:30:00,UNFLAGGED,nan,nan,nan,nan\n+2016-07-25 08:45:00,nan,nan,2016-07-25 10:00:00,UNFLAGGED,nan,nan,nan,nan\n+2016-08-22 08:15:00,nan,nan,2016-08-22 09:20:00,UNFLAGGED,nan,nan,nan,nan\n+2016-09-20 07:00:00,nan,nan,2016-09-20 09:15:00,UNFLAGGED,nan,nan,nan,nan\n+2016-10-25 08:15:00,nan,nan,2016-10-25 09:50:00,UNFLAGGED,nan,nan,nan,nan\n+2016-11-08 09:40:00,nan,nan,2016-11-08 11:50:00,UNFLAGGED,nan,nan,nan,nan\n+2016-11-14 09:35:00,nan,nan,2016-11-14 10:40:00,UNFLAGGED,nan,nan,nan,nan\n+2016-12-01 06:47:00,104.147,UNFLAGGED,nan,nan,23.8925,BAD,2.68,UNFLAGGED\n+2016-12-01 07:02:00,104.175,UNFLAGGED,nan,nan,22.3412,BAD,2.68,UNFLAGGED\n+2016-12-01 07:17:00,104.192,UNFLAGGED,nan,nan,23.5587,BAD,2.7,UNFLAGGED\n+2016-12-01 07:32:00,104.23,UNFLAGGED,nan,nan,23.5575,BAD,2.71,UNFLAGGED\n+2016-12-01 07:47:00,104.245,UNFLAGGED,nan,nan,24.8387,BAD,2.72,UNFLAGGED\n+2016-12-01 08:02:00,104.255,UNFLAGGED,nan,nan,23.2125,BAD,2.75,UNFLAGGED\n+2016-12-01 08:17:00,104.31,UNFLAGGED,nan,nan,23.845,BAD,2.77,UNFLAGGED\n+2016-12-01 08:32:00,104.39,UNFLAGGED,nan,nan,23.2087,BAD,2.8,UNFLAGGED\n+2016-12-01 08:47:00,104.495,UNFLAGGED,nan,nan,24.0362,BAD,2.82,UNFLAGGED\n+2016-12-01 09:02:00,104.577,UNFLAGGED,nan,nan,23.5062,BAD,2.84,UNFLAGGED\n+2016-12-01 09:17:00,104.777,UNFLAGGED,nan,nan,24.3425,BAD,2.88,UNFLAGGED\n+2016-12-01 09:32:00,104.87,UNFLAGGED,nan,nan,24.7375,BAD,2.89,UNFLAGGED\n+2016-12-01 09:47:00,104.967,UNFLAGGED,nan,nan,22.3887,BAD,2.9,UNFLAGGED\n+2016-12-01 10:02:00,105.015,UNFLAGGED,nan,nan,24.29,BAD,2.93,UNFLAGGED\n+2016-12-01 10:17:00,105.022,UNFLAGGED,nan,nan,24.5437,BAD,2.96,UNFLAGGED\n+2016-12-01 10:32:00,104.917,UNFLAGGED,nan,nan,24.085,BAD,2.98,UNFLAGGED\n+2016-12-01 10:47:00,104.802,UNFLAGGED,nan,nan,23.1162,BAD,3.01,UNFLAGGED\n+2016-12-01 11:02:00,104.707,UNFLAGGED,nan,nan,22.8312,BAD,3.03,UNFLAGGED\n+2016-12-01 11:17:00,104.727,UNFLAGGED,nan,nan,23.7562,BAD,3.04,UNFLAGGED\n+2016-12-01 11:32:00,104.722,UNFLAGGED,nan,nan,21.6137,BAD,3.06,UNFLAGGED\n+2016-12-01 11:47:00,104.805,UNFLAGGED,nan,nan,22.9737,BAD,3.09,UNFLAGGED\n+2016-12-01 12:02:00,104.947,UNFLAGGED,nan,nan,23.8925,BAD,3.12,UNFLAGGED\n+2016-12-01 12:17:00,105.042,UNFLAGGED,nan,nan,25.1375,BAD,3.14,UNFLAGGED\n+2016-12-01 12:32:00,105.12,UNFLAGGED,nan,nan,23.0212,BAD,3.14,UNFLAGGED\n+2016-12-01 12:47:00,105.217,UNFLAGGED,nan,nan,22.295,BAD,3.16,UNFLAGGED\n+2016-12-01 13:02:00,105.245,UNFLAGGED,nan,nan,23.2188,BAD,3.17,UNFLAGGED\n+2016-12-01 13:17:00,105.253,UNFLAGGED,nan,nan,23.66,BAD,3.2,UNFLAGGED\n+2016-12-01 13:32:00,105.152,UNFLAGGED,nan,nan,22.8325,BAD,3.21,UNFLAGGED\n+2016-12-01 13:47:00,105.11,UNFLAGGED,nan,nan,23.8913,BAD,3.22,UNFLAGGED\n+2016-12-01 14:02:00,105.065,UNFLAGGED,nan,nan,24.1912,BAD,3.24,UNFLAGGED\n+2016-12-01 14:17:00,105.128,UNFLAGGED,nan,nan,21.9088,BAD,3.25,UNFLAGGED\n+2016-12-01 14:32:00,105.212,UNFLAGGED,nan,nan,21.22'..b'GED\n+2017-01-02 17:32:00,85.3625,UNFLAGGED,nan,nan,25.6675,BAD,2.03,UNFLAGGED\n+2017-01-02 17:47:00,85.3425,UNFLAGGED,nan,nan,26.445,BAD,2.01,UNFLAGGED\n+2017-01-02 18:02:00,85.325,UNFLAGGED,nan,nan,26.6,BAD,2.01,UNFLAGGED\n+2017-01-02 18:17:00,85.3125,UNFLAGGED,nan,nan,26.3,BAD,1.99,UNFLAGGED\n+2017-01-02 18:32:00,85.265,UNFLAGGED,nan,nan,27.3387,BAD,1.97,UNFLAGGED\n+2017-01-02 18:47:00,85.245,UNFLAGGED,nan,nan,26.5987,BAD,1.95,UNFLAGGED\n+2017-01-02 19:02:00,85.265,UNFLAGGED,nan,nan,27.4938,BAD,1.95,UNFLAGGED\n+2017-01-02 19:17:00,85.3025,UNFLAGGED,nan,nan,27.8875,BAD,1.94,UNFLAGGED\n+2017-01-02 19:32:00,85.3425,UNFLAGGED,nan,nan,26.0625,BAD,1.93,UNFLAGGED\n+2017-01-02 19:47:00,85.375,UNFLAGGED,nan,nan,26.6987,BAD,1.92,UNFLAGGED\n+2017-01-02 20:02:00,85.4175,UNFLAGGED,nan,nan,26.745,BAD,1.91,UNFLAGGED\n+2017-01-02 20:17:00,85.4325,UNFLAGGED,nan,nan,27.4412,BAD,1.9,UNFLAGGED\n+2017-01-02 20:32:00,85.41,UNFLAGGED,nan,nan,25.1862,BAD,1.91,UNFLAGGED\n+2017-01-02 20:47:00,85.38,UNFLAGGED,nan,nan,27.235,BAD,1.9,UNFLAGGED\n+2017-01-02 21:02:00,85.375,UNFLAGGED,nan,nan,28.735,BAD,1.88,UNFLAGGED\n+2017-01-02 21:17:00,85.3325,UNFLAGGED,nan,nan,27.7875,BAD,1.86,UNFLAGGED\n+2017-01-02 21:32:00,85.32,UNFLAGGED,nan,nan,27.5413,BAD,1.85,UNFLAGGED\n+2017-01-02 21:47:00,85.3275,UNFLAGGED,nan,nan,27.29,BAD,1.86,UNFLAGGED\n+2017-01-02 22:02:00,85.3675,UNFLAGGED,nan,nan,26.0625,BAD,1.85,UNFLAGGED\n+2017-01-02 22:17:00,85.4025,UNFLAGGED,nan,nan,26.015,BAD,1.84,UNFLAGGED\n+2017-01-02 22:32:00,85.4125,UNFLAGGED,nan,nan,28.6338,BAD,1.83,UNFLAGGED\n+2017-01-02 22:47:00,85.4375,UNFLAGGED,nan,nan,27.2875,BAD,1.84,UNFLAGGED\n+2017-01-02 23:02:00,85.44,UNFLAGGED,nan,nan,26.8475,BAD,1.84,UNFLAGGED\n+2017-01-02 23:17:00,85.4375,UNFLAGGED,nan,nan,26.1575,BAD,1.87,UNFLAGGED\n+2017-01-02 23:32:00,85.405,UNFLAGGED,nan,nan,28.4875,BAD,1.86,UNFLAGGED\n+2017-01-02 23:47:00,85.37,UNFLAGGED,nan,nan,27.8375,BAD,1.85,UNFLAGGED\n+2017-01-03 00:02:00,85.3475,UNFLAGGED,nan,nan,26.3475,BAD,1.85,UNFLAGGED\n+2017-01-03 00:17:00,85.305,UNFLAGGED,nan,nan,28.2425,BAD,1.86,UNFLAGGED\n+2017-01-03 00:32:00,85.2675,UNFLAGGED,nan,nan,27.3875,BAD,1.86,UNFLAGGED\n+2017-01-03 00:47:00,85.2625,UNFLAGGED,nan,nan,25.52,BAD,1.84,UNFLAGGED\n+2017-01-03 01:02:00,85.2875,UNFLAGGED,nan,nan,26.445,BAD,1.85,UNFLAGGED\n+2017-01-03 01:17:00,85.3225,UNFLAGGED,nan,nan,26.25,BAD,1.85,UNFLAGGED\n+2017-01-03 01:32:00,85.3575,UNFLAGGED,nan,nan,24.635,BAD,1.86,UNFLAGGED\n+2017-01-03 01:47:00,85.345,UNFLAGGED,nan,nan,26.2487,BAD,1.85,UNFLAGGED\n+2017-01-03 02:02:00,85.33,UNFLAGGED,nan,nan,26.6462,BAD,1.84,UNFLAGGED\n+2017-01-03 02:17:00,85.3125,UNFLAGGED,nan,nan,24.5863,BAD,1.85,UNFLAGGED\n+2017-01-03 02:32:00,85.305,UNFLAGGED,nan,nan,26.3475,BAD,1.86,UNFLAGGED\n+2017-01-03 02:47:00,85.2575,UNFLAGGED,nan,nan,25.4713,BAD,1.84,UNFLAGGED\n+2017-01-03 03:02:00,85.2425,UNFLAGGED,nan,nan,27.235,BAD,1.86,UNFLAGGED\n+2017-01-10 08:05:00,nan,nan,2017-01-10 09:15:00,UNFLAGGED,nan,nan,nan,nan\n+2017-02-06 09:20:00,nan,nan,2017-02-06 11:00:00,UNFLAGGED,nan,nan,nan,nan\n+2017-03-07 08:40:00,nan,nan,2017-03-07 09:40:00,UNFLAGGED,nan,nan,nan,nan\n+2017-04-05 07:50:00,nan,nan,2017-04-05 09:20:00,UNFLAGGED,nan,nan,nan,nan\n+2017-04-12 07:30:00,nan,nan,2017-04-12 09:45:00,UNFLAGGED,nan,nan,nan,nan\n+2017-05-02 13:20:00,nan,nan,2017-05-02 13:30:00,UNFLAGGED,nan,nan,nan,nan\n+2017-05-30 07:20:00,nan,nan,2017-05-30 08:30:00,UNFLAGGED,nan,nan,nan,nan\n+2017-06-14 08:25:00,nan,nan,2017-06-14 09:30:00,UNFLAGGED,nan,nan,nan,nan\n+2017-06-27 07:30:00,nan,nan,2017-06-27 08:40:00,UNFLAGGED,nan,nan,nan,nan\n+2017-07-24 08:00:00,nan,nan,2017-07-24 09:00:00,UNFLAGGED,nan,nan,nan,nan\n+2017-09-20 14:00:00,nan,nan,2017-09-20 14:35:00,UNFLAGGED,nan,nan,nan,nan\n+2017-10-17 08:55:00,nan,nan,2017-10-17 10:20:00,UNFLAGGED,nan,nan,nan,nan\n+2017-11-14 15:30:00,nan,nan,2017-11-14 16:20:00,UNFLAGGED,nan,nan,nan,nan\n+2017-11-27 09:10:00,nan,nan,2017-11-27 10:10:00,UNFLAGGED,nan,nan,nan,nan\n+2017-12-12 14:10:00,nan,nan,2017-12-12 14:50:00,UNFLAGGED,nan,nan,nan,nan\n\\ No newline at end of file\n' |
| b |
| diff -r 55bbea0cdc60 -r 724dcbb35c9a test_macros.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test_macros.xml Sat Aug 16 11:43:23 2025 +0000 |
| [ |
| b'@@ -0,0 +1,3539 @@\n+<?xml version="1.0" encoding="utf-8"?>\n+<macros>\n+ <xml name="config_tests">\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="breaks" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="flagIsolated" />\n+ <param name="field" value="test_variable" />\n+ <param name="gap_window" value="default_string" />\n+ <param name="group_window" value="default_string" />\n+ <param name="flag" value="255.0" />\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*flagIsolated\\(.*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="breaks" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="flagJumps" />\n+ <param name="field" value="test_variable" />\n+ <param name="thresh" value="1.0" />\n+ <param name="window" value="default_string" />\n+ <param name="min_periods" value="1" />\n+ <param name="flag" value="255.0" />\n+ <param name="dfilter" value="-inf" />\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*flagJumps\\(.*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="breaks" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="flagMissing" />\n+ <param name="field" value="test_variable" />\n+ <param name="flag" value="255.0" />\n+ <param name="dfilter" value="-inf" />\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*flagMissing\\(.*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="changepoints" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="assignChangePointCluster" />\n+ <param name="field" value="test_variable" />\n+ <param name="stat_func" value="default_string" />\n+ <param name="thresh_func" value="default_string" />\n+ <param name="window" value="default_string" />\n+ <param name="min_periods" value="1" />\n+ <param name="reduce_window" value="default_string" />\n+ <param name="reduce_func" value="<function ChangepointsMixin.<lambda> at 0x714978652160>" />\n+ <param name="model_by_resids" value="true" />\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*assignChangePointCluster\\(.*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <t'..b'"test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="tools" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="selectTime" />\n+ <param name="field" value="test_variable" />\n+ <param name="mode" value="selection_field" />\n+ <param name="selection_field" value="default_string" />\n+ <param name="start" value="default_string" />\n+ <param name="end" value="default_string" />\n+ <param name="closed" value="false" />\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*selectTime\\((?=.*mode\\s*=\\s*["\']selection_field["\']).*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="transformation" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="transform" />\n+ <param name="field" value="test_variable" />\n+ <param name="func" value="default_string" />\n+ <conditional name="freq_cond">\n+ <param name="freq_select_type" value="number" />\n+ <param name="freq" value="1.0" />\n+ </conditional>\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*transform\\(.*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="transformation" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="transform" />\n+ <param name="field" value="test_variable" />\n+ <param name="func" value="default_string" />\n+ <conditional name="freq_cond">\n+ <param name="freq_select_type" value="number" />\n+ <param name="freq" value="45.6" />\n+ </conditional>\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*transform\\((?=.*freq\\s*=\\s*45\\.6).*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="data" value="test1/data.csv" ftype="csv" />\n+ <param name="run_test_mode" value="true" />\n+ <repeat name="methods_repeat">\n+ <conditional name="module_cond">\n+ <param name="module_select" value="transformation" />\n+ <conditional name="method_cond">\n+ <param name="method_select" value="transform" />\n+ <param name="field" value="test_variable" />\n+ <param name="func" value="default_string" />\n+ <conditional name="freq_cond">\n+ <param name="freq_select_type" value="offset" />\n+ <param name="freq" value="a_string" />\n+ </conditional>\n+ </conditional>\n+ </conditional>\n+ </repeat>\n+ <output name="config_out" ftype="txt">\n+ <assert_contents>\n+ <has_text_matching expression="test_variable;\\s*transform\\((?=.*freq\\s*=\\s*["\']a_string["\']).*\\)$" />\n+ </assert_contents>\n+ </output>\n+ </test>\n+ </xml>\n+</macros>\n\\ No newline at end of file\n' |
