# HG changeset patch
# User galaxyp
# Date 1718400548 0
# Node ID f3ec820189ca2b56ca648281939dc80f507ab8f5
# Parent 83f5e5c1209c66edcf8ee098ece1a99d44653a03
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 5c080b1e2b99f1c88f4557e9fec8c45c9d23b906
diff -r 83f5e5c1209c -r f3ec820189ca AssayGeneratorMetabo.xml
--- a/AssayGeneratorMetabo.xml Thu Dec 01 18:57:35 2022 +0000
+++ b/AssayGeneratorMetabo.xml Fri Jun 14 21:29:08 2024 +0000
@@ -1,6 +1,5 @@
-
-
+
Assay library generation from DDA data (Metabolomics)
@@ -14,23 +13,19 @@
#import re
## Preprocessing
-#if $sirius_executable:
- mkdir sirius_executable &&
- ln -s '$sirius_executable' 'sirius_executable/${re.sub("[^\w\-_]", "_", $sirius_executable.element_identifier)}.$gxy2omsext($sirius_executable.ext)' &&
-#end if
mkdir in_cond.in &&
#if $in_cond.in_select == "no"
mkdir ${' '.join(["'in_cond.in/%s'" % (i) for i, f in enumerate($in_cond.in) if f])} &&
-${' '.join(["ln -s '%s' 'in_cond.in/%s/%s.%s' && " % (f, i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($in_cond.in) if f])}
+${' '.join(["cp '%s' 'in_cond.in/%s/%s.%s' && " % (f, i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($in_cond.in) if f])}
#else
-ln -s '$in_cond.in' 'in_cond.in/${re.sub("[^\w\-_]", "_", $in_cond.in.element_identifier)}.$gxy2omsext($in_cond.in.ext)' &&
+cp '$in_cond.in' 'in_cond.in/${re.sub("[^\w\-_]", "_", $in_cond.in.element_identifier)}.$gxy2omsext($in_cond.in.ext)' &&
#end if
mkdir in_id_cond.in_id &&
#if $in_id_cond.in_id_select == "no"
mkdir ${' '.join(["'in_id_cond.in_id/%s'" % (i) for i, f in enumerate($in_id_cond.in_id) if f])} &&
-${' '.join(["ln -s '%s' 'in_id_cond.in_id/%s/%s.%s' && " % (f, i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($in_id_cond.in_id) if f])}
+${' '.join(["cp '%s' 'in_id_cond.in_id/%s/%s.%s' && " % (f, i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($in_id_cond.in_id) if f])}
#else
-ln -s '$in_id_cond.in_id' 'in_id_cond.in_id/${re.sub("[^\w\-_]", "_", $in_id_cond.in_id.element_identifier)}.$gxy2omsext($in_id_cond.in_id.ext)' &&
+cp '$in_id_cond.in_id' 'in_id_cond.in_id/${re.sub("[^\w\-_]", "_", $in_id_cond.in_id.element_identifier)}.$gxy2omsext($in_id_cond.in_id.ext)' &&
#end if
mkdir out &&
@@ -40,10 +35,6 @@
@EXECUTABLE@ -write_ctd ./ &&
python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hardcoded_json' &&
@EXECUTABLE@ -ini @EXECUTABLE@.ctd
-#if $sirius_executable:
- -sirius_executable
- 'sirius_executable/${re.sub("[^\w\-_]", "_", $sirius_executable.element_identifier)}.$gxy2omsext($sirius_executable.ext)'
-#end if
-in
#if $in_cond.in_select == "no"
${' '.join(["'in_cond.in/%s/%s.%s'"%(i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($in_cond.in) if f])}
@@ -66,20 +57,19 @@
#end if]]>
-
+
-
-
+
-
+
@@ -88,139 +78,146 @@
-
+
-
+
-
+
-
+
-
-
-
-
+
+
+
+
-
+
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
+
+
+
-
+
-
+
-
+
+
+
+
-
+
-
-
+
+
-
-
+
+
+
-
+
@@ -239,7 +236,8 @@
OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS
-
+
+
@@ -257,7 +256,7 @@
-
+
@@ -296,31 +295,33 @@
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
+
+
+
+
-
+
@@ -347,7 +352,7 @@
-
+
@@ -386,31 +391,33 @@
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
+
+
+
+
-
+
@@ -437,7 +448,7 @@
-
+
@@ -476,31 +487,33 @@
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
+
+
+
+
-
+
@@ -527,7 +544,7 @@
-
+
@@ -566,31 +583,33 @@
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
+
+
+
+
-
+
@@ -617,7 +640,7 @@
-
+
@@ -656,1123 +679,48 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+For more information, visit https://openms.de/doxygen/release/3.1.0/html/TOPP_AssayGeneratorMetabo.html]]>
diff -r 83f5e5c1209c -r f3ec820189ca fill_ctd_clargs.py
--- a/fill_ctd_clargs.py Thu Dec 01 18:57:35 2022 +0000
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-
-import operator
-from argparse import ArgumentParser
-from functools import reduce # forward compatibility for Python 3
-from io import StringIO
-
-from CTDopts.CTDopts import (
- _Null,
- CTDModel,
- ModelTypeError,
- Parameters
-)
-
-
-def getFromDict(dataDict, mapList):
- return reduce(operator.getitem, mapList, dataDict)
-
-
-def setInDict(dataDict, mapList, value):
- getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value
-
-
-if __name__ == "__main__":
- # note add_help=False since otherwise arguments starting with -h will
- # trigger an error (despite allow_abbreviate)
- parser = ArgumentParser(prog="fill_ctd_clargs",
- description="fill command line arguments"
- "into a CTD file and write the CTD file to stdout",
- add_help=False, allow_abbrev=False)
- parser.add_argument("--ini_file", dest="ini_file", help="input ini file",
- metavar='INI', default=None, required=True)
- parser.add_argument("--ctd_file", dest="ctd_file", help="input ctd file"
- "if given then optional parameters from the ini file"
- "will be filled with the defaults from this CTD file",
- metavar='CTD', default=None, required=False)
- args, cliargs = parser.parse_known_args()
-
- # load CTDModel
- ini_model = None
- try:
- ini_model = CTDModel(from_file=args.ini_file)
- except ModelTypeError:
- pass
- try:
- ini_model = Parameters(from_file=args.ini_file)
- except ModelTypeError:
- pass
- assert ini_model is not None, "Could not parse %s, seems to be no CTD/PARAMS" % (args.ini_file)
-
- # get a dictionary of the ctd arguments where the values of the parameters
- # given on the command line are overwritten
- ini_values = ini_model.parse_cl_args(cl_args=cliargs, ignore_required=True)
-
- if args.ctd_file:
- ctd_model = CTDModel(from_file=args.ctd_file)
- ctd_values = ctd_model.get_defaults()
- for param in ini_model.get_parameters():
- if not param.required and (param.default is None or type(param.default) is _Null):
- lineage = param.get_lineage(name_only=True)
- try:
- default = getFromDict(ctd_values, lineage)
- except KeyError:
- continue
- setInDict(ini_values, lineage, default)
-
- # write the ctd with the values taken from the dictionary
- out = StringIO()
- ctd_tree = ini_model.write_ctd(out, ini_values)
- print(out.getvalue())
diff -r 83f5e5c1209c -r f3ec820189ca generate-foo.sh
--- a/generate-foo.sh Thu Dec 01 18:57:35 2022 +0000
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,215 +0,0 @@
-#!/usr/bin/env bash
-
-# parse test definitions from OpenMS sources for a tool with a given id
-function get_tests2 {
- id=$1
- >&2 echo "generate tests for $id"
- echo ''
-
- # get the tests from the CMakeLists.txt
- # 1st remove some tests
- # - OpenSwathMzMLFileCacher with -convert_back argument https://github.com/OpenMS/OpenMS/issues/4399
- # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt
- # - several tools with duplicated input (leads to conflict when linking)
- # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456)
- # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed)
- # - some input files are originally in a subdir (degenerated cases/), but not in test-data
- # - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet)
- # - SiriusAdapter_4 depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
- # - SiriusAdapter_10 should work in >2.8 https://github.com/OpenMS/OpenMS/issues/5869
- CMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake |
- sed 's@${DATA_DIR_SHARE}/@@g' |
- grep -v 'OpenSwathMzMLFileCacher .*-convert_back' |
- sed 's/${TMP_RIP_PATH}/""/' |
- grep -v "MaRaClusterAdapter.*-consensus_out"|
- grep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " |
- sed 's@degenerate_cases/@@g' |
- egrep -v 'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"' |
- sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
- grep -v '"TOPP_SiriusAdapter_10"')
-
- # 1st part is a dirty hack to join lines containing a single function call, e.g.
- # addtest(....
- # ....)
- echo "$CMAKE" | sed 's/#.*//; s/^\s*//; s/\s*$//' | grep -v "^#" | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
- grep -iE "add_test\(\"(TOPP|UTILS)_.*/$id " | egrep -v "_prepare\"|_convert|WRITEINI|WRITECTD|INVALIDVALUE" | while read -r line
- do
- line=$(echo "$line" | sed 's/add_test("\([^"]\+\)"/\1/; s/)$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g')
- # >&2 echo $line
- test_id=$(echo "$line" | cut -d" " -f 1)
- tool_id=$(echo "$line" | cut -d" " -f 2)
- # >&2 echo "test_id $test_id"
- if [[ $test_id =~ _out_?[0-9]? ]]; then
- >&2 echo " skip $test_id $line"
- continue
- fi
- if [[ ${id,,} != ${tool_id,,} ]]; then
- >&2 echo " skip $test_id ($id != $tool_id) $line"
- continue
- fi
-
- #remove tests with set_tests_properties(....PROPERTIES WILL_FAIL 1)
- if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then
- >&2 echo " skip failing "$test_id
- continue
- fi
- tes=" \n"
- line=$(fix_tmp_files "$line")
- line=$(unique_files "$line")
- # >&2 echo LINE $line
- #if there is an ini file then we use this to generate the test
- #otherwise the ctd file is used
- #other command line parameters are inserted later into this xml
- if grep -lq "\-ini" <<<"$line"; then
- ini=$(echo $line | sed 's/.*-ini \([^ ]\+\).*/\1/')
- ini="test-data/$ini"
- else
- ini="ctd/$tool_id.ctd"
- fi
- # >&2 echo "========================================================"
- # >&2 echo "USING ini $ini"
- cli=$(echo $line |cut -d" " -f3- | sed 's/-ini [^ ]\+//')
-
- ctdtmp=$(mktemp)
- # using eval: otherwise for some reason quoted values are not used properly ('A B' -> ["'A", "B'"])
- # >&2 echo "python3 fill_ctd_clargs.py --ini_file $ini $cli"
- eval "python3 fill_ctd_clargs.py --ini_file $ini $cli" > "$ctdtmp"
- # >&2 echo $ctdtmp
- # >&2 cat $ctdtmp
- testtmp=$(mktemp)
- # >&2 echo CTDConverter galaxy -i $ctdtmp -o $testtmp -s aux/tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p aux/hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib --test-condition "compare=sim_size" "delta_frac=0.7"
- CTDConverter galaxy -i $ctdtmp -o $testtmp -s aux/tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p aux/hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib --test-condition "compare=sim_size" "delta_frac=0.7" > /dev/null
- echo ""
- cat $testtmp | grep -v ' /dev/null
-
- #rm $testtmp
- done
- echo ''
-}
-
-#some tests use the same file twice which does not work in planemo tests
-#hence we create symlinks for each file used twice
-function unique_files {
- line=$@
- for arg in $@
- do
- if [[ ! -f "test-data/$arg" ]]; then
- continue
- fi
- cnt=$(grep -c $arg <<< $(echo "$line" | tr ' ' '\n'))
- while [[ $cnt -gt 1 ]]; do
- new_arg=$(echo $arg | sed "s/\(.*\)\./\1_$cnt./")
- ln -fs $arg test-data/$new_arg
- line=$(echo $line | sed "s/\($arg.*\)$arg/\1$new_arg/")
- cnt=$(grep -c $arg <<< $(echo "$line" | tr ' ' '\n'))
- done
- done
-
- echo $line
-}
-
-# options of out_type selects need to be fixed to Galaxy data types
-function fix_out_type {
- grep "^$1" "$2" | awk '{print $2}'
-}
-
-#OpenMS tests output to tmp files and compare with FuzzyDiff to the expected file.
-#problem: the extension of the tmp files is unusable for test generation.
-#unfortunately the extensions used in the DIFF lines are not always usable for the CLI
-#(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt)
-#this function replaces the tmp file by the expected file.
-function fix_tmp_files {
- # >&2 echo "FIX $line"
- ret=""
- for a in $@; do
- # >&2 echo " a "$a
- if [[ ! $a =~ .tmp$ ]] && [[ ! $a =~ _tmp_ ]]; then
- ret="$ret $a"
- continue
- fi
- diff_line=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep '\${DIFF}.*'"$a")
- # >&2 echo " diff_line "$diff_line
- in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$diff_line)
- # >&2 echo " in1 "$in1
- if [[ "$a" != "$in1" ]]; then
- ret="$ret $a"
- continue
- fi
- in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$diff_line)
- in2=$(basename $in2 | sed 's/)$//')
- # >&2 echo " in2 "$in2
- if [[ -f "test-data/$in2" ]]; then
- ln -fs "$in1" "test-data/$in2"
- ret="$ret $in2"
- else
- ret="$ret $a"
- fi
- done
-# >&2 echo "--> $ret"
- echo "$ret"
-}
-
-function link_tmp_files {
- # note this also considers commented lines (starting with a #)
- # because of tests where the diff command is commented and we
- # still want to use the extension of these files
- cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep "\${DIFF}" | while read -r line
- do
- in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$line)
- in1=$(basename $in1 | sed 's/)$//')
- in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$line)
- in2=$(basename $in2 | sed 's/)$//')
- if [[ "$in1" == "$in2" ]]; then
- >&2 echo "not linking equal $in1 $in2"
- continue
- fi
- ln -f -s $in1 test-data/$in2
- done
-
- find test-data/ -name "*.tmp" -print0 |
- while IFS= read -r -d '' i; do
- if [ ! -e test-data/$(basename $i .tmp) ]; then
- ln -s $(basename $i) test-data/$(basename $i .tmp)
- else
- ln -fs $(basename $i) test-data/$(basename $i .tmp)
- fi
- done
-}
-
-
-
-# parse data preparation calls from OpenMS sources for a tool with a given id
-function prepare_test_data {
-# id=$1
-# | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?"
-
- # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
- cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
- sed 's/degenerate_cases\///' |
- egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" |
- grep add_test |
- egrep "TOPP|UTILS" |
- sed 's@${DATA_DIR_SHARE}/@@g;'|
- sed 's@${TMP_RIP_PATH}@./@g'|
- sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'|
- sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
- while read line
- do
- test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1)
-
- if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then
- >&2 echo " skip failing "$test_id
- continue
- fi
-
- line=$(echo "$line" | sed 's/add_test("//; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f2-)
- # line="$(fix_tmp_files $line)"
- echo 'echo executing "'$test_id'"'
- echo "$line > $test_id.stdout 2> $test_id.stderr"
- echo "if [[ \"\$?\" -ne \"0\" ]]; then >&2 echo '$test_id failed'; >&2 echo -e \"stderr:\n\$(cat $test_id.stderr | sed 's/^/ /')\"; echo -e \"stdout:\n\$(cat $test_id.stdout)\";fi"
- done
-}
diff -r 83f5e5c1209c -r f3ec820189ca get_tests.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/get_tests.py Fri Jun 14 21:29:08 2024 +0000
@@ -0,0 +1,344 @@
+#!/usr/bin/env python
+
+import argparse
+import os.path
+import re
+import shlex
+import sys
+import tempfile
+from typing import (
+ Dict,
+ List,
+ Optional,
+ TextIO,
+ Tuple,
+)
+
+from ctdconverter.common.utils import (
+ ParameterHardcoder,
+ parse_hardcoded_parameters,
+ parse_input_ctds,
+)
+from ctdconverter.galaxy.converter import convert_models
+from CTDopts.CTDopts import (
+ CTDModel,
+ ModelTypeError,
+ Parameters,
+)
+
+SKIP_LIST = [
+ r"_prepare\"",
+ r"_convert",
+ r"WRITEINI",
+ r"WRITECTD",
+ r"INVALIDVALUE",
+ r"\.ini\.json",
+ r"OpenSwathMzMLFileCacher .*-convert_back", # - OpenSwathMzMLFileCacher with -convert_back argument https://github.com/OpenMS/OpenMS/issues/4399
+ r"MaRaClusterAdapter.*-consensus_out", # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456)
+ r"FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta ", # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed)
+ r'^(TOPP_OpenSwathAnalyzer_test_3|TOPP_OpenSwathAnalyzer_test_4)$', # no suppert for cached mzML
+ r'TOPP_SiriusAdapter_[0-9]+$', # Do not test SiriusAdapter https://github.com/OpenMS/OpenMS/issues/7000 .. will be removed anyway
+ r'TOPP_AssayGeneratorMetabo_(7|8|9|10|11|12|13|14|15|16|17|18)$' # Skip AssayGeneratorMetabo tests using Sirius https://github.com/OpenMS/OpenMS/issues/7150 (will be replaced by two tools)
+]
+
+
+def get_failing_tests(cmake: List[str]) -> List[str]:
+ failing_tests = []
+ re_fail = re.compile(r"set_tests_properties\(\"([^\"]+)\" PROPERTIES WILL_FAIL 1\)")
+
+ for cmake in args.cmake:
+ with open(cmake) as cmake_fh:
+ for line in cmake_fh:
+ match = re_fail.search(line)
+ if match:
+ failing_tests.append(match.group(1))
+ return failing_tests
+
+
+def fix_tmp_files(line: str, diff_pairs: Dict[str, str]) -> str:
+ """
+ OpenMS tests output to tmp files and compare with FuzzyDiff to the expected file.
+ problem: the extension of the tmp files is unusable for test generation.
+ unfortunately the extensions used in the DIFF lines are not always usable for the CLI
+ (e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt)
+ this function replaces the tmp file by the expected file.
+ """
+ cmd = shlex.split(line)
+ for i, e in enumerate(cmd):
+ if e in diff_pairs:
+ dst = os.path.join("test-data", diff_pairs[e])
+ if os.path.exists(dst):
+ os.unlink(dst)
+ sys.stderr.write(f"symlink {e} {dst}\n")
+ os.symlink(e, dst)
+ cmd[i] = diff_pairs[e]
+ return shlex.join(cmd)
+
+
+def get_ini(line: str, tool_id: str) -> Tuple[str, str]:
+ """
+ if there is an ini file then we use this to generate the test
+ otherwise the ctd file is used
+ other command line parameters are inserted later into this xml
+ """
+ cmd = shlex.split(line)
+ ini = None
+ for i, e in enumerate(cmd):
+ if e == "-ini":
+ ini = cmd[i + 1]
+ cmd = cmd[:i] + cmd[i + 2:]
+ if ini:
+ return os.path.join("test-data", ini), shlex.join(cmd)
+ else:
+ return os.path.join("ctd", f"{tool_id}.ctd"), line
+
+
+def unique_files(line: str):
+ """
+ some tests use the same file twice which does not work in planemo tests
+ hence we create symlinks for each file used twice
+ """
+ cmd = shlex.split(line)
+ # print(f"{cmd}")
+ files = {}
+ # determine the list of indexes where each file argument (anything appearing in test-data/) appears
+ for idx, e in enumerate(cmd):
+ p = os.path.join("test-data", e)
+ if not os.path.exists(p) and not os.path.islink(p):
+ continue
+ try:
+ files[e].append(idx)
+ except KeyError:
+ files[e] = [idx]
+ # print(f"{files=}")
+ for f in files:
+ if len(files[f]) < 2:
+ continue
+ for i, idx in enumerate(files[f]):
+ f_parts = f.split(".")
+ f_parts[0] = f"{f_parts[0]}_{i}"
+ new_f = ".".join(f_parts)
+ # if os.path.exists(os.path.join("test-data", new_f)):
+ # os.unlink(os.path.join("test-data", new_f))
+ sys.stderr.write(
+ f'\tsymlink {os.path.join("test-data", new_f)} {f}\n'
+ )
+ try:
+ os.symlink(f, os.path.join("test-data", new_f))
+ except FileExistsError:
+ pass
+ cmd[idx] = new_f
+ return shlex.join(cmd)
+
+
+def fill_ctd_clargs(ini: str, line: str, ctd_tmp: TextIO) -> None:
+ cmd = shlex.split(line)
+
+ # load CTDModel
+ ini_model = None
+ try:
+ ini_model = CTDModel(from_file=ini)
+ except ModelTypeError:
+ pass
+ try:
+ ini_model = Parameters(from_file=ini)
+ except ModelTypeError:
+ pass
+ assert ini_model is not None, "Could not parse %s, seems to be no CTD/PARAMS" % (
+ args.ini_file
+ )
+
+ # get a dictionary of the ctd arguments where the values of the parameters
+ # given on the command line are overwritten
+ ini_values = ini_model.parse_cl_args(cl_args=cmd, ignore_required=True)
+ ini_model.write_ctd(ctd_tmp, ini_values)
+
+
+def process_test_line(
+ id: str,
+ line: str,
+ failing_tests: List[str],
+ skip_list: List[str],
+ diff_pairs: Dict[str, str],
+) -> Optional[str]:
+
+ re_test_id = re.compile(r"add_test\(\"([^\"]+)\" ([^ ]+) (.*)")
+ re_id_out_test = re.compile(r"_out_?[0-9]?")
+
+ # TODO auto extract from set(OLD_OSW_PARAM ... lin
+ line = line.replace(
+ "${OLD_OSW_PARAM}",
+ " -test -mz_extraction_window 0.05 -mz_extraction_window_unit Th -ms1_isotopes 0 -Scoring:TransitionGroupPicker:compute_peak_quality -Scoring:Scores:use_ms1_mi false -Scoring:Scores:use_mi_score false",
+ )
+
+ line = line.replace("${TOPP_BIN_PATH}/", "")
+ line = line.replace("${DATA_DIR_TOPP}/", "")
+ line = line.replace("THIRDPARTY/", "")
+ line = line.replace("${DATA_DIR_SHARE}/", "")
+ # IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differently
+ line = line.replace("${TMP_RIP_PATH}/", "")
+ # some input files are originally in a subdir (degenerated cases/), but not in test-data
+ line = line.replace("degenerate_cases/", "")
+ # determine the test and tool ids and remove the 1) add_test("TESTID" 2) trailing )
+ match = re_test_id.match(line)
+ if not match:
+ sys.exit(f"Ill formated test line {line}\n")
+ test_id = match.group(1)
+ tool_id = match.group(2)
+
+ line = f"{match.group(2)} {match.group(3)}"
+
+ if test_id in failing_tests:
+ sys.stderr.write(f" skip failing {test_id} {line}\n")
+ return
+
+ if id != tool_id:
+ sys.stderr.write(f" skip {test_id} ({id} != {tool_id}) {line}\n")
+ return
+
+ if re_id_out_test.search(test_id):
+ sys.stderr.write(f" skip {test_id} {line}\n")
+ return
+
+ for skip in skip_list:
+ if re.search(skip, line):
+ return
+ if re.search(skip, test_id):
+ return
+
+ line = fix_tmp_files(line, diff_pairs)
+ # print(f"fix {line=}")
+ line = unique_files(line)
+ # print(f"unq {line=}")
+ ini, line = get_ini(line, tool_id)
+
+ from dataclasses import dataclass, field
+
+ @dataclass
+ class CTDConverterArgs:
+ input_files: list
+ output_destination: str
+ default_executable_path: Optional[str] = None
+ hardcoded_parameters: Optional[str] = None
+ parameter_hardcoder: Optional[ParameterHardcoder] = None
+ xsd_location: Optional[str] = None
+ formats_file: Optional[str] = None
+ add_to_command_line: str = ""
+ required_tools_file: Optional[str] = None
+ skip_tools_file: Optional[str] = None
+ macros_files: Optional[List[str]] = field(default_factory=list)
+ test_macros_files: Optional[List[str]] = field(default_factory=list)
+ test_macros_prefix: Optional[List[str]] = field(default_factory=list)
+ test_test: bool = False
+ test_only: bool = False
+ test_unsniffable: Optional[List[str]] = field(default_factory=list)
+ test_condition: Optional[List[str]] = ("compare=sim_size", "delta_frac=0.05")
+ tool_version: str = None
+ tool_profile: str = None
+ bump_file: str = None
+
+ # create an ini/ctd file where the values are equal to the arguments from the command line
+ # and transform it to xml
+ test = [f"\n"]
+ with tempfile.NamedTemporaryFile(
+ mode="w+", delete_on_close=False
+ ) as ctd_tmp, tempfile.NamedTemporaryFile(
+ mode="w+", delete_on_close=False
+ ) as xml_tmp:
+ fill_ctd_clargs(ini, line, ctd_tmp)
+ ctd_tmp.close()
+ xml_tmp.close()
+ parsed_ctd = parse_input_ctds(None, [ctd_tmp.name], xml_tmp.name, "xml")
+ ctd_args = CTDConverterArgs(
+ input_files=[ctd_tmp.name],
+ output_destination=xml_tmp.name,
+ macros_files=["macros.xml"],
+ skip_tools_file="aux/tools_blacklist.txt",
+ formats_file="aux/filetypes.txt",
+ # tool_conf_destination = "tool.conf",
+ hardcoded_parameters="aux/hardcoded_params.json",
+ tool_version="3.1",
+ test_only=True,
+ test_unsniffable=[
+ "csv",
+ "tsv",
+ "txt",
+ "dta",
+ "dta2d",
+ "edta",
+ "mrm",
+ "splib",
+ ],
+ test_condition=["compare=sim_size", "delta_frac=0.7"],
+ )
+ ctd_args.parameter_hardcoder = parse_hardcoded_parameters(
+ ctd_args.hardcoded_parameters
+ )
+ convert_models(ctd_args, parsed_ctd)
+ xml_tmp = open(xml_tmp.name, "r")
+ for l in xml_tmp:
+ test.append(l)
+
+ return "".join(test)
+
+
+parser = argparse.ArgumentParser(description="Create Galaxy tests for a OpenMS tools")
+parser.add_argument("--id", dest="id", help="tool id")
+parser.add_argument("--cmake", nargs="+", help="OpenMS test CMake files")
+args = parser.parse_args()
+sys.stderr.write(f"generate tests for {args.id}\n")
+
+re_comment = re.compile("#.*")
+re_empty_prefix = re.compile(r"^\s*")
+re_empty_suffix = re.compile(r"\s*$")
+re_add_test = re.compile(r"add_test\(\"(TOPP|UTILS)_.*/" + args.id)
+re_diff = re.compile(r"\$\{DIFF\}.* -in1 ([^ ]+) -in2 ([^ ]+)")
+failing_tests = get_failing_tests(args.cmake)
+tests = []
+
+# process the given CMake files and compile lists of
+# - test lines .. essentially add_test(...)
+# - and pairs of files that are diffed
+jline = ""
+test_lines = []
+diff_pairs = {}
+for cmake in args.cmake:
+ with open(cmake) as cmake_fh:
+ for line in cmake_fh:
+ # remove comments, empty prefixes and suffixes
+ line = re_comment.sub("", line)
+ line = re_empty_prefix.sub("", line)
+ line = re_empty_suffix.sub("", line)
+ # skip empty lines
+ if line == "":
+ continue
+
+ # join test statements that are split over multiple lines
+ if line.endswith(")"):
+ jline += " " + line[:-1]
+ else:
+ jline = line
+ continue
+ line, jline = jline.strip(), ""
+ match = re_diff.search(line)
+ if match:
+ in1 = match.group(1).split("/")[-1]
+ in2 = match.group(2).split("/")[-1]
+ if in1 != in2:
+ diff_pairs[in1] = in2
+ elif re_add_test.match(line):
+ test_lines.append(line)
+
+for line in test_lines:
+ test = process_test_line(args.id, line, failing_tests, SKIP_LIST, diff_pairs)
+ if test:
+ tests.append(test)
+
+tests = "\n".join(tests)
+print(
+ f"""
+
+{tests}
+
+"""
+)
diff -r 83f5e5c1209c -r f3ec820189ca macros.xml
--- a/macros.xml Thu Dec 01 18:57:35 2022 +0000
+++ b/macros.xml Fri Jun 14 21:29:08 2024 +0000
@@ -3,19 +3,19 @@
You can edit this file to add your own macros, if you so desire, or you can
add additional macro files using the m/macros parameter -->
- 2.8
+ 3.1
0
+
openms
openms-thirdparty
- omssa
- blast
+ blast
- ctdopts
+ ctdopts
@@ -26,6 +26,8 @@
+
+
@@ -113,11 +115,11 @@
diff -r 83f5e5c1209c -r f3ec820189ca prepare_test_data_manual.sh
--- a/prepare_test_data_manual.sh Thu Dec 01 18:57:35 2022 +0000
+++ b/prepare_test_data_manual.sh Fri Jun 14 21:29:08 2024 +0000
@@ -1,8 +1,3 @@
-MSSimulator -test -in DecoyDatabase_1.fasta -out MSsimulator.mzml -algorithm:RandomNumberGenerators:biological reproducible -algorithm:RandomNumberGenerators:technical reproducible > MSSimulator_1.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'MSSimulator_1 failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
-MSSimulator -test -in DecoyDatabase_1.fasta -out MSsimulator_MALDI.mzml -algorithm:RandomNumberGenerators:biological reproducible -algorithm:RandomNumberGenerators:technical reproducible -algorithm:MSSim:Global:ionization_type MALDI > MSSimulator_2.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'MSSimulator_2 failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
ClusterMassTracesByPrecursor -test -in_ms1 ConsensusMapNormalizer_input.consensusXML -in_swath ConsensusMapNormalizer_input.consensusXML -out ClusterMassTracesByPrecursor.mzml > ClusterMassTracesByPrecursor.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'ClusterMassTracesByPrecursor failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
@@ -13,8 +8,7 @@
CVInspector -test -cv_files CHEMISTRY/XLMOD.obo -cv_names XLMOD -mapping_file MAPPING/ms-mapping.xml -html CVInspector.html > CVInspector.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'CVInspector failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-DeMeanderize -test -in MSsimulator_MALDI.mzml -out DeMeanderize.mzml > DeMeanderize.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'DeMeanderize failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
+# TODO DeMeanderize
# TODO DigestorMotif
@@ -30,26 +24,17 @@
if [[ "$?" -ne "0" ]]; then >&2 echo 'FeatureFinderIsotopeWavelet failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-FFEval -test -in FeatureFinderCentroided_1_output.featureXML -truth FeatureFinderCentroided_1_output.featureXML -out FFEval.featureXML -out_roc FFEval_roc.csv > FFEval.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'FFEval failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
# TODO? deprecated IDDecoyProbability
IDExtractor -test -in MSGFPlusAdapter_1_out.idXML -best_hits -number_of_peptides 1 -out IDExtractor.idXML > IDExtractor.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'IDExtractor failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-LabeledEval -test -in FeatureLinkerLabeled_1_input.featureXML -truth FeatureLinkerLabeled_1_output.consensusXML> LabeledEval.txt > LabeledEval.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'LabeledEval failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
MapStatistics -test -in SiriusAdapter_3_input.featureXML -out MapStatistics.txt > MapStatistics_1.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'MapStatistics_1 failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
MapStatistics -test -in ConsensusXMLFile_1.consensusXML -out MapStatistics2.txt > MapStatistics_2.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'MapStatistics_2 failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-MetaboliteAdductDecharger -test -in Decharger_input.featureXML -out_cm MetaboliteAdductDecharger_cm.consensusXML -out_fm MetaboliteAdductDecharger_fm.featureXML -outpairs MetaboliteAdductDecharger_pairs.consensusXML > MetaboliteAdductDecharger.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'MetaboliteAdductDecharger failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
MetaboliteSpectralMatcher -test -in spectra.mzML -database MetaboliteSpectralDB.mzML -out MetaboliteSpectralMatcher.mzTab > MetaboliteSpectralMatcher.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'MetaboliteSpectralMatcher failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
@@ -68,20 +53,6 @@
OpenSwathRewriteToFeatureXML -featureXML OpenSwathFeatureXMLToTSV_input.featureXML -out OpenSwathRewriteToFeatureXML.featureXML > OpenSwathRewriteToFeatureXML.stdout 2> stderr
# if [[ "$?" -ne "0" ]]; then >&2 echo 'OpenSwathRewriteToFeatureXML failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-# adapted from the commented tests in OpenMS TODO may be removed later https://github.com/OpenMS/OpenMS/issues/4719
-FileConverter -in PepNovo.mzXML -out PepNovo_1.mzML > /dev/null 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'FileConverter failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
-PepNovoAdapter -ini PepNovoAdapter_1_parameters.ini -in PepNovo_1.mzML -out PepNovoAdapter_3_output.idXML -model_directory pepnovo_models/ -pepnovo_executable pepnovo > PepNovo_1.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'PhosphoScoring failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
-FileConverter -in PepNovo.mzData -out PepNovo_4.mzML > /dev/null 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'FileConverter failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-PepNovoAdapter -ini PepNovoAdapter_1_parameters.ini -in PepNovo_4.mzML -out PepNovoAdapter_4_output.idXML -model_directory pepnovo_models/ -pepnovo_executable pepnovo > PepNovo_1.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'PhosphoScoring failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
-#PepNovoAdapter -ini PepNovoAdapter_5_parameters.ini -in PepNovoAdapter_5_output.pepnovo_out -out PepNovoAdapter_5_output.idXML -model_directory pepnovo_models/
-
# TODO PhosphoScoring
PhosphoScoring -in spectra.mzML -id MSGFPlusAdapter_1_out1.tmp -out PhosphoScoring.idxml > PhosphoScoring.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'PhosphoScoring failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
@@ -118,9 +89,6 @@
RNPxlXICFilter -test -control FileFilter_1_input.mzML -treatment FileFilter_1_input.mzML -out RNPxlXICFilter.mzML > RNPxlXICFilter.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'RNPxlXICFilter failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-RTEvaluation -in PeptideIndexer_1.idXML -out RTEvaluation.tsv > RTEvaluation.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'RTEvaluation failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
SemanticValidator -test -in FileFilter_1_input.mzML -mapping_file MAPPING/ms-mapping.xml > SemanticValidator.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'SemanticValidator failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
@@ -152,13 +120,8 @@
SpectraFilterThresholdMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterThresholdMower.mzML > SpectraFilterThresholdMower.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'SpectraFilterThresholdMower failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-SpectraMerger -test -in NovorAdapter_in.mzML -out SpectraMerger_1.mzML > SpectraMerger.stdout 2> stderr
+SpectraMerger -test -in NovorAdapter_in.mzML -out SpectraMerger_1.mzML -algorithm:average_gaussian:ms_level 2 > SpectraMerger.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'SpectraMerger failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-# TODO SvmTheoreticalSpectrumGeneratorTrainer
-
-TransformationEvaluation -test -in FileInfo_16_input.trafoXML -out TransformationEvaluation.trafoXML > TransformationEvaluation.stdout 2> stderr
-if [[ "$?" -ne "0" ]]; then >&2 echo 'TransformationEvaluation failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
-
XMLValidator -test -in FileFilter_1_input.mzML > XMLValidator.stdout 2> stderr
if [[ "$?" -ne "0" ]]; then >&2 echo 'XMLValidator failed'; >&2 echo -e "stderr:\n$(cat stderr | sed 's/^/ /')"; fi
diff -r 83f5e5c1209c -r f3ec820189ca readme.md
--- a/readme.md Thu Dec 01 18:57:35 2022 +0000
+++ b/readme.md Fri Jun 14 21:29:08 2024 +0000
@@ -11,7 +11,7 @@
* https://www.openms.de/
The wrappers for these tools and most of their tests are automatically
-generated using the `generate.sh` script. The generation of the tools is
+generated using the `./aux/generate.sh` script. The generation of the tools is
based on the CTDConverter (https://github.com/WorkflowConversion/CTDConverter)
which can be fine tuned via the `hardcoded_params.json` file. This file allows
to blacklist and hardcode parameters and to modify or set arbitrary
@@ -129,15 +129,6 @@
Open problems
=============
-Some tools stall in CI testing using `--biocontainers` which is why the OpenMS
-tools are currently listed in `.tt_biocontainer_skip`. This is
-
-- AssayGeneratorMetabo and SiriusAdapter (both depend on sirius)
-- OMSSAAdapter
-
-Using `docker -t` seems to solve the problem (see
-https://github.com/galaxyproject/galaxy/issues/10153).
-
Licence (MIT)
=============
diff -r 83f5e5c1209c -r f3ec820189ca test-data.sh
--- a/test-data.sh Thu Dec 01 18:57:35 2022 +0000
+++ b/test-data.sh Fri Jun 14 21:29:08 2024 +0000
@@ -1,11 +1,10 @@
#!/usr/bin/env bash
-VERSION=2.8
+# set -x
+
+VERSION=3.1
FILETYPES="aux/filetypes.txt"
-CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2"
-
-# import the magic
-. ./generate-foo.sh
+CONDAPKG="https://anaconda.org/bioconda/openms/3.1.0/download/linux-64/openms-3.1.0-h8964181_1.tar.bz2"
# install conda
if [ -z "$tmp" ]; then
@@ -45,17 +44,20 @@
echo "Clone OpenMS $VERSION sources"
if [[ ! -d $OPENMSGIT ]]; then
- # TODO >2.8 reenable original release branch .. also in else branch
- # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917
- # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
- git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT
- cd $OPENMSGIT
- git submodule init
- git submodule update
- cd -
+ if [[ "$created" == "yes" ]]; then
+ GIT_DIR=$(mktemp -d --dry-run)
+ GIT_EXTRA_OPTS="--separate-git-dir=$GIT_DIR"
+ fi
+ git clone -b release/$VERSION.0 --depth 1 --recurse-submodules=THIRDPARTY --shallow-submodules $GIT_EXTRA_OPTS https://github.com/OpenMS/OpenMS.git $OPENMSGIT
+ ## save some space by just keeping the needed binaries
+ find $OPENMSGIT/THIRDPARTY/ -type f -not \( -name maracluster -o -name spectrast \) -delete
+ find $OPENMSGIT/THIRDPARTY/ -empty -type d -delete
+ if [[ "$created" == "yes" ]]; then
+ rm -rf $GIT_DIR
+ fi
else
cd $OPENMSGIT
- git pull origin release/$VERSION.0-plus
+ git pull origin release/$VERSION.0
cd -
fi
@@ -65,7 +67,7 @@
if conda env list | grep "$OPENMSENV"; then
true
else
- conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml
+ conda create -y --quiet --solver libmamba --override-channels --strict-channel-priority --channel conda-forge --channel bioconda -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.5 lxml
# chmod -R u-w $OPENMSENV
fi
###############################################################################
@@ -101,33 +103,34 @@
conda deactivate
-# ###############################################################################
-# ## copy all the test data files to test-data
-# ## most of it (outputs) will be overwritten later, but its needed for
-# ## prepare_test_data
-# ###############################################################################
+# # ###############################################################################
+# # ## copy all the test data files to test-data
+# # ## most of it (outputs) will be overwritten later, but its needed for
+# # ## prepare_test_data
+# # ###############################################################################
echo "Get test data"
-find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete
+find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" ! -name MetaboliteSpectralDB.mzML -delete
cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/
cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then
- wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
+ wget -nc https://raw.githubusercontent.com/sneumann/OpenMS/master/share/OpenMS/CHEMISTRY/MetaboliteSpectralDB.mzML
+ # wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
mv MetaboliteSpectralDB.mzML test-data/
fi
ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
-if [ ! -d test-data/pepnovo_models/ ]; then
- mkdir -p /tmp/pepnovo
- wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
- unzip PepNovo.20120423.zip -d /tmp/pepnovo/
- mv /tmp/pepnovo/Models test-data/pepnovo_models/
- rm PepNovo.20120423.zip
- rm -rf /tmp/pepnovo
-fi
+# if [ ! -d test-data/pepnovo_models/ ]; then
+# mkdir -p /tmp/pepnovo
+# wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
+# unzip PepNovo.20120423.zip -d /tmp/pepnovo/
+# mv /tmp/pepnovo/Models test-data/pepnovo_models/
+# rm PepNovo.20120423.zip
+# rm -rf /tmp/pepnovo
+# fi
###############################################################################
## generate ctd files using the binaries in the conda package
###############################################################################
@@ -170,6 +173,47 @@
###############################################################################
## create script to create results for the tests and run it
###############################################################################
+# parse data preparation calls from OpenMS sources for a tool with a given id
+function prepare_test_data {
+# id=$1
+# | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?"
+
+ OLD_OSW_PARAM=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt |sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' |awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep OLD_OSW_PARAM | head -n 1 | sed 's/^[^"]\+//; s/)$//; s/"//g')
+ # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010
+ cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake |
+ sed "s/\${OLD_OSW_PARAM}/$OLD_OSW_PARAM/" |
+ grep -v "\.ini\.json" |
+ sed 's/.ini.json /ini /' |
+ sed 's/#.*$//'|
+ sed 's/^\s*//; s/\s*$//' |
+ grep -v "^$" |
+ awk '{printf("%s@NEWLINE@", $0)}' |
+ sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' |
+ sed 's/degenerate_cases\///' |
+ egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" |
+ grep add_test |
+ egrep "TOPP|UTILS" |
+ sed 's@${DATA_DIR_SHARE}/@@g;'|
+ sed 's@${TMP_RIP_PATH}@./@g'|
+ sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'|
+ sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' |
+ while read line
+ do
+ test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1)
+
+ if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then
+ >&2 echo " skip failing "$test_id
+ continue
+ fi
+
+ line=$(echo "$line" | sed 's/add_test("//; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f2-)
+ # line="$(fix_tmp_files $line)"
+ echo 'echo executing "'$test_id'"'
+ echo "$line > $test_id.stdout 2> $test_id.stderr"
+ echo "if [[ \"\$?\" -ne \"0\" ]]; then >&2 echo '$test_id failed'; >&2 echo -e \"stderr:\n\$(cat $test_id.stderr | sed 's/^/ /')\"; echo -e \"stdout:\n\$(cat $test_id.stdout)\";fi"
+ done
+}
+
echo "Create test shell script"
echo -n "" > prepare_test_data.sh
@@ -180,31 +224,19 @@
echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh
echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh
-echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"'>> prepare_test_data.sh
+echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-3.5/MSFragger-3.5.jar"'>> prepare_test_data.sh
echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh
echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh
echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh
-echo 'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"'>> prepare_test_data.sh
echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh
echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh
echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh
echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh
echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh
+echo 'export SAGE_BINARY=sage' >> prepare_test_data.sh
prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh
-## prepare_test_data > tmp_test_data.sh
-## # remove calls not needed for the tools listed in any .list file
-## echo LIST $LIST
-## if [ ! -z "$LIST" ]; then
-## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//')
-## else
-## REX=".*"
-## fi
-## echo REX $REX
-## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh
-## rm tmp_test_data.sh
-
echo "Execute test shell script"
chmod u+x prepare_test_data.sh
cd ./test-data || exit
@@ -234,14 +266,11 @@
for i in $(ls ctd/*ctd)
do
b=$(basename "$i" .ctd)
- get_tests2 "$b" >> "$autotests"
+ ./get_tests.py --id "$b" --cmake "$OPENMSGIT"/src/tests/topp/CMakeLists.txt "$OPENMSGIT"/src/tests/topp/THIRDPARTY/third_party_tests.cmake >> "$autotests"
+ wc -l "$autotests"
done
echo "" >> "$autotests"
-# echo "Create test data links"
-# Breaks DecoyDatabase
-# link_tmp_files
-
# tests for tools using output_prefix parameters can not be auto generated
# hence we output the tests for manual curation in macros_test.xml
# and remove them from the autotests
@@ -255,6 +284,7 @@
#
# not able to specify composite test data
# -> SpectraSTSearchAdapter
+echo "Discard some tests"
if [[ ! -z "$1" ]]; then
echo "" > macros_discarded_auto.xml
for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter
@@ -272,7 +302,7 @@
## remove broken symlinks in test-data
find test-data/ -xtype l -delete
-if [ ! -z "$created" ]; then
+if [[ "$created" == "yes" ]]; then
echo "Removing temporary directory"
rm -rf "$tmp"
fi