Mercurial > repos > galaxyp > openms_peakpickerhires
comparison generate-foo.sh @ 10:6b01678ec2fc draft
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author | galaxyp |
---|---|
date | Thu, 01 Dec 2022 19:20:52 +0000 |
parents | 146b59d2d3e5 |
children |
comparison
equal
deleted
inserted
replaced
9:decc397c3b7e | 10:6b01678ec2fc |
---|---|
6 >&2 echo "generate tests for $id" | 6 >&2 echo "generate tests for $id" |
7 echo '<xml name="autotest_'"$id"'">' | 7 echo '<xml name="autotest_'"$id"'">' |
8 | 8 |
9 # get the tests from the CMakeLists.txt | 9 # get the tests from the CMakeLists.txt |
10 # 1st remove some tests | 10 # 1st remove some tests |
11 # - OpenSwathMzMLFileCacher with -convert_back argumen https://github.com/OpenMS/OpenMS/issues/4399 | 11 # - OpenSwathMzMLFileCacher with -convert_back argument https://github.com/OpenMS/OpenMS/issues/4399 |
12 # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt | 12 # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt |
13 # - several tools with duplicated input (leads to conflict when linking) | 13 # - several tools with duplicated input (leads to conflict when linking) |
14 # - TOFCalibration inputs we extension (also in prepare_test_data) https://github.com/OpenMS/OpenMS/pull/4525 | |
15 # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456) | 14 # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456) |
16 # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed) | 15 # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed) |
17 # - some input files are originally in a subdir (degenerated cases/), but not in test-data | 16 # - some input files are originally in a subdir (degenerated cases/), but not in test-data |
18 # - SeedListGenerator: https://github.com/OpenMS/OpenMS/issues/4404 | |
19 # - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet) | 17 # - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet) |
20 # - FeatureFinderIdentification name clash of two tests https://github.com/OpenMS/OpenMS/pull/5002 | 18 # - SiriusAdapter_4 depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010 |
21 # - TODO SiriusAdapter https://github.com/OpenMS/OpenMS/pull/5010 | 19 # - SiriusAdapter_10 should work in >2.8 https://github.com/OpenMS/OpenMS/issues/5869 |
22 CMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | | 20 CMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | |
23 sed 's@${DATA_DIR_SHARE}/@@g' | | 21 sed 's@${DATA_DIR_SHARE}/@@g' | |
24 grep -v 'OpenSwathMzMLFileCacher .*-convert_back' | | 22 grep -v 'OpenSwathMzMLFileCacher .*-convert_back' | |
25 sed 's/${TMP_RIP_PATH}/""/' | | 23 sed 's/${TMP_RIP_PATH}/""/' | |
26 grep -v "MaRaClusterAdapter.*-consensus_out"| | 24 grep -v "MaRaClusterAdapter.*-consensus_out"| |
27 grep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " | | 25 grep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " | |
28 sed 's@degenerate_cases/@@g' | | 26 sed 's@degenerate_cases/@@g' | |
29 grep -v 'TOPP_SeedListGenerator_3"' | | |
30 egrep -v 'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"' | | 27 egrep -v 'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"' | |
31 egrep -v '"TOPP_FeatureFinderIdentification_4"' | | 28 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' | |
32 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/') | 29 grep -v '"TOPP_SiriusAdapter_10"') |
33 | |
34 | |
35 # grep -v 'FileFilter.*-spectra:select_polarity ""' | | |
36 # grep -v 'MassTraceExtractor_2.ini ' | | |
37 # grep -v "FileMerger_6_input2.mzML.*FileMerger_6_input2.mzML" | | |
38 # grep -v "IDMerger_1_input1.idXML.*IDMerger_1_input1.idXML" | | |
39 # grep -v "degenerated_empty.idXML.*degenerated_empty.idXML" | | |
40 # grep -v "FeatureLinkerUnlabeledKD_1_output.consensusXML.*FeatureLinkerUnlabeledKD_1_output.consensusXML" | | |
41 # grep -v "FeatureLinkerUnlabeledQT_1_output.consensusXML.*FeatureLinkerUnlabeledQT_1_output.consensusXML" | | |
42 | 30 |
43 # 1st part is a dirty hack to join lines containing a single function call, e.g. | 31 # 1st part is a dirty hack to join lines containing a single function call, e.g. |
44 # addtest(.... | 32 # addtest(.... |
45 # ....) | 33 # ....) |
46 echo "$CMAKE" | sed 's/#.*//; s/^\s*//; s/\s*$//' | grep -v "^#" | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | | 34 echo "$CMAKE" | sed 's/#.*//; s/^\s*//; s/\s*$//' | grep -v "^#" | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | |
48 do | 36 do |
49 line=$(echo "$line" | sed 's/add_test("\([^"]\+\)"/\1/; s/)$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g') | 37 line=$(echo "$line" | sed 's/add_test("\([^"]\+\)"/\1/; s/)$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g') |
50 # >&2 echo $line | 38 # >&2 echo $line |
51 test_id=$(echo "$line" | cut -d" " -f 1) | 39 test_id=$(echo "$line" | cut -d" " -f 1) |
52 tool_id=$(echo "$line" | cut -d" " -f 2) | 40 tool_id=$(echo "$line" | cut -d" " -f 2) |
41 # >&2 echo "test_id $test_id" | |
53 if [[ $test_id =~ _out_?[0-9]? ]]; then | 42 if [[ $test_id =~ _out_?[0-9]? ]]; then |
54 >&2 echo " skip $test_id $line" | 43 >&2 echo " skip $test_id $line" |
55 continue | 44 continue |
56 fi | 45 fi |
57 if [[ ${id,,} != ${tool_id,,} ]]; then | 46 if [[ ${id,,} != ${tool_id,,} ]]; then |
65 continue | 54 continue |
66 fi | 55 fi |
67 tes=" <test>\n" | 56 tes=" <test>\n" |
68 line=$(fix_tmp_files "$line") | 57 line=$(fix_tmp_files "$line") |
69 line=$(unique_files "$line") | 58 line=$(unique_files "$line") |
70 # >&2 echo $line | 59 # >&2 echo LINE $line |
71 #if there is an ini file then we use this to generate the test | 60 #if there is an ini file then we use this to generate the test |
72 #otherwise the ctd file is used | 61 #otherwise the ctd file is used |
73 #other command line parameters are inserted later into this xml | 62 #other command line parameters are inserted later into this xml |
74 if grep -lq "\-ini" <<<"$line"; then | 63 if grep -lq "\-ini" <<<"$line"; then |
75 ini=$(echo $line | sed 's/.*-ini \([^ ]\+\).*/\1/') | 64 ini=$(echo $line | sed 's/.*-ini \([^ ]\+\).*/\1/') |
76 ini="test-data/$ini" | 65 ini="test-data/$ini" |
77 else | 66 else |
78 ini="ctd/$tool_id.ctd" | 67 ini="ctd/$tool_id.ctd" |
79 fi | 68 fi |
69 # >&2 echo "========================================================" | |
70 # >&2 echo "USING ini $ini" | |
80 cli=$(echo $line |cut -d" " -f3- | sed 's/-ini [^ ]\+//') | 71 cli=$(echo $line |cut -d" " -f3- | sed 's/-ini [^ ]\+//') |
81 | 72 |
82 ctdtmp=$(mktemp) | 73 ctdtmp=$(mktemp) |
83 #echo python3 fill_ctd_clargs.py --ctd $ini $cli | |
84 # using eval: otherwise for some reason quoted values are not used properly ('A B' -> ["'A", "B'"]) | 74 # using eval: otherwise for some reason quoted values are not used properly ('A B' -> ["'A", "B'"]) |
85 # >&2 echo "python3 fill_ctd_clargs.py --ctd $ini $cli" | 75 # >&2 echo "python3 fill_ctd_clargs.py --ini_file $ini $cli" |
86 eval "python3 fill_ctd_clargs.py --ctd $ini $cli" > "$ctdtmp" | 76 eval "python3 fill_ctd_clargs.py --ini_file $ini $cli" > "$ctdtmp" |
87 # echo $ctdtmp | 77 # >&2 echo $ctdtmp |
88 # >&2 cat $ctdtmp | 78 # >&2 cat $ctdtmp |
89 testtmp=$(mktemp) | 79 testtmp=$(mktemp) |
90 python3 $CTDCONVERTER/convert.py galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null | 80 # >&2 echo CTDConverter galaxy -i $ctdtmp -o $testtmp -s aux/tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p aux/hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib --test-condition "compare=sim_size" "delta_frac=0.7" |
81 CTDConverter galaxy -i $ctdtmp -o $testtmp -s aux/tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p aux/hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib --test-condition "compare=sim_size" "delta_frac=0.7" > /dev/null | |
82 echo "<!-- $test_id -->" | |
91 cat $testtmp | grep -v '<output.*file=""' # | grep -v 'CHEMISTRY/' | 83 cat $testtmp | grep -v '<output.*file=""' # | grep -v 'CHEMISTRY/' |
92 rm $ctdtmp $testtmp | 84 |
85 rm "$ctdtmp" "$testtmp" | |
93 | 86 |
94 #> /dev/null | 87 #> /dev/null |
95 | 88 |
96 #rm $testtmp | 89 #rm $testtmp |
97 done | 90 done |
128 #problem: the extension of the tmp files is unusable for test generation. | 121 #problem: the extension of the tmp files is unusable for test generation. |
129 #unfortunately the extensions used in the DIFF lines are not always usable for the CLI | 122 #unfortunately the extensions used in the DIFF lines are not always usable for the CLI |
130 #(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt) | 123 #(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt) |
131 #this function replaces the tmp file by the expected file. | 124 #this function replaces the tmp file by the expected file. |
132 function fix_tmp_files { | 125 function fix_tmp_files { |
133 # >&2 echo "FIX $line" | 126 # >&2 echo "FIX $line" |
134 ret="" | 127 ret="" |
135 for a in $@; do | 128 for a in $@; do |
136 if [[ ! $a =~ .tmp$ ]]; then | 129 # >&2 echo " a "$a |
130 if [[ ! $a =~ .tmp$ ]] && [[ ! $a =~ _tmp_ ]]; then | |
137 ret="$ret $a" | 131 ret="$ret $a" |
138 continue | 132 continue |
139 fi | 133 fi |
140 # >&2 echo " a "$a | 134 diff_line=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep '\${DIFF}.*'"$a") |
141 g=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep '\${DIFF}.*'"$a") | 135 # >&2 echo " diff_line "$diff_line |
142 # >&2 echo " g "$g | 136 in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$diff_line) |
143 in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$g) | |
144 # >&2 echo " in1 "$in1 | 137 # >&2 echo " in1 "$in1 |
145 if [[ "$a" != "$in1" ]]; then | 138 if [[ "$a" != "$in1" ]]; then |
146 ret="$ret $a" | 139 ret="$ret $a" |
147 continue | 140 continue |
148 fi | 141 fi |
149 in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$g) | 142 in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$diff_line) |
150 in2=$(basename $in2 | sed 's/)$//') | 143 in2=$(basename $in2 | sed 's/)$//') |
151 # >&2 echo " in2 "$in2 | 144 # >&2 echo " in2 "$in2 |
152 if [[ -f "test-data/$in2" ]]; then | 145 if [[ -f "test-data/$in2" ]]; then |
153 ln -fs "$in1" "test-data/$in2" | 146 ln -fs "$in1" "test-data/$in2" |
154 ret="$ret $in2" | 147 ret="$ret $in2" |
174 >&2 echo "not linking equal $in1 $in2" | 167 >&2 echo "not linking equal $in1 $in2" |
175 continue | 168 continue |
176 fi | 169 fi |
177 ln -f -s $in1 test-data/$in2 | 170 ln -f -s $in1 test-data/$in2 |
178 done | 171 done |
179 for i in test-data/*.tmp | 172 |
180 do | 173 find test-data/ -name "*.tmp" -print0 | |
174 while IFS= read -r -d '' i; do | |
181 if [ ! -e test-data/$(basename $i .tmp) ]; then | 175 if [ ! -e test-data/$(basename $i .tmp) ]; then |
182 ln -s $(basename $i) test-data/$(basename $i .tmp) | 176 ln -s $(basename $i) test-data/$(basename $i .tmp) |
183 #ln -s $(basename $i) test-data/$(basename $i .tmp) | |
184 else | 177 else |
185 ln -fs $(basename $i) test-data/$(basename $i .tmp) | 178 ln -fs $(basename $i) test-data/$(basename $i .tmp) |
186 fi | 179 fi |
187 done | 180 done |
188 } | 181 } |
192 # parse data preparation calls from OpenMS sources for a tool with a given id | 185 # parse data preparation calls from OpenMS sources for a tool with a given id |
193 function prepare_test_data { | 186 function prepare_test_data { |
194 # id=$1 | 187 # id=$1 |
195 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?" | 188 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?" |
196 | 189 |
197 # TODO SiriusAdapter https://github.com/OpenMS/OpenMS/pull/5010 | 190 # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010 |
198 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | | 191 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | |
199 sed 's/degenerate_cases\///' | | 192 sed 's/degenerate_cases\///' | |
200 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | | 193 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | |
201 grep add_test | | 194 grep add_test | |
202 egrep "TOPP|UTILS" | | 195 egrep "TOPP|UTILS" | |
203 sed 's@${DATA_DIR_SHARE}/@@g;'| | 196 sed 's@${DATA_DIR_SHARE}/@@g;'| |
204 sed 's@${TMP_RIP_PATH}@dummy2.tmp@g'| | 197 sed 's@${TMP_RIP_PATH}@./@g'| |
205 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'| | 198 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'| |
206 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' | | 199 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' | |
207 while read line | 200 while read line |
208 do | 201 do |
209 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1) | 202 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1) |