Mercurial > repos > galaxyp > openms_textexporter
comparison generate-foo.sh @ 9:27aeff17fd2e draft
"planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit ddf41e8bda1ba065f5cdec98e93dee8165ffc1b9"
author | galaxyp |
---|---|
date | Thu, 27 Aug 2020 19:45:29 -0400 |
parents | |
children | 1c596fdaf34a |
comparison
equal
deleted
inserted
replaced
8:1294873c8d1d | 9:27aeff17fd2e |
---|---|
1 #!/usr/bin/env bash | |
2 | |
3 # parse test definitions from OpenMS sources for a tool with a given id | |
4 function get_tests2 { | |
5 id=$1 | |
6 >&2 echo "generate tests for $id" | |
7 echo '<xml name="autotest_'"$id"'">' | |
8 | |
9 # get the tests from the CMakeLists.txt | |
10 # 1st remove some tests | |
11 # - Filefilter with empty select_palarity value (empty is not in the list of allowed options) | |
12 # - MassTraceExtractor with outdated ini file leading to wrong parameters https://github.com/OpenMS/OpenMS/issues/4386 | |
13 # - OpenSwathMzMLFileCacher with -convert_back argumen https://github.com/OpenMS/OpenMS/issues/4399 | |
14 # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt | |
15 # - several tools with duplicated input (leads to conflict when linking) | |
16 # - TOFCalibration inputs we extension (also in prepare_test_data) https://github.com/OpenMS/OpenMS/pull/4525 | |
17 # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456) | |
18 # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed) | |
19 # - some input files are originally in a subdir (degenerated cases/), but not in test-data | |
20 # - SeedListGenerator: https://github.com/OpenMS/OpenMS/issues/4404 | |
21 # - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet) | |
22 CMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | | |
23 sed 's@${DATA_DIR_SHARE}/@@g' | | |
24 grep -v 'OpenSwathMzMLFileCacher .*-convert_back' | | |
25 sed 's/${TMP_RIP_PATH}/""/' | | |
26 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @' | | |
27 grep -v "MaRaClusterAdapter.*-consensus_out"| | |
28 grep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " | | |
29 sed 's@degenerate_cases/@@g' | | |
30 grep -v 'TOPP_SeedListGenerator_3"' | | |
31 egrep -v 'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"') | |
32 | |
33 | |
34 # grep -v 'FileFilter.*-spectra:select_polarity ""' | | |
35 # grep -v 'MassTraceExtractor_2.ini ' | | |
36 # grep -v "FileMerger_6_input2.mzML.*FileMerger_6_input2.mzML" | | |
37 # grep -v "IDMerger_1_input1.idXML.*IDMerger_1_input1.idXML" | | |
38 # grep -v "degenerated_empty.idXML.*degenerated_empty.idXML" | | |
39 # grep -v "FeatureLinkerUnlabeledKD_1_output.consensusXML.*FeatureLinkerUnlabeledKD_1_output.consensusXML" | | |
40 # grep -v "FeatureLinkerUnlabeledQT_1_output.consensusXML.*FeatureLinkerUnlabeledQT_1_output.consensusXML" | | |
41 | |
42 # 1st part is a dirty hack to join lines containing a single function call, e.g. | |
43 # addtest(.... | |
44 # ....) | |
45 echo "$CMAKE" | sed 's/#.*//; s/^\s*//; s/\s*$//' | grep -v "^#" | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | | |
46 grep -iE "add_test\(\"(TOPP|UTILS)_.*/$id " | egrep -v "_prepare\"|_convert|WRITEINI|WRITECTD|INVALIDVALUE" | while read -r line | |
47 do | |
48 line=$(echo "$line" | sed 's/add_test("\([^"]\+\)"/\1/; s/)$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g') | |
49 # >&2 echo $line | |
50 test_id=$(echo "$line" | cut -d" " -f 1) | |
51 tool_id=$(echo "$line" | cut -d" " -f 2) | |
52 if [[ $test_id =~ _out_?[0-9]? ]]; then | |
53 >&2 echo " skip $test_id $line" | |
54 continue | |
55 fi | |
56 if [[ ${id,,} != ${tool_id,,} ]]; then | |
57 >&2 echo " skip $test_id ($id != $tool_id) $line" | |
58 continue | |
59 fi | |
60 | |
61 #remove tests with set_tests_properties(....PROPERTIES WILL_FAIL 1) | |
62 if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then | |
63 >&2 echo " skip failing "$test_id | |
64 continue | |
65 fi | |
66 tes=" <test>\n" | |
67 line=$(fix_tmp_files "$line") | |
68 line=$(unique_files "$line") | |
69 # >&2 echo $line | |
70 #if there is an ini file then we use this to generate the test | |
71 #otherwise the ctd file is used | |
72 #other command line parameters are inserted later into this xml | |
73 if grep -lq "\-ini" <<<"$line"; then | |
74 ini=$(echo $line | sed 's/.*-ini \([^ ]\+\).*/\1/') | |
75 ini="test-data/$ini" | |
76 else | |
77 ini="ctd/$tool_id.ctd" | |
78 fi | |
79 cli=$(echo $line |cut -d" " -f3- | sed 's/-ini [^ ]\+//') | |
80 | |
81 ctdtmp=$(mktemp) | |
82 #echo python3 fill_ctd_clargs.py --ctd $ini $cli | |
83 # using eval: otherwise for some reason quoted values are not used properly ('A B' -> ["'A", "B'"]) | |
84 # >&2 echo "python3 fill_ctd_clargs.py --ctd $ini $cli" | |
85 eval "python3 fill_ctd_clargs.py --ctd $ini $cli" > "$ctdtmp" | |
86 # echo $ctdtmp | |
87 # >&2 cat $ctdtmp | |
88 testtmp=$(mktemp) | |
89 python3 $CTDCONVERTER/convert.py galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null | |
90 cat $testtmp | grep -v '<output.*file=""' # | grep -v 'CHEMISTRY/' | |
91 rm $ctdtmp $testtmp | |
92 | |
93 #> /dev/null | |
94 | |
95 #rm $testtmp | |
96 done | |
97 echo '</xml>' | |
98 } | |
99 | |
100 #some tests use the same file twice which does not work in planemo tests | |
101 #hence we create symlinks for each file used twice | |
102 function unique_files { | |
103 line=$@ | |
104 for arg in $@ | |
105 do | |
106 if [[ ! -f "test-data/$arg" ]]; then | |
107 continue | |
108 fi | |
109 cnt=$(grep -c $arg <<< $(echo "$line" | tr ' ' '\n')) | |
110 while [[ $cnt -gt 1 ]]; do | |
111 new_arg=$(echo $arg | sed "s/\(.*\)\./\1_$cnt./") | |
112 ln -fs $arg test-data/$new_arg | |
113 line=$(echo $line | sed "s/\($arg.*\)$arg/\1$new_arg/") | |
114 cnt=$(grep -c $arg <<< $(echo "$line" | tr ' ' '\n')) | |
115 done | |
116 done | |
117 | |
118 echo $line | |
119 } | |
120 | |
121 # options of out_type selects need to be fixed to Galaxy data types | |
122 function fix_out_type { | |
123 grep "^$1" "$2" | awk '{print $2}' | |
124 } | |
125 | |
126 #OpenMS tests output to tmp files and compare with FuzzyDiff to the expected file. | |
127 #problem: the extension of the tmp files is unusable for test generation. | |
128 #unfortunately the extensions used in the DIFF lines are not always usable for the CLI | |
129 #(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt) | |
130 #this function replaces the tmp file by the expected file. | |
131 function fix_tmp_files { | |
132 # >&2 echo "FIX $line" | |
133 ret="" | |
134 for a in $@; do | |
135 if [[ ! $a =~ .tmp$ ]]; then | |
136 ret="$ret $a" | |
137 continue | |
138 fi | |
139 # >&2 echo " a "$a | |
140 g=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep '\${DIFF}.*'"$a") | |
141 # >&2 echo " g "$g | |
142 in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$g) | |
143 # >&2 echo " in1 "$in1 | |
144 if [[ "$a" != "$in1" ]]; then | |
145 ret="$ret $a" | |
146 continue | |
147 fi | |
148 in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$g) | |
149 in2=$(basename $in2 | sed 's/)$//') | |
150 # >&2 echo " in2 "$in2 | |
151 if [[ -f "test-data/$in2" ]]; then | |
152 ln -fs "$in1" "test-data/$in2" | |
153 ret="$ret $in2" | |
154 else | |
155 ret="$ret $a" | |
156 fi | |
157 done | |
158 # >&2 echo "--> $ret" | |
159 echo "$ret" | |
160 } | |
161 | |
162 function link_tmp_files { | |
163 # note this also considers commented lines (starting with a #) | |
164 # because of tests where the diff command is commented and we | |
165 # still want to use the extension of these files | |
166 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep "\${DIFF}" | while read -r line | |
167 do | |
168 in1=$(sed 's/.*-in1 \([^ ]\+\).*/\1/' <<<$line) | |
169 in1=$(basename $in1 | sed 's/)$//') | |
170 in2=$(sed 's/.*-in2 \([^ ]\+\).*/\1/' <<<$line) | |
171 in2=$(basename $in2 | sed 's/)$//') | |
172 if [[ "$in1" == "$in2" ]]; then | |
173 >&2 echo "not linking equal $in1 $in2" | |
174 continue | |
175 fi | |
176 ln -f -s $in1 test-data/$in2 | |
177 done | |
178 for i in test-data/*.tmp | |
179 do | |
180 if [ ! -e test-data/$(basename $i .tmp) ]; then | |
181 ln -s $(basename $i) test-data/$(basename $i .tmp) | |
182 #ln -s $(basename $i) test-data/$(basename $i .tmp) | |
183 else | |
184 ln -fs $(basename $i) test-data/$(basename $i .tmp) | |
185 fi | |
186 done | |
187 } | |
188 | |
189 | |
190 | |
191 # parse data preparation calls from OpenMS sources for a tool with a given id | |
192 function prepare_test_data { | |
193 # id=$1 | |
194 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?" | |
195 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' | grep -v "^$" | awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | | |
196 sed 's/degenerate_cases\///' | | |
197 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | | |
198 grep add_test | | |
199 egrep "TOPP|UTILS" | | |
200 sed 's@${DATA_DIR_SHARE}/@@g;'| | |
201 sed 's@${TMP_RIP_PATH}@dummy2.tmp@g'| | |
202 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'| | |
203 while read line | |
204 do | |
205 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1) | |
206 | |
207 if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then | |
208 >&2 echo " skip failing "$test_id | |
209 continue | |
210 fi | |
211 | |
212 line=$(echo "$line" | sed 's/add_test("//; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f2-) | |
213 # line="$(fix_tmp_files $line)" | |
214 echo "$line > $test_id.stdout 2> $test_id.stderr" | |
215 echo "if [[ \"\$?\" -ne \"0\" ]]; then >&2 echo '$test_id failed'; >&2 echo -e \"stderr:\n\$(cat $test_id.stderr | sed 's/^/ /')\"; echo -e \"stdout:\n\$(cat $test_id.stdout)\";fi" | |
216 done | |
217 } |