Mercurial > repos > galaxyp > openms_metabolitespectralmatcher
comparison test-data.sh @ 14:e46a9e72c395 draft default tip
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 5c080b1e2b99f1c88f4557e9fec8c45c9d23b906
author | galaxyp |
---|---|
date | Fri, 14 Jun 2024 21:31:08 +0000 |
parents | 75ccdbc2475d |
children |
comparison
equal
deleted
inserted
replaced
13:75ccdbc2475d | 14:e46a9e72c395 |
---|---|
1 #!/usr/bin/env bash | 1 #!/usr/bin/env bash |
2 | 2 |
3 VERSION=2.8 | 3 # set -x |
4 | |
5 VERSION=3.1 | |
4 FILETYPES="aux/filetypes.txt" | 6 FILETYPES="aux/filetypes.txt" |
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2" | 7 CONDAPKG="https://anaconda.org/bioconda/openms/3.1.0/download/linux-64/openms-3.1.0-h8964181_1.tar.bz2" |
6 | |
7 # import the magic | |
8 . ./generate-foo.sh | |
9 | 8 |
10 # install conda | 9 # install conda |
11 if [ -z "$tmp" ]; then | 10 if [ -z "$tmp" ]; then |
12 tmp=$(mktemp -d) | 11 tmp=$(mktemp -d) |
13 created="yes" | 12 created="yes" |
43 ## - the git clone of OpenMS (for generating the tests) | 42 ## - the git clone of OpenMS (for generating the tests) |
44 ############################################################################### | 43 ############################################################################### |
45 | 44 |
46 echo "Clone OpenMS $VERSION sources" | 45 echo "Clone OpenMS $VERSION sources" |
47 if [[ ! -d $OPENMSGIT ]]; then | 46 if [[ ! -d $OPENMSGIT ]]; then |
48 # TODO >2.8 reenable original release branch .. also in else branch | 47 if [[ "$created" == "yes" ]]; then |
49 # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917 | 48 GIT_DIR=$(mktemp -d --dry-run) |
50 # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT | 49 GIT_EXTRA_OPTS="--separate-git-dir=$GIT_DIR" |
51 git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT | 50 fi |
51 git clone -b release/$VERSION.0 --depth 1 --recurse-submodules=THIRDPARTY --shallow-submodules $GIT_EXTRA_OPTS https://github.com/OpenMS/OpenMS.git $OPENMSGIT | |
52 ## save some space by just keeping the needed binaries | |
53 find $OPENMSGIT/THIRDPARTY/ -type f -not \( -name maracluster -o -name spectrast \) -delete | |
54 find $OPENMSGIT/THIRDPARTY/ -empty -type d -delete | |
55 if [[ "$created" == "yes" ]]; then | |
56 rm -rf $GIT_DIR | |
57 fi | |
58 else | |
52 cd $OPENMSGIT | 59 cd $OPENMSGIT |
53 git submodule init | 60 git pull origin release/$VERSION.0 |
54 git submodule update | |
55 cd - | |
56 else | |
57 cd $OPENMSGIT | |
58 git pull origin release/$VERSION.0-plus | |
59 cd - | 61 cd - |
60 fi | 62 fi |
61 | 63 |
62 echo "Create OpenMS $VERSION conda env" | 64 echo "Create OpenMS $VERSION conda env" |
63 # TODO currently add lxml (needed by CTDConverter) | 65 # TODO currently add lxml (needed by CTDConverter) |
64 # TODO for some reason a to recent openjdk is used | 66 # TODO for some reason a to recent openjdk is used |
65 if conda env list | grep "$OPENMSENV"; then | 67 if conda env list | grep "$OPENMSENV"; then |
66 true | 68 true |
67 else | 69 else |
68 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml | 70 conda create -y --quiet --solver libmamba --override-channels --strict-channel-priority --channel conda-forge --channel bioconda -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.5 lxml |
69 # chmod -R u-w $OPENMSENV | 71 # chmod -R u-w $OPENMSENV |
70 fi | 72 fi |
71 ############################################################################### | 73 ############################################################################### |
72 ## get the | 74 ## get the |
73 ## - conda package (for easy access and listing of the OpenMS binaries), | 75 ## - conda package (for easy access and listing of the OpenMS binaries), |
99 python -m pip install . --no-deps | 101 python -m pip install . --no-deps |
100 cd - | 102 cd - |
101 conda deactivate | 103 conda deactivate |
102 | 104 |
103 | 105 |
104 # ############################################################################### | 106 # # ############################################################################### |
105 # ## copy all the test data files to test-data | 107 # # ## copy all the test data files to test-data |
106 # ## most of it (outputs) will be overwritten later, but its needed for | 108 # # ## most of it (outputs) will be overwritten later, but its needed for |
107 # ## prepare_test_data | 109 # # ## prepare_test_data |
108 # ############################################################################### | 110 # # ############################################################################### |
109 echo "Get test data" | 111 echo "Get test data" |
110 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete | 112 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" ! -name MetaboliteSpectralDB.mzML -delete |
111 | 113 |
112 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ | 114 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ |
113 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ | 115 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ |
114 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ | 116 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ |
115 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ | 117 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ |
116 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then | 118 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then |
117 wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML | 119 wget -nc https://raw.githubusercontent.com/sneumann/OpenMS/master/share/OpenMS/CHEMISTRY/MetaboliteSpectralDB.mzML |
120 # wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML | |
118 mv MetaboliteSpectralDB.mzML test-data/ | 121 mv MetaboliteSpectralDB.mzML test-data/ |
119 fi | 122 fi |
120 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt | 123 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt |
121 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv | 124 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv |
122 | 125 |
123 if [ ! -d test-data/pepnovo_models/ ]; then | 126 # if [ ! -d test-data/pepnovo_models/ ]; then |
124 mkdir -p /tmp/pepnovo | 127 # mkdir -p /tmp/pepnovo |
125 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip | 128 # wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip |
126 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ | 129 # unzip PepNovo.20120423.zip -d /tmp/pepnovo/ |
127 mv /tmp/pepnovo/Models test-data/pepnovo_models/ | 130 # mv /tmp/pepnovo/Models test-data/pepnovo_models/ |
128 rm PepNovo.20120423.zip | 131 # rm PepNovo.20120423.zip |
129 rm -rf /tmp/pepnovo | 132 # rm -rf /tmp/pepnovo |
130 fi | 133 # fi |
131 ############################################################################### | 134 ############################################################################### |
132 ## generate ctd files using the binaries in the conda package | 135 ## generate ctd files using the binaries in the conda package |
133 ############################################################################### | 136 ############################################################################### |
134 echo "Create CTD files" | 137 echo "Create CTD files" |
135 conda activate $OPENMSENV | 138 conda activate $OPENMSENV |
168 done | 171 done |
169 | 172 |
170 ############################################################################### | 173 ############################################################################### |
171 ## create script to create results for the tests and run it | 174 ## create script to create results for the tests and run it |
172 ############################################################################### | 175 ############################################################################### |
176 # parse data preparation calls from OpenMS sources for a tool with a given id | |
177 function prepare_test_data { | |
178 # id=$1 | |
179 # | egrep -i "$id\_.*[0-9]+(_prepare\"|_convert)?" | |
180 | |
181 OLD_OSW_PARAM=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt |sed 's/#.*$//'| sed 's/^\s*//; s/\s*$//' |awk '{printf("%s@NEWLINE@", $0)}' | sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | grep OLD_OSW_PARAM | head -n 1 | sed 's/^[^"]\+//; s/)$//; s/"//g') | |
182 # TODO SiriusAdapter depends on online service which may timeout .. so keep disabled https://github.com/OpenMS/OpenMS/pull/5010 | |
183 cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | | |
184 sed "s/\${OLD_OSW_PARAM}/$OLD_OSW_PARAM/" | | |
185 grep -v "\.ini\.json" | | |
186 sed 's/.ini.json /ini /' | | |
187 sed 's/#.*$//'| | |
188 sed 's/^\s*//; s/\s*$//' | | |
189 grep -v "^$" | | |
190 awk '{printf("%s@NEWLINE@", $0)}' | | |
191 sed 's/)@NEWLINE@/)\n/g' | sed 's/@NEWLINE@/ /g' | | |
192 sed 's/degenerate_cases\///' | | |
193 egrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | | |
194 grep add_test | | |
195 egrep "TOPP|UTILS" | | |
196 sed 's@${DATA_DIR_SHARE}/@@g;'| | |
197 sed 's@${TMP_RIP_PATH}@./@g'| | |
198 sed 's@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @'| | |
199 sed 's/\("TOPP_SiriusAdapter_4".*\)-sirius:database all\(.*\)/\1-sirius:database pubchem\2/' | | |
200 while read line | |
201 do | |
202 test_id=$(echo "$line" | sed 's/add_test(//; s/"//g; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f1) | |
203 | |
204 if grep -lq "$test_id"'\".* PROPERTIES WILL_FAIL 1' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then | |
205 >&2 echo " skip failing "$test_id | |
206 continue | |
207 fi | |
208 | |
209 line=$(echo "$line" | sed 's/add_test("//; s/)[^)]*$//; s/\${TOPP_BIN_PATH}\///g;s/\${DATA_DIR_TOPP}\///g; s#THIRDPARTY/##g' | cut -d" " -f2-) | |
210 # line="$(fix_tmp_files $line)" | |
211 echo 'echo executing "'$test_id'"' | |
212 echo "$line > $test_id.stdout 2> $test_id.stderr" | |
213 echo "if [[ \"\$?\" -ne \"0\" ]]; then >&2 echo '$test_id failed'; >&2 echo -e \"stderr:\n\$(cat $test_id.stderr | sed 's/^/ /')\"; echo -e \"stdout:\n\$(cat $test_id.stdout)\";fi" | |
214 done | |
215 } | |
216 | |
173 echo "Create test shell script" | 217 echo "Create test shell script" |
174 | 218 |
175 echo -n "" > prepare_test_data.sh | 219 echo -n "" > prepare_test_data.sh |
176 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh | 220 echo 'export COMET_BINARY="comet"' >> prepare_test_data.sh |
177 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh | 221 echo 'export CRUX_BINARY="crux"' >> prepare_test_data.sh |
178 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh | 222 echo 'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"' >> prepare_test_data.sh |
179 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh | 223 echo 'export FIDO_BINARY="Fido"' >> prepare_test_data.sh |
180 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh | 224 echo 'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"' >> prepare_test_data.sh |
181 | 225 |
182 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh | 226 echo 'export MARACLUSTER_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"'>> prepare_test_data.sh |
183 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"'>> prepare_test_data.sh | 227 echo 'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-3.5/MSFragger-3.5.jar"'>> prepare_test_data.sh |
184 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh | 228 echo 'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"' >> prepare_test_data.sh |
185 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh | 229 echo 'export MYRIMATCH_BINARY="myrimatch"'>> prepare_test_data.sh |
186 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh | 230 echo 'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"' >> prepare_test_data.sh |
187 echo 'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"'>> prepare_test_data.sh | |
188 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh | 231 echo 'export PERCOLATOR_BINARY="percolator"'>> prepare_test_data.sh |
189 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh | 232 echo 'export SIRIUS_BINARY="$(which sirius)"' >> prepare_test_data.sh |
190 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh | 233 echo 'export SPECTRAST_BINARY="'"$OPENMSGIT"'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"' >> prepare_test_data.sh |
191 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh | 234 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh |
192 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh | 235 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh |
236 echo 'export SAGE_BINARY=sage' >> prepare_test_data.sh | |
193 | 237 |
194 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh | 238 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh |
195 | |
196 ## prepare_test_data > tmp_test_data.sh | |
197 ## # remove calls not needed for the tools listed in any .list file | |
198 ## echo LIST $LIST | |
199 ## if [ ! -z "$LIST" ]; then | |
200 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//') | |
201 ## else | |
202 ## REX=".*" | |
203 ## fi | |
204 ## echo REX $REX | |
205 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh | |
206 ## rm tmp_test_data.sh | |
207 | 239 |
208 echo "Execute test shell script" | 240 echo "Execute test shell script" |
209 chmod u+x prepare_test_data.sh | 241 chmod u+x prepare_test_data.sh |
210 cd ./test-data || exit | 242 cd ./test-data || exit |
211 ../prepare_test_data.sh | 243 ../prepare_test_data.sh |
232 echo "<macros>" > "$autotests" | 264 echo "<macros>" > "$autotests" |
233 | 265 |
234 for i in $(ls ctd/*ctd) | 266 for i in $(ls ctd/*ctd) |
235 do | 267 do |
236 b=$(basename "$i" .ctd) | 268 b=$(basename "$i" .ctd) |
237 get_tests2 "$b" >> "$autotests" | 269 ./get_tests.py --id "$b" --cmake "$OPENMSGIT"/src/tests/topp/CMakeLists.txt "$OPENMSGIT"/src/tests/topp/THIRDPARTY/third_party_tests.cmake >> "$autotests" |
270 wc -l "$autotests" | |
238 done | 271 done |
239 echo "</macros>" >> "$autotests" | 272 echo "</macros>" >> "$autotests" |
240 | |
241 # echo "Create test data links" | |
242 # Breaks DecoyDatabase | |
243 # link_tmp_files | |
244 | 273 |
245 # tests for tools using output_prefix parameters can not be auto generated | 274 # tests for tools using output_prefix parameters can not be auto generated |
246 # hence we output the tests for manual curation in macros_test.xml | 275 # hence we output the tests for manual curation in macros_test.xml |
247 # and remove them from the autotests | 276 # and remove them from the autotests |
248 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator | 277 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator |
253 # Furthermore we remove tests for tools without binaries in conda | 282 # Furthermore we remove tests for tools without binaries in conda |
254 # -> MSFragger MaRaClusterAdapter NovorAdapter | 283 # -> MSFragger MaRaClusterAdapter NovorAdapter |
255 # | 284 # |
256 # not able to specify composite test data | 285 # not able to specify composite test data |
257 # -> SpectraSTSearchAdapter | 286 # -> SpectraSTSearchAdapter |
287 echo "Discard some tests" | |
258 if [[ ! -z "$1" ]]; then | 288 if [[ ! -z "$1" ]]; then |
259 echo "" > macros_discarded_auto.xml | 289 echo "" > macros_discarded_auto.xml |
260 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter | 290 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter |
261 do | 291 do |
262 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml | 292 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml |
270 conda deactivate | 300 conda deactivate |
271 | 301 |
272 ## remove broken symlinks in test-data | 302 ## remove broken symlinks in test-data |
273 find test-data/ -xtype l -delete | 303 find test-data/ -xtype l -delete |
274 | 304 |
275 if [ ! -z "$created" ]; then | 305 if [[ "$created" == "yes" ]]; then |
276 echo "Removing temporary directory" | 306 echo "Removing temporary directory" |
277 rm -rf "$tmp" | 307 rm -rf "$tmp" |
278 fi | 308 fi |