Mercurial > repos > galaxyp > openms_fuzzydiff
comparison test-data.sh @ 15:c940abd8d622 draft
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 3d1e5f37fd16524a415f707772eeb7ead848c5e3
author | galaxyp |
---|---|
date | Thu, 01 Dec 2022 19:04:01 +0000 |
parents | ed9a839cbbda |
children | 5369624840b9 |
comparison
equal
deleted
inserted
replaced
14:d98117e7b3b2 | 15:c940abd8d622 |
---|---|
1 #!/usr/bin/env bash | 1 #!/usr/bin/env bash |
2 | 2 |
3 VERSION=2.6 | 3 VERSION=2.8 |
4 FILETYPES="filetypes.txt" | 4 FILETYPES="aux/filetypes.txt" |
5 CONDAPKG="https://anaconda.org/bioconda/openms/2.6.0/download/linux-64/openms-2.6.0-h4afb90d_0.tar.bz2" | 5 CONDAPKG="https://anaconda.org/bioconda/openms/2.8.0/download/linux-64/openms-2.8.0-h7ca0330_0.tar.bz2" |
6 | 6 |
7 # import the magic | 7 # import the magic |
8 . ./generate-foo.sh | 8 . ./generate-foo.sh |
9 | 9 |
10 # install conda | 10 # install conda |
11 if [ -z "$tmp" ]; then | 11 if [ -z "$tmp" ]; then |
12 tmp=$(mktemp -d) | 12 tmp=$(mktemp -d) |
13 created="yes" | 13 created="yes" |
14 fi | 14 fi |
15 | 15 |
16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git" | 16 export OPENMSGIT="$tmp/OpenMS$VERSION.0-git" |
17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/" | 17 export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/" |
18 export OPENMSENV="$tmp/OpenMS$VERSION-env" | 18 export OPENMSENV="OpenMS$VERSION-env" |
19 export CTDCONVERTER="$tmp/CTDConverter" | 19 |
20 if [ -z "$CTDCONVERTER" ]; then | |
21 export CTDCONVERTER="$tmp/CTDConverter" | |
22 fi | |
20 | 23 |
21 if [[ -z "$1" ]]; then | 24 if [[ -z "$1" ]]; then |
22 autotests="/dev/null" | 25 autotests="/dev/null" |
23 else | 26 else |
24 autotests="$1" | 27 autotests="$1" |
25 fi | 28 fi |
26 | 29 |
27 if type conda > /dev/null; then | 30 if type conda > /dev/null; then |
28 true | 31 true |
29 else | 32 else |
30 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh | 33 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh |
31 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda" | 34 bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda" |
32 source "$tmp/miniconda/bin/activate" | 35 source "$tmp/miniconda/bin/activate" |
33 fi | 36 fi |
34 eval "$(conda shell.bash hook)" | 37 eval "$(conda shell.bash hook)" |
35 | 38 |
36 | 39 |
37 ############################################################################### | 40 ############################################################################### |
40 ## - the git clone of OpenMS (for generating the tests) | 43 ## - the git clone of OpenMS (for generating the tests) |
41 ############################################################################### | 44 ############################################################################### |
42 | 45 |
43 echo "Clone OpenMS $VERSION sources" | 46 echo "Clone OpenMS $VERSION sources" |
44 if [[ ! -d $OPENMSGIT ]]; then | 47 if [[ ! -d $OPENMSGIT ]]; then |
45 git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT | 48 # TODO >2.8 reenable original release branch .. also in else branch |
46 cd $OPENMSGIT | 49 # the plus branch contains commits from https://github.com/OpenMS/OpenMS/pull/5920 and https://github.com/OpenMS/OpenMS/pull/5917 |
47 git submodule init | 50 # git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT |
48 git submodule update | 51 git clone -b release/$VERSION.0-plus https://github.com/bernt-matthias/OpenMS.git $OPENMSGIT |
49 cd - | 52 cd $OPENMSGIT |
50 else | 53 git submodule init |
51 cd $OPENMSGIT | 54 git submodule update |
52 git pull origin release/$VERSION.0 | 55 cd - |
53 cd - | 56 else |
57 cd $OPENMSGIT | |
58 git pull origin release/$VERSION.0-plus | |
59 cd - | |
54 fi | 60 fi |
55 | 61 |
56 echo "Create OpenMS $VERSION conda env" | 62 echo "Create OpenMS $VERSION conda env" |
57 # TODO currently add lxml (needed by CTDConverter) | 63 # TODO currently add lxml (needed by CTDConverter) |
58 # TODO for some reason a to recent openjdk is used | 64 # TODO for some reason a to recent openjdk is used |
59 if conda env list | grep "$OPENMSENV"; then | 65 if conda env list | grep "$OPENMSENV"; then |
60 true | 66 true |
61 else | 67 else |
62 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.4 lxml | 68 conda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION omssa=2.1.9 ctdopts=1.5 lxml |
63 # chmod -R u-w $OPENMSENV | 69 # chmod -R u-w $OPENMSENV |
64 fi | 70 fi |
65 ############################################################################### | 71 ############################################################################### |
66 ## get the | 72 ## get the |
67 ## - conda package (for easy access and listing of the OpenMS binaries), | 73 ## - conda package (for easy access and listing of the OpenMS binaries), |
68 ############################################################################### | 74 ############################################################################### |
69 echo "Download OpenMS $VERSION package $CONDAPKG" | 75 echo "Download OpenMS $VERSION package $CONDAPKG" |
70 | 76 |
71 if [[ ! -d $OPENMSPKG ]]; then | 77 if [[ ! -d $OPENMSPKG ]]; then |
72 mkdir $OPENMSPKG | 78 mkdir $OPENMSPKG |
73 wget -q -P $OPENMSPKG/ "$CONDAPKG" | 79 wget -q -P $OPENMSPKG/ "$CONDAPKG" |
74 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/ | 80 tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/ |
75 rm $OPENMSPKG/"$(basename $CONDAPKG)" | 81 rm $OPENMSPKG/"$(basename $CONDAPKG)" |
76 fi | 82 fi |
77 | 83 |
78 ############################################################################### | 84 ############################################################################### |
79 ## Get python libaries for CTD -> Galaxy conversion | 85 ## Get python libaries for CTD -> Galaxy conversion |
80 ## TODO fix to main repo OR conda packkage if PRs are merged | 86 ## TODO fix to main repo OR conda packkage if PRs are merged |
81 ############################################################################### | 87 ############################################################################### |
82 echo "Clone CTDConverter" | 88 echo "Clone CTDConverter" |
83 if [[ ! -d $CTDCONVERTER ]]; then | 89 if [[ ! -d $CTDCONVERTER ]]; then |
84 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter | 90 #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter |
85 git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER | 91 git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER |
86 else | 92 else |
87 cd $CTDCONVERTER | 93 cd $CTDCONVERTER |
88 git pull origin topic/cdata | 94 git pull origin topic/fix-selects2 |
89 cd - | 95 cd - |
90 fi | 96 fi |
91 | 97 conda activate $OPENMSENV |
92 ############################################################################### | 98 cd $CTDCONVERTER |
93 ## copy all the test data files to test-data | 99 python -m pip install . --no-deps |
94 ## most of it (outputs) will be overwritten later, but its needed for | 100 cd - |
95 ## prepare_test_data | 101 conda deactivate |
96 ############################################################################### | 102 |
103 | |
104 # ############################################################################### | |
105 # ## copy all the test data files to test-data | |
106 # ## most of it (outputs) will be overwritten later, but its needed for | |
107 # ## prepare_test_data | |
108 # ############################################################################### | |
97 echo "Get test data" | 109 echo "Get test data" |
98 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" -delete | 110 find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" -delete |
99 | 111 |
100 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ | 112 cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/ |
101 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ | 113 cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ |
102 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ | 114 cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ |
103 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ | 115 cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ |
104 if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then | 116 if [ ! -f test-data/MetaboliteSpectralDB.mzML ]; then |
105 wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML | 117 wget -nc https://abibuilder.cs.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML |
106 mv MetaboliteSpectralDB.mzML test-data/ | 118 mv MetaboliteSpectralDB.mzML test-data/ |
107 fi | 119 fi |
108 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt | 120 ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt |
109 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv | 121 ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv |
110 | 122 |
111 if [ ! -d test-data/pepnovo_models/ ]; then | 123 if [ ! -d test-data/pepnovo_models/ ]; then |
112 mkdir -p /tmp/pepnovo | 124 mkdir -p /tmp/pepnovo |
113 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip | 125 wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip |
114 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ | 126 unzip PepNovo.20120423.zip -d /tmp/pepnovo/ |
115 mv /tmp/pepnovo/Models test-data/pepnovo_models/ | 127 mv /tmp/pepnovo/Models test-data/pepnovo_models/ |
116 rm PepNovo.20120423.zip | 128 rm PepNovo.20120423.zip |
117 rm -rf /tmp/pepnovo | 129 rm -rf /tmp/pepnovo |
118 fi | 130 fi |
119 ############################################################################### | 131 ############################################################################### |
120 ## generate ctd files using the binaries in the conda package | 132 ## generate ctd files using the binaries in the conda package |
121 ############################################################################### | 133 ############################################################################### |
122 echo "Create CTD files" | 134 echo "Create CTD files" |
123 conda activate $OPENMSENV | 135 conda activate $OPENMSENV |
124 rm -rf ctd | 136 rm -rf ctd |
125 mkdir -p ctd | 137 mkdir -p ctd |
126 | 138 |
127 # TODO because of https://github.com/OpenMS/OpenMS/issues/4641 | |
128 # this needs to be done from within test-data | |
129 cd test-data | |
130 for i in $OPENMSPKG/bin/* | 139 for i in $OPENMSPKG/bin/* |
131 do | 140 do |
132 b=$(basename $i) | 141 b=$(basename $i) |
133 echo $b | 142 echo $b |
134 $b -write_ctd ../ctd/ | 143 $b -write_ctd ctd/ |
135 sed -i -e 's/²/^2/' ../ctd/$b.ctd | 144 sed -i -e 's/²/^2/' ctd/$b.ctd |
136 done | 145 done |
137 cd - | |
138 ############################################################################### | 146 ############################################################################### |
139 ## fix ini files: OpenMS test data contains ini files with outdated ini files. | 147 ## fix ini files: OpenMS test data contains ini files with outdated ini files. |
140 ## e.g. variables might be in different nodes, outdated variables present, new | 148 ## e.g. variables might be in different nodes, outdated variables present, new |
141 ## variables missing, ... | 149 ## variables missing, ... |
142 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests) | 150 ## OpenMS tools fix this on the fly (so its no problem for the OpenMS tests) |
144 ## see https://github.com/OpenMS/OpenMS/issues/4462 | 152 ## see https://github.com/OpenMS/OpenMS/issues/4462 |
145 ############################################################################### | 153 ############################################################################### |
146 echo "Update test INI files" | 154 echo "Update test INI files" |
147 for ini in test-data/*ini | 155 for ini in test-data/*ini |
148 do | 156 do |
149 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/') | 157 tool=$(cat $ini | grep 'NODE name="' | head -n 1 | sed 's/.*name="\([^"]\+\)".*/\1/') |
150 bin=$(which $tool) | 158 bin=$(which $tool) |
151 if [[ -z $bin ]]; then | 159 if [[ -z $bin ]]; then |
152 >&2 echo "missing binary to convert $ini" | 160 >&2 echo "missing binary to convert $ini" |
153 continue | 161 continue |
154 fi | 162 fi |
155 cp $ini $ini.backup | 163 cp $ini $ini.backup |
156 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr | 164 $bin -ini $ini -write_ini $ini > $ini.stdout 2> $ini.stderr |
157 if [[ "$?" -ne "0" ]]; then | 165 if [[ "$?" -ne "0" ]]; then |
158 >&2 echo "could not convert $ini" | 166 >&2 echo "could not convert $ini" |
159 fi | 167 fi |
160 done | 168 done |
161 | 169 |
162 ############################################################################### | 170 ############################################################################### |
163 ## create script to create results for the tests and run it | 171 ## create script to create results for the tests and run it |
164 ############################################################################### | 172 ############################################################################### |
183 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh | 191 echo 'export XTANDEM_BINARY="xtandem"' >> prepare_test_data.sh |
184 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh | 192 echo 'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"' >> prepare_test_data.sh |
185 | 193 |
186 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh | 194 prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh |
187 | 195 |
188 # prepare_test_data > tmp_test_data.sh | 196 ## prepare_test_data > tmp_test_data.sh |
189 # # remove calls not needed for the tools listed in any .list file | 197 ## # remove calls not needed for the tools listed in any .list file |
190 # echo LIST $LIST | 198 ## echo LIST $LIST |
191 # if [ ! -z "$LIST" ]; then | 199 ## if [ ! -z "$LIST" ]; then |
192 # REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//') | 200 ## REX=$(echo $LIST | sed 's/ /\n/g' | sed 's@.*/\([^/]\+\).xml$@\1@' | tr '\n' '|' | sed 's/|$//') |
193 # else | 201 ## else |
194 # REX=".*" | 202 ## REX=".*" |
195 # fi | 203 ## fi |
196 # echo REX $REX | 204 ## echo REX $REX |
197 # cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh | 205 ## cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh |
198 # rm tmp_test_data.sh | 206 ## rm tmp_test_data.sh |
199 | 207 |
200 echo "Execute test shell script" | 208 echo "Execute test shell script" |
201 chmod u+x prepare_test_data.sh | 209 chmod u+x prepare_test_data.sh |
202 cd ./test-data || exit | 210 cd ./test-data || exit |
203 ../prepare_test_data.sh | 211 ../prepare_test_data.sh |
204 cd - || exit | 212 cd - || exit |
205 | 213 |
206 | 214 |
207 ############################################################################### | 215 # ############################################################################### |
208 ## create/update test data for the manually generated tests | 216 # ## create/update test data for the manually generated tests |
209 ## - run convert once with the manual tests only and | 217 # ## - run convert once with the manual tests only and |
210 ## - update test-data (needs to run 2x) | 218 # ## - update test-data (needs to run 2x) |
211 ############################################################################### | 219 # ############################################################################### |
212 echo "Execute test shell script for manually curated tests" | 220 echo "Execute test shell script for manually curated tests" |
213 chmod u+x prepare_test_data_manual.sh | 221 chmod u+x prepare_test_data_manual.sh |
214 | |
215 cd ./test-data || exit | 222 cd ./test-data || exit |
216 ../prepare_test_data_manual.sh | 223 ../prepare_test_data_manual.sh |
217 cd - || exit | 224 cd - || exit |
218 | 225 |
219 | 226 |
220 ############################################################################### | 227 ############################################################################### |
221 ## auto generate tests | 228 ## auto generate tests |
222 ############################################################################### | 229 ############################################################################### |
230 | |
223 echo "Write test macros to $autotests" | 231 echo "Write test macros to $autotests" |
224 echo "<macros>" > "$autotests" | 232 echo "<macros>" > "$autotests" |
225 for i in $(ls *xml |grep -v macros) | 233 |
234 for i in $(ls ctd/*ctd) | |
226 do | 235 do |
227 b=$(basename "$i" .xml) | 236 b=$(basename "$i" .ctd) |
228 get_tests2 "$b" >> "$autotests" | 237 get_tests2 "$b" >> "$autotests" |
229 done | 238 done |
230 echo "</macros>" >> "$autotests" | 239 echo "</macros>" >> "$autotests" |
231 | 240 |
232 echo "Create test data links" | 241 # echo "Create test data links" |
233 link_tmp_files | 242 # Breaks DecoyDatabase |
243 # link_tmp_files | |
234 | 244 |
235 # tests for tools using output_prefix parameters can not be auto generated | 245 # tests for tools using output_prefix parameters can not be auto generated |
236 # hence we output the tests for manual curation in macros_test.xml | 246 # hence we output the tests for manual curation in macros_test.xml |
237 # and remove them from the autotests | 247 # and remove them from the autotests |
238 # -> OpenSwathFileSplitter IDRipper MzMLSplitter | 248 # -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator |
249 # TODO reevaluate in >2.8 | |
250 # - https://github.com/OpenMS/OpenMS/pull/5873 | |
251 # - https://github.com/OpenMS/OpenMS/pull/5912 | |
239 # | 252 # |
240 # Furthermore we remove tests for tools without binaries in conda | 253 # Furthermore we remove tests for tools without binaries in conda |
241 # -> MSFragger MaRaClusterAdapter NovorAdapter | 254 # -> MSFragger MaRaClusterAdapter NovorAdapter |
242 # | 255 # |
243 # not able to specify composite test data | 256 # not able to specify composite test data |
244 # -> SpectraSTSearchAdapter | 257 # -> SpectraSTSearchAdapter |
245 if [[ ! -z "$1" ]]; then | 258 if [[ ! -z "$1" ]]; then |
246 echo "" > macros_discarded_auto.xml | 259 echo "" > macros_discarded_auto.xml |
247 for i in OpenSwathFileSplitter IDRipper MzMLSplitter MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter | 260 for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter |
248 do | 261 do |
249 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml | 262 echo "<xml name=\"manutest_$i\">" >> macros_discarded_auto.xml |
250 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml | 263 xmlstarlet sel -t -c "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml >> macros_discarded_auto.xml |
251 echo "</xml>" >> macros_discarded_auto.xml | 264 echo "</xml>" >> macros_discarded_auto.xml |
252 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp | 265 xmlstarlet ed -d "/macros/xml[@name='autotest_$i']/test" macros_autotest.xml > tmp |
253 mv tmp macros_autotest.xml | 266 mv tmp macros_autotest.xml |
254 done | 267 done |
255 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml" | 268 >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml" |
256 fi | 269 fi |
257 conda deactivate | 270 conda deactivate |
258 | 271 |
259 ## remove broken symlinks in test-data | 272 ## remove broken symlinks in test-data |
260 find test-data/ -xtype l -delete | 273 find test-data/ -xtype l -delete |
261 | 274 |
262 # if [ ! -z "$created" ]; then | 275 if [ ! -z "$created" ]; then |
263 # echo "Removing temporary directory" | 276 echo "Removing temporary directory" |
264 # rm -rf "$tmp" | 277 rm -rf "$tmp" |
265 # fi | 278 fi |