| Previous changeset 8:a604e5e1bb06 (2019-05-17) Next changeset 10:416fc18f347a (2020-09-24) |
|
Commit message:
"planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 020906fb54bde7fc143c356f41975c378a741315" |
|
modified:
SvmTheoreticalSpectrumGeneratorTrainer.xml filetypes.txt macros.xml readme.md tools_blacklist.txt |
|
added:
OMSSAAdapter.patch PepNovoAdapter.patch fill_ctd.py fill_ctd_clargs.py generate-foo.sh generate.sh hardcoded_params.json macros_autotest.xml macros_discarded_auto.xml macros_test.xml prepare_test_data.sh prepare_test_data_manual.sh test-data.sh tool-data/pepnovo_models.loc.sample tool_data_table_conf.xml.sample tool_data_table_conf.xml.test |
|
removed:
tool.conf |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d OMSSAAdapter.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OMSSAAdapter.patch Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| @@ -0,0 +1,10 @@ +--- OMSSAAdapter.xml 2020-06-16 15:51:40.315400730 +0200 ++++ /tmp/OMSSAAdapter.xml 2020-06-16 15:50:23.536086074 +0200 +@@ -22,6 +22,7 @@ + mkdir database && + ln -s '$database' 'database/${re.sub("[^\w\-_]", "_", $database.element_identifier)}.$gxy2omsext($database.ext)' && + ++makeblastdb -dbtype prot -in 'database/${re.sub("[^\w\-_]", "_", $database.element_identifier)}.$gxy2omsext($database.ext)' && + ## Main program call + + set -o pipefail && |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d PepNovoAdapter.patch --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PepNovoAdapter.patch Wed Sep 09 19:57:32 2020 +0000 |
| b |
| @@ -0,0 +1,35 @@ +--- PepNovoAdapter.xml 2020-05-12 15:55:24.712831518 +0200 ++++ /tmp/PepNovoAdapter.xml 2020-05-12 15:36:31.267276757 +0200 +@@ -42,8 +42,13 @@ + </configfiles> + <inputs> + <param name="in" argument="-in" type="data" format="mzml" optional="false" label="input file" help=" select mzml data sets(s)"/> +- <param name="model_directory" argument="-model_directory" type="text" optional="false" value="" label="Name of the directory where the model files are kept" help=""> +- <expand macro="list_string_san"/> ++ <param name="model_directory" argument="-model_directory" type="select" optional="false" label="Name of the directory where the model files are kept" help=""> ++ <options from_data_table="pepnovo_models"> ++ <column name="name" index="0"/> ++ <column name="value" index="2"/> ++ <filter type="unique_value" name="unique_set" column="0"/> ++ <validator type="no_options" message="No model directory available"/> ++ </options> + </param> + <param name="correct_pm" argument="-correct_pm" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Find optimal precursor mass and charge values" help=""/> + <param name="use_spectrum_charge" argument="-use_spectrum_charge" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Do not correct charge" help=""/> +@@ -51,8 +56,14 @@ + <param name="no_quality_filter" argument="-no_quality_filter" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Do not remove low quality spectra" help=""/> + <param name="fragment_tolerance" argument="-fragment_tolerance" type="float" optional="true" value="-1.0" label="The fragment tolerance (between 0 and 0.75 Da" help="Set to -1.0 to use model's default setting)"/> + <param name="pm_tolerance" argument="-pm_tolerance" type="float" optional="true" value="-1.0" label="The precursor mass tolerance (between 0 and 5.0 Da" help="Set to -1.0 to use model's default setting)"/> +- <param name="model" argument="-model" type="text" optional="true" value="CID_IT_TRYP" label="Name of the model that should be used" help=""> +- <expand macro="list_string_san"/> ++ <param name="model" argument="-model" type="select" label="Name of the model that should be used" help=""> ++ <options from_data_table="pepnovo_models"> ++ <column name="name" index="1"/> ++ <column name="value" index="1"/> ++ <filter type="param_value" ref="model_directory" column="2"/> ++ <filter type="unique_value" column="1"/> ++ <validator type="no_options" message="No model available"/> ++ </options> + </param> + <param name="digest" argument="-digest" display="radio" type="select" optional="false" label="Enzyme used for digestion (default TRYPSIN)" help=""> + <option value="TRYPSIN" selected="true">TRYPSIN</option> |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d SvmTheoreticalSpectrumGeneratorTrainer.xml --- a/SvmTheoreticalSpectrumGeneratorTrainer.xml Fri May 17 10:07:55 2019 -0400 +++ b/SvmTheoreticalSpectrumGeneratorTrainer.xml Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| b'@@ -1,324 +1,145 @@\n <?xml version=\'1.0\' encoding=\'UTF-8\'?>\n <!--This is a configuration file for the integration of a tools into Galaxy (https://galaxyproject.org/). This file was automatically generated using CTDConverter.-->\n <!--Proposed Tool Section: [Utilities]-->\n-<tool id="SvmTheoreticalSpectrumGeneratorTrainer" name="SvmTheoreticalSpectrumGeneratorTrainer" version="2.3.0">\n+<tool id="SvmTheoreticalSpectrumGeneratorTrainer" name="SvmTheoreticalSpectrumGeneratorTrainer" version="@TOOL_VERSION@+galaxy@GALAXY_VERSION@" profile="20.05">\n <description>Trainer for SVM models as input for SvmTheoreticalSpectrumGenerator</description>\n <macros>\n <token name="@EXECUTABLE@">SvmTheoreticalSpectrumGeneratorTrainer</token>\n <import>macros.xml</import>\n+ <import>macros_autotest.xml</import>\n+ <import>macros_test.xml</import>\n </macros>\n- <expand macro="references"/>\n+ <expand macro="requirements"/>\n <expand macro="stdio"/>\n- <expand macro="requirements"/>\n- <command detect_errors="aggressive"><![CDATA[SvmTheoreticalSpectrumGeneratorTrainer\n+ <command detect_errors="exit_code"><![CDATA[@QUOTE_FOO@\n+@EXT_FOO@\n+#import re\n+\n+## Preprocessing\n+mkdir in_spectra &&\n+ln -s \'$in_spectra\' \'in_spectra/${re.sub("[^\\w\\-_]", "_", $in_spectra.element_identifier)}.$gxy2omsext($in_spectra.ext)\' &&\n+mkdir in_identifications &&\n+ln -s \'$in_identifications\' \'in_identifications/${re.sub("[^\\w\\-_]", "_", $in_identifications.element_identifier)}.$gxy2omsext($in_identifications.ext)\' &&\n+mkdir model_output_file &&\n+\n+## Main program call\n+\n+set -o pipefail &&\n+@EXECUTABLE@ -write_ctd ./ &&\n+python3 \'$__tool_directory__/fill_ctd.py\' \'@EXECUTABLE@.ctd\' \'$args_json\' \'$hardcoded_json\' &&\n+@EXECUTABLE@ -ini @EXECUTABLE@.ctd\n+-in_spectra\n+\'in_spectra/${re.sub("[^\\w\\-_]", "_", $in_spectra.element_identifier)}.$gxy2omsext($in_spectra.ext)\'\n+-in_identifications\n+\'in_identifications/${re.sub("[^\\w\\-_]", "_", $in_identifications.element_identifier)}.$gxy2omsext($in_identifications.ext)\'\n+-model_output_file\n+\'model_output_file/output.${gxy2omsext("txt")}\'\n \n-#if $param_in_spectra:\n- -in_spectra $param_in_spectra\n-#end if\n-#if $param_in_identifications:\n- -in_identifications $param_in_identifications\n-#end if\n-#if $param_model_output_file:\n- -model_output_file $param_model_output_file\n-#end if\n-#if $param_precursor_charge:\n- -precursor_charge $param_precursor_charge\n-#end if\n-#if $param_write_training_files:\n- -write_training_files\n-#end if\n-#if $param_algorithm_number_intensity_levels:\n- -algorithm:number_intensity_levels $param_algorithm_number_intensity_levels\n-#end if\n-#if $param_algorithm_number_regions:\n- -algorithm:number_regions $param_algorithm_number_regions\n-#end if\n-#if $param_algorithm_parent_tolerance:\n- -algorithm:parent_tolerance $param_algorithm_parent_tolerance\n-#end if\n-#if $param_algorithm_peak_tolerance:\n- -algorithm:peak_tolerance $param_algorithm_peak_tolerance\n-#end if\n-#if $param_algorithm_add_b_ions:\n- -algorithm:add_b_ions\n- #if " " in str($param_algorithm_add_b_ions):\n- "$param_algorithm_add_b_ions"\n- #else\n- $param_algorithm_add_b_ions\n- #end if\n-#end if\n-#if $param_algorithm_add_y_ions:\n- -algorithm:add_y_ions\n- #if " " in str($param_algorithm_add_y_ions):\n- "$param_algorithm_add_y_ions"\n- #else\n- $param_algorithm_add_y_ions\n- #end if\n-#end if\n-#if $param_algorithm_add_a_ions:\n- -algorithm:add_a_ions\n-#end if\n-#if $param_algorithm_add_c_ions:\n- -algorithm:add_c_ions\n-#end if\n-#if $param_algorithm_add_x_ions:\n- -algorithm:add_x_ions\n-#end if\n-#if $param_algorithm_add_z_ions:\n- -algorithm:add_z_ions\n-#end if\n-#if $param_algorithm_add_losses:\n- -algorithm:add_losses\n-#end if\n-#if $param_algorithm_add_b2_ions:\n- -algorithm:add_b2_ions\n-#end if\n-#if $param_algorithm_add_y2_ions:\n- -algorithm:add_y2_ions\n-#end if\n-#if $param_algorithm_svm_svc_type:\n- -algorithm:svm:svc_type $param_algorithm_svm_svc_type\n-#end if\n-#if $param_algorithm_svm_svr_type:\n- -algorithm:svm:svr_type $param'..b'"(-degree) "/>\n- <param name="param_algorithm_svm_svr_gamma" type="float" min="0.0" optional="True" value="0.0" label="For POLY/RBF/SIGMOID" help="(-gamma) "/>\n- <param name="param_algorithm_svm_svr_C" type="float" value="1.0" label="Cost of constraint violation" help="(-C) "/>\n- <param name="param_algorithm_svm_svr_p" type="float" value="0.1" label="The epsilon for the loss function in epsilon-SVR" help="(-p) "/>\n- <param name="param_algorithm_svm_svr_nu" type="float" value="0.5" label="For NU_SVC, ONE_CLASS and NU_SVR" help="(-nu) "/>\n- <param name="param_algorithm_svm_svr_degree_start" type="integer" min="1" optional="True" value="1" label="starting point of degree" help="(-degree_start) "/>\n- <param name="param_algorithm_svm_svr_degree_step_size" type="integer" value="2" label="step size point of degree" help="(-degree_step_size) "/>\n- <param name="param_algorithm_svm_svr_degree_stop" type="integer" value="4" label="stopping point of degree" help="(-degree_stop) "/>\n- <param name="param_algorithm_svm_svr_gamma_start" type="float" min="0.0" max="1.0" optional="True" value="1e-05" label="starting point of gamma" help="(-gamma_start) "/>\n- <param name="param_algorithm_svm_svr_gamma_step_size" type="integer" value="100" label="step size point of gamma" help="(-gamma_step_size) "/>\n- <param name="param_algorithm_svm_svr_gamma_stop" type="float" value="0.1" label="stopping point of gamma" help="(-gamma_stop) "/>\n- <param name="param_algorithm_svm_svr_p_start" type="float" value="1e-05" label="starting point of p" help="(-p_start) "/>\n- <param name="param_algorithm_svm_svr_p_step_size" type="integer" value="100" label="step size point of p" help="(-p_step_size) "/>\n- <param name="param_algorithm_svm_svr_p_stop" type="float" value="0.1" label="stopping point of p" help="(-p_stop) "/>\n- <param name="param_algorithm_svm_svr_c_start" type="float" value="0.1" label="starting point of c" help="(-c_start) "/>\n- <param name="param_algorithm_svm_svr_c_step_size" type="integer" value="100" label="step size of c" help="(-c_step_size) "/>\n- <param name="param_algorithm_svm_svr_c_stop" type="integer" value="1000" label="stopping point of c" help="(-c_stop) "/>\n- <param name="param_algorithm_svm_svr_nu_start" type="float" min="0.0" max="1.0" optional="True" value="0.3" label="starting point of nu" help="(-nu_start) "/>\n- <param name="param_algorithm_svm_svr_nu_step_size" type="integer" value="2" label="step size of nu" help="(-nu_step_size) "/>\n- <param name="param_algorithm_svm_svr_nu_stop" type="float" min="0.0" max="1.0" optional="True" value="0.6" label="stopping point of nu" help="(-nu_stop) "/>\n- <expand macro="advanced_options">\n- <param name="param_force" display="radio" type="boolean" truevalue="-force" falsevalue="" checked="false" optional="True" label="Overwrite tool specific checks" help="(-force) "/>\n- </expand>\n </inputs>\n <outputs>\n- <data name="param_model_output_file" format="data"/>\n+ <data name="model_output_file" label="${tool.name} on ${on_string}: model_output_file" format="txt"/>\n+ <data name="ctd_out" format="xml" label="${tool.name} on ${on_string}: ctd">\n+ <filter>OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS</filter>\n+ </data>\n </outputs>\n- <help>Trainer for SVM models as input for SvmTheoreticalSpectrumGenerator\n+ <tests>\n+ <expand macro="autotest_SvmTheoreticalSpectrumGeneratorTrainer"/>\n+ <expand macro="manutest_SvmTheoreticalSpectrumGeneratorTrainer"/>\n+ </tests>\n+ <help><![CDATA[Trainer for SVM models as input for SvmTheoreticalSpectrumGenerator\n \n \n-For more information, visit https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Documentation/release/2.3.0/html/UTILS_SvmTheoreticalSpectrumGeneratorTrainer.html</help>\n+For more information, visit http://www.openms.de/documentation/UTILS_SvmTheoreticalSpectrumGeneratorTrainer.html]]></help>\n+ <expand macro="references"/>\n </tool>\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d filetypes.txt --- a/filetypes.txt Fri May 17 10:07:55 2019 -0400 +++ b/filetypes.txt Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| @@ -1,29 +1,85 @@ -# CTD type # Galaxy type # Long Galaxy data type # Mimetype -csv tabular galaxy.datatypes.tabular:Tabular -fasta fasta galaxy.datatypes.sequence:Fasta -FASTA fasta galaxy.datatypes.sequence:Fasta -ini txt galaxy.datatypes.data:Text -txt txt galaxy.datatypes.data:Text -options txt galaxy.datatypes.data:Text -grid grid galaxy.datatypes.data:Grid -html html galaxy.datatypes.text:Html text/html -HTML html galaxy.datatypes.text:Html text/html -TraML traml galaxy.datatypes.proteomics:TraML application/xml -traML traml galaxy.datatypes.proteomics:TraML application/xml -XML xml galaxy.datatypes.xml:GenericXml application/xml -consensusXML consensusxml galaxy.datatypes.proteomics:ConsensusXML application/xml -edta tabular galaxy.datatypes.tabular:Tabular -featureXML featurexml galaxy.datatypes.proteomics:FeatureXML application/xml -idXML idxml galaxy.datatypes.proteomics:IdXML application/xml -mzML mzml galaxy.datatypes.proteomics:MzML application/xml -mzXML mzxml galaxy.datatypes.proteomics:MzXML application/xml -pepXML pepxml galaxy.datatypes.proteomics:PepXml application/xml -qcML qcml galaxy.datatypes.xml:GenericXml application/xml -trafoXML trafoxml galaxy.datatypes.xml:GenericXml application/xml -tsv tabular galaxy.datatypes.tabular:Tabular -xsd txt galaxy.datatypes.data:Text -mzq mzq galaxy.datatypes.proteomics:MzQuantML application/xml -msp msp galaxy.datatypes.proteomics:Msp -mzid mzid galaxy.datatypes.proteomics:MzIdentML application/xml -png png galaxy.datatypes.images:Png image/png -mgf mgf galaxy.datatypes.proteomics:Mgf +# CTD type # Galaxy type +# the following lines need to be at the top in order to ensure +# correct translation Galaxy->CTD type for the ambiguous cases +# (should only be relevant for the autogenerated tests [which +# do not set the ftype of the inputs]) +txt txt +tsv tabular + +##analysisXML +# XTandemAdapter output is named xml in OMS (which is to unspecific) and bioml in Galaxy .. so this is renamed via hardcoded parameters +bioml xml +consensusXML consensusxml +# TODO csv is problematic, since csv often actually means tsv .. but not always +csv csv +##dat +dta dta +dta2d dta2d +edta edta +fa fasta +fas fasta +fasta fasta +FASTA fasta +featureXML featurexml +featurexml featurexml +# fid +html html +HTML html +idXML idxml +##ini txt +json json +kroenik kroenik +mascotXML mascotxml +mgf mgf +mrm mrm +ms sirius.ms +ms2 ms2 +msp msp +mzData mzdata +mzid mzid +# important to have mzML first, since LuciphorAdapter is case sensitive https://github.com/OpenMS/OpenMS/issues/4444 +mzML mzml +mzml mzml +mzq mzq +mzTab mztab +mzXML mzxml +novor txt +obo obo +# I guess this is the idXML output of omssa +omssaXML idxml +osw osw +OSW osw +params txt +paramXML paramxml +fasta peff +peplist peplist +# TODO pep.xml should be removed with OMS 2.6 https://github.com/OpenMS/OpenMS/pull/4541 +pep.xml pepxml +pepXML pepxml +png png +PNG png +protXML protxml +psms psms +# TODO implement or use +# psq +pqp pqp +qcML qcml +spec.xml spec.xml +splib splib +sqMass sqmass +tandem.xml tandem +trafoXML trafoxml +traML traml +TraML traml +tab tabular +## MOVED TO TOP txt txt +raw thermo.raw +## xls: SpectraSTSearchAdapter https://github.com/OpenMS/OpenMS/pull/4419 +xls tsv +XML xml +xml xml +xquest.xml xquest.xml +xsd xml + +# TODO needs to be implemented, needs to be below xml in order that Galaxy->OMS mapping gives xml +# cachedMzML xml |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d fill_ctd.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/fill_ctd.py Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| @@ -0,0 +1,190 @@ +import collections +import json +import operator +import os +import re +import subprocess +import sys +from functools import reduce # forward compatibility for Python 3 + +from CTDopts.CTDopts import ( + _Choices, + _InFile, + _Null, + _NumericRange, + CTDModel +) + + +def getFromDict(dataDict, mapList): + return reduce(operator.getitem, mapList, dataDict) + + +def setInDict(dataDict, mapList, value): + getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value + + +def mergeDicts(d, e): + """ + insert values from the dict e into dict d + no values of d are overwritten + """ + for k, v in e.items(): + if (k in d and isinstance(d[k], dict) and isinstance(e[k], collections.abc.Mapping)): + mergeDicts(d[k], e[k]) + elif k not in d and not isinstance(e[k], collections.abc.Mapping): + d[k] = e[k] + else: + sys.stderr.write("fill_ctd.py: could not merge key %s for %s in %s" % (k, d, e)) + sys.exit(1) + + +def _json_object_hook_noenvlookup(d): + return _json_object_hook(d, envlookup=False) + + +def _json_object_hook(d, envlookup=True): + """ + wee helper to transform the json written by galaxy + while loading + - True/False (bool objects) -> "true"/"false" (lowercase string) + - data inputs with multiple and optional true give [None] if no file is given -> [] + - None -> "" (empty string) + - replace bash expressions (if envlookup is True): + - environment variables (need to consist capital letters and _) by their value + - expressions + """ + for k in d.keys(): + # if type(d[k]) is bool: + # d[k] = str(d[k]).lower() + # else + if type(d[k]) is list and len(d[k]) == 1 and d[k][0] is None: + d[k] = [] + elif d[k] is None: + d[k] = "" + elif envlookup and type(d[k]) is str and d[k].startswith("$"): + m = re.fullmatch(r"\$([A-Z_]+)", d[k]) + if m: + d[k] = os.environ.get(m.group(1), "") + continue + m = re.fullmatch(r"\$(\{[A-Z_]+):-(.*)\}", d[k]) + if m: + d[k] = os.environ.get(m.group(1), m.group(2)) + continue + + try: + p = subprocess.run("echo %s" % d[k], shell=True, check=True, stdout=subprocess.PIPE, encoding="utf8") + d[k] = p.stdout.strip() + except subprocess.CalledProcessError: + sys.stderr.write("fill_ctd error: Could not evaluate %s" % d[k]) + continue + return d + + +def qstring2list(qs): + """ + transform a space separated string that is quoted by " into a list + """ + lst = list() + qs = qs.split(" ") + quoted = False + for p in qs: + if p == "": + continue + if p.startswith('"') and p.endswith('"'): + lst.append(p[1:-1]) + elif p.startswith('"'): + quoted = True + lst.append(p[1:] + " ") + elif p.endswith('"'): + quoted = False + lst[-1] += p[:-1] + else: + if quoted: + lst[-1] += p + " " + else: + lst.append(p) + return lst + + +def fix_underscores(args): + if type(args) is dict: + for k in list(args.keys()): + v = args[k] + if type(v) is dict: + fix_underscores(args[k]) + if k.startswith("_"): + args[k[1:]] = v + del args[k] + elif type(args) is list: + for i, v in enumerate(args): + if type(v) is dict: + fix_underscores(args[i]) + + +input_ctd = sys.argv[1] + +# load user specified parameters from json +with open(sys.argv[2]) as fh: + args = json.load(fh, object_hook=_json_object_hook_noenvlookup) + +# load hardcoded parameters from json +with open(sys.argv[3]) as fh: + hc_args = json.load(fh, object_hook=_json_object_hook) + +# insert the hc_args into the args +mergeDicts(args, hc_args) + +if "adv_opts_cond" in args: + args.update(args["adv_opts_cond"]) + del args["adv_opts_cond"] + +# IDMapper has in and spectra:in params, in is used in out as format_source", +# which does not work in Galaxy: https://github.com/galaxyproject/galaxy/pull/9493" +# therefore hardcoded params change the name of spectra:in to spectra:_in +# which is corrected here again +# TODO remove once PR is in and adapt profile accordingly +fix_underscores(args) + +model = CTDModel(from_file=input_ctd) + +# transform values from json that correspond to +# - old style booleans (string + restrictions) -> transformed to a str +# - unrestricted ITEMLIST which are represented as strings +# ("=quoted and space separated) in Galaxy -> transform to lists +# - optional data input parameters that have defaults and for which no +# value is given -> overwritte with the default +for p in model.get_parameters(): + + # check if the parameter is in the arguments from the galaxy tool + # (from the json file(s)), since advanced parameters are absent + # if the conditional is set to basic parameters + try: + getFromDict(args, p.get_lineage(name_only=True)) + except KeyError: + # few tools use dashes in parameters which are automatically replaced + # by underscores by Galaxy. in these cases the dictionary needs to be + # updated + # TODO might be removed later https://github.com/OpenMS/OpenMS/pull/4529 + try: + lineage = [_.replace("-", "_") for _ in p.get_lineage(name_only=True)] + val = getFromDict(args, lineage) + except KeyError: + continue + else: + setInDict(args, lineage, val) + + if p.type is str and type(p.restrictions) is _Choices and set(p.restrictions.choices) == set(["true", "false"]): + v = getFromDict(args, p.get_lineage(name_only=True)) + setInDict(args, p.get_lineage(name_only=True), str(v).lower()) + + elif p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange): + v = getFromDict(args, p.get_lineage(name_only=True)) + if type(v) is str: + setInDict(args, p.get_lineage(name_only=True), qstring2list(v)) + elif p.type is _InFile and not (p.default is None or type(p.default) is _Null): + v = getFromDict(args, p.get_lineage(name_only=True)) + if v in [[], ""]: + setInDict(args, p.get_lineage(name_only=True), p.default) + +model.write_ctd(input_ctd, arg_dict=args) |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d fill_ctd_clargs.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/fill_ctd_clargs.py Wed Sep 09 19:57:32 2020 +0000 |
| b |
| @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +from argparse import ArgumentParser +from io import StringIO + +from CTDopts.CTDopts import ( + CTDModel, + ModelTypeError, + Parameters +) + +if __name__ == "__main__": + # note add_help=False since otherwise arguments starting with -h will + # trigger an error (despite allow_abbreviate) + parser = ArgumentParser(prog="fill_ctd_clargs", + description="fill command line arguments" + "into a CTD file and write the CTD file to", + add_help=False, allow_abbrev=False) + parser.add_argument("--ctd", dest="ctd", help="input ctd file", + metavar='CTD', default=None, required=True) + args, cliargs = parser.parse_known_args() + # load CTDModel + model = None + try: + model = CTDModel(from_file=args.ctd) + except ModelTypeError: + pass + try: + model = Parameters(from_file=args.ctd) + except ModelTypeError: + pass + assert model is not None, "Could not parse %s, seems to be no CTD/PARAMS" % (args.ctd) + + # get a dictionary of the ctd arguments where the values of the parameters + # given on the command line are overwritten + margs = model.parse_cl_args(cl_args=cliargs, ignore_required=True) + + # write the ctd with the values taken from the dictionary + out = StringIO() + ctd_tree = model.write_ctd(out, margs) + print(out.getvalue()) |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d generate-foo.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/generate-foo.sh Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| b'@@ -0,0 +1,217 @@\n+#!/usr/bin/env bash\n+\n+# parse test definitions from OpenMS sources for a tool with a given id\n+function get_tests2 {\n+\tid=$1\n+\t>&2 echo "generate tests for $id"\n+\techo \'<xml name="autotest_\'"$id"\'">\'\n+\n+\t# get the tests from the CMakeLists.txt\n+\t# 1st remove some tests\n+\t# - Filefilter with empty select_palarity value (empty is not in the list of allowed options)\n+\t# - MassTraceExtractor with outdated ini file leading to wrong parameters https://github.com/OpenMS/OpenMS/issues/4386\n+\t# - OpenSwathMzMLFileCacher with -convert_back argumen https://github.com/OpenMS/OpenMS/issues/4399\n+ # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt\n+\t# - several tools with duplicated input (leads to conflict when linking)\n+\t# - TOFCalibration inputs we extension (also in prepare_test_data) https://github.com/OpenMS/OpenMS/pull/4525\n+\t# - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456)\n+\t# - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed)\n+\t# - some input files are originally in a subdir (degenerated cases/), but not in test-data\n+\t# - SeedListGenerator: https://github.com/OpenMS/OpenMS/issues/4404\n+\t# - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet)\n+\tCMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake |\n+\t\tsed \'s@${DATA_DIR_SHARE}/@@g\' |\n+\t\tgrep -v \'OpenSwathMzMLFileCacher .*-convert_back\' |\n+\t \tsed \'s/${TMP_RIP_PATH}/""/\' |\n+\t\tsed \'s@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @\' |\n+\t\tgrep -v "MaRaClusterAdapter.*-consensus_out"|\n+ \t\tgrep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " |\n+\t\tsed \'s@degenerate_cases/@@g\' |\n+\t\tgrep -v \'TOPP_SeedListGenerator_3"\' | \n+\t\tegrep -v \'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"\')\n+\n+\n+# \t\tgrep -v \'FileFilter.*-spectra:select_polarity ""\' |\n+# \t\tgrep -v \'MassTraceExtractor_2.ini \' |\n+# \t\tgrep -v "FileMerger_6_input2.mzML.*FileMerger_6_input2.mzML" |\n+# \t\tgrep -v "IDMerger_1_input1.idXML.*IDMerger_1_input1.idXML" |\n+# \t\tgrep -v "degenerated_empty.idXML.*degenerated_empty.idXML" |\n+# \t\tgrep -v "FeatureLinkerUnlabeledKD_1_output.consensusXML.*FeatureLinkerUnlabeledKD_1_output.consensusXML" |\n+# \t\tgrep -v "FeatureLinkerUnlabeledQT_1_output.consensusXML.*FeatureLinkerUnlabeledQT_1_output.consensusXML" |\n+\n+\t# 1st part is a dirty hack to join lines containing a single function call, e.g.\n+\t# addtest(....\n+\t# ....)\n+\techo "$CMAKE" | sed \'s/#.*//; s/^\\s*//; s/\\s*$//\' | grep -v "^#" | grep -v "^$" | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | \n+\t\tgrep -iE "add_test\\(\\"(TOPP|UTILS)_.*/$id " | egrep -v "_prepare\\"|_convert|WRITEINI|WRITECTD|INVALIDVALUE" | while read -r line\n+\tdo\n+\t\tline=$(echo "$line" | sed \'s/add_test("\\([^"]\\+\\)"/\\1/; s/)$//; s/\\${TOPP_BIN_PATH}\\///g;s/\\${DATA_DIR_TOPP}\\///g; s#THIRDPARTY/##g\')\n+\t\t# >&2 echo $line\n+\t\ttest_id=$(echo "$line" | cut -d" " -f 1)\n+\t\ttool_id=$(echo "$line" | cut -d" " -f 2)\n+\t\tif [[ $test_id =~ _out_?[0-9]? ]]; then\n+\t\t\t>&2 echo " skip $test_id $line"\n+\t\t\tcontinue\n+\t\tfi\n+\t\tif [[ ${id,,} != ${tool_id,,} ]]; then\n+\t\t\t>&2 echo " skip $test_id ($id != $tool_id) $line"\n+\t\t\tcontinue\n+\t\tfi\n+\n+\t\t#remove tests with set_tests_properties(....PROPERTIES WILL_FAIL 1)\n+\t\tif grep -lq "$test_id"\'\\".* PROPERTIES WILL_FAIL 1\' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then\n+\t\t\t>&2 echo " skip failing "$test_id\n+\t\t\tcontinue\n+\t\tfi\n+\t\ttes=" <test>\\n"\n+\t\tline=$(fix_tmp_files "$line")\n+\t\tline=$(unique_files "$line")\n+\t\t# >&2 echo $line\n+\t\t#if there is an ini file then we use this to generate the test\n+\t\t#otherwise the ctd file is used\n+\t\t#other command line parameters are inserted later into this xml\n+\t\tif grep -lq "\\'..b'f to the expected file.\n+#problem: the extension of the tmp files is unusable for test generation.\n+#unfortunately the extensions used in the DIFF lines are not always usable for the CLI\n+#(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt)\n+#this function replaces the tmp file by the expected file. \n+function fix_tmp_files {\n+#\t>&2 echo "FIX $line"\n+\tret=""\n+\tfor a in $@; do\n+\t\tif [[ ! $a =~ .tmp$ ]]; then\n+\t\t\tret="$ret $a"\n+\t\t\tcontinue\n+\t\tfi\n+#\t\t>&2 echo " a "$a\n+\t\tg=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | grep \'\\${DIFF}.*\'"$a")\n+#\t\t>&2 echo " g "$g\n+\t\tin1=$(sed \'s/.*-in1 \\([^ ]\\+\\).*/\\1/\' <<<$g)\n+\t\t# >&2 echo " in1 "$in1\n+\t\tif [[ "$a" != "$in1" ]]; then\n+\t\t\tret="$ret $a"\n+\t\t\tcontinue\n+\t\tfi\n+\t\tin2=$(sed \'s/.*-in2 \\([^ ]\\+\\).*/\\1/\' <<<$g)\n+\t\tin2=$(basename $in2 | sed \'s/)$//\')\n+\t\t# >&2 echo " in2 "$in2\n+\t\tif [[ -f "test-data/$in2" ]]; then\n+\t\t\tln -fs "$in1" "test-data/$in2"\n+\t\t\tret="$ret $in2"\n+\t\telse\n+\t\t\tret="$ret $a"\n+\t\tfi\n+\tdone\n+#\t>&2 echo "--> $ret"\n+\techo "$ret"\n+}\n+\n+function link_tmp_files {\n+ # note this also considers commented lines (starting with a #)\n+ # because of tests where the diff command is commented and we\n+ # still want to use the extension of these files\n+ cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed \'s/^\\s*//; s/\\s*$//\' | grep -v "^$" | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | grep "\\${DIFF}" | while read -r line\n+ do\n+ in1=$(sed \'s/.*-in1 \\([^ ]\\+\\).*/\\1/\' <<<$line)\n+ in1=$(basename $in1 | sed \'s/)$//\')\n+ in2=$(sed \'s/.*-in2 \\([^ ]\\+\\).*/\\1/\' <<<$line)\n+ in2=$(basename $in2 | sed \'s/)$//\')\n+ if [[ "$in1" == "$in2" ]]; then\n+ >&2 echo "not linking equal $in1 $in2"\n+ continue\n+ fi\n+ ln -f -s $in1 test-data/$in2\n+ done\n+ for i in test-data/*.tmp\n+ do\n+ if [ ! -e test-data/$(basename $i .tmp) ]; then\n+ ln -s $(basename $i) test-data/$(basename $i .tmp)\n+ #ln -s $(basename $i) test-data/$(basename $i .tmp)\n+ else\n+ ln -fs $(basename $i) test-data/$(basename $i .tmp)\n+ fi\n+ done\n+}\n+\n+\n+\n+# parse data preparation calls from OpenMS sources for a tool with a given id\n+function prepare_test_data {\n+# \tid=$1\n+# | egrep -i "$id\\_.*[0-9]+(_prepare\\"|_convert)?"\n+\tcat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed \'s/#.*$//\'| sed \'s/^\\s*//; s/\\s*$//\' | grep -v "^$" | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | \n+\t\tsed \'s/degenerate_cases\\///\' | \n+\t\tegrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | \n+\t\tgrep add_test | \n+\t\tegrep "TOPP|UTILS" |\n+\t\tsed \'s@${DATA_DIR_SHARE}/@@g;\'|\n+\t\tsed \'s@${TMP_RIP_PATH}@dummy2.tmp@g\'|\n+\t\tsed \'s@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @\'| \n+\twhile read line\n+\tdo\n+\t\ttest_id=$(echo "$line" | sed \'s/add_test(//; s/"//g; s/)[^)]*$//; s/\\${TOPP_BIN_PATH}\\///g;s/\\${DATA_DIR_TOPP}\\///g; s#THIRDPARTY/##g\' | cut -d" " -f1)\n+\n+\t\tif grep -lq "$test_id"\'\\".* PROPERTIES WILL_FAIL 1\' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then\n+\t\t\t>&2 echo " skip failing "$test_id\n+\t\t\tcontinue\n+\t\tfi\n+\n+\t\tline=$(echo "$line" | sed \'s/add_test("//; s/)[^)]*$//; s/\\${TOPP_BIN_PATH}\\///g;s/\\${DATA_DIR_TOPP}\\///g; s#THIRDPARTY/##g\' | cut -d" " -f2-)\n+\t\t# line="$(fix_tmp_files $line)"\n+\t\techo "$line > $test_id.stdout 2> $test_id.stderr"\n+\t\techo "if [[ \\"\\$?\\" -ne \\"0\\" ]]; then >&2 echo \'$test_id failed\'; >&2 echo -e \\"stderr:\\n\\$(cat $test_id.stderr | sed \'s/^/ /\')\\"; echo -e \\"stdout:\\n\\$(cat $test_id.stdout)\\";fi"\t\n+ done\n+}\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d generate.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/generate.sh Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# VERSION=2.3 +# CONDAPKG=https://anaconda.org/bioconda/openms/2.3.0/download/linux-64/openms-2.3.0-py27h932d754_3.tar.bz2 +# VERSION=2.4 +# CONDAPKG=https://anaconda.org/bioconda/openms/2.4.0/download/linux-64/openms-2.4.0-py27h574aadf_1.tar.bz2 + +VERSION=2.5 +FILETYPES="filetypes.txt" +PROFILE="20.05" +## FILETYPES_RE=$(grep -v "^#" $FILETYPES | grep -v "^$" | cut -f 1 -d" " | tr '\n' '|' | sed 's/|$//'| sed 's/|/\\|/g') + +export tmp=$(mktemp -d) + +export CTDCONVERTER="$tmp/CTDConverter" + +export PYTHONPATH="$(pwd)/CTDopts/" +############################################################################### +## reset old data +############################################################################### +# rm $(ls *xml |grep -v macros) +# rm -rf ctd +# mkdir -p ctd +# echo "" > prepare_test_data.sh + +############################################################################### +## generate tests +## also creates +## - conda environment (for executing the binaries) and +## - the git clone of OpenMS (for generating the tests) +## - ctd files +############################################################################### +bash ./test-data.sh ./macros_autotest.xml + +############################################################################### +## get the +## - conda package (for easy access and listing of the OpenMS binaries), +############################################################################### +# if [ ! -d $OPENMSPKG ]; then +# mkdir $OPENMSPKG/ +# wget -P $OPENMSPKG/ "$CONDAPKG" +# tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C OpenMS$VERSION-pkg/ +# rm $OPENMSPKG/"$(basename $CONDAPKG)" +# fi + +############################################################################### +## Get python libaries for CTD -> Galaxy conversion +## TODO fix to main repo OR conda packkage if PRs are merged +############################################################################### +# if [ ! -d CTDopts ]; then +# # git clone https://github.com/genericworkflownodes/CTDopts CTDopts +# git clone -b topic/no-1-2x https://github.com/bernt-matthias/CTDopts CTDopts +# fi +if [ ! -d $CTDCONVERTER ]; then + #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter + git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER +fi +# export PYTHONPATH=$(pwd)/CTDopts +############################################################################### +## conversion ctd->xml +############################################################################### +source $(dirname $(which conda))/../etc/profile.d/conda.sh +conda activate $tmp/OpenMS$VERSION-env +python $CTDCONVERTER/convert.py galaxy -i ctd/*ctd -o ./ -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --test-macros macros_autotest.xml --test-macros-prefix autotest_ --test-macros macros_test.xml --test-macros-prefix manutest_ --tool-version $VERSION --tool-profile $PROFILE > convert.out 2> convert.err +if [[ "$?" -ne "0" ]]; then >&2 echo 'CTD -> XML conversion failed'; >&2 echo -e "stderr:\n$(cat convert.err)"; fi +conda deactivate + +patch PepNovoAdapter.xml < PepNovoAdapter.patch +patch OMSSAAdapter.xml < OMSSAAdapter.patch + +# #-b version log debug test in_type executable pepnovo_executable param_model_directory rt_concat_trafo_out param_id_pool + +# for i in A-E F-H I-L M-N O-P Q-Z +# do +# planemo t [$i]*xml --galaxy_branch release_20.05 --galaxy_python_version 3.7 --test_output $i.html --test_output_json $i.json & +# done |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d hardcoded_params.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hardcoded_params.json Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| @@ -0,0 +1,247 @@ +{ + "#": "blacklist parameters", + + "version": [{"value": "@"}], + "debug": [{"value": "@"}], + "algorithm:debug": [{"value": "@"}], + "java_memory": [{"value": "@"}], + "java_permgen": [{"value": "@"}], + "#": "type of input is always determined from the file extension ", + "in_type": [{"value": "@"}], + + "#": "tool specific blacklist parameters", + + "convert_back": [{"value": "@", "tools": ["OpenSwathMzMLFileCacher"]}], + "NET_executable": [{ + "value": "@", + "tools": ["FileConverter"] + }], + + + "params_file": [{"value": "@", "tools": ["SpectraSTSearchAdapter"]}], + + "#": "TODO not usable in 2.5 https://github.com/OpenMS/OpenMS/issues/4456, corresponding test currently disabled", + "consensus_out": [{"value": "@", "tools": ["MaRaClusterAdapter"]}], + "#": "TODO would need treatment as prefix-output", + "output_directory": [{"value": "@", "tools": ["MaRaClusterAdapter"]}], + + "#": "hardcode parameter values", + + "comet_version": [{ + "value":"2016.01 rev. 3" + }], + "comet_executable": [{ + "value":"comet" + }], + "crux_executable": [{ + "value": "crux" + }], + "fido_executable": [{ + "value":"Fido" + }], + "fidocp_executable": [{ + "value":"FidoChooseParameters" + }], + "maracluster_executable": [{ + "value":"/home/berntm/projects/tools-galaxyp/tools/openms/OpenMS2.5.0-git/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster" + }], + "mascot_directory": [{ + "value":"TODO" + }], + "myrimatch_executable": [{ + "value":"myrimatch" + }], + "omssa_executable": [{ + "value":"$(dirname $(realpath $(which omssacl)))/omssacl" + }], + "ThermoRaw_executable": [{ + "value": "ThermoRawFileParser.exe", + "tools": ["FileConverter"] + }], + "pepnovo_executable": [{ + "value":"pepnovo" + }], + "percolator_executable": [{ + "value":"percolator" + }], + "xtandem_executable": [{ + "value":"xtandem" + }], + "executable": [ + { + "value":"$(dirname $(realpath $(which luciphor2)))/luciphor2.jar", + "tools": ["LuciphorAdapter"] + }, { + "value":"/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar", + "tools": ["MSFraggerAdapter"] + }, { + "value":"$(msgf_plus -get_jar_path)", + "tools": ["MSGFPlusAdapter"] + }, { + "value": "/home/berntm/Downloads/novor/lib/novor.jar", + "tools": ["NovorAdapter"] + }, { + "value":"$(which sirius)", + "tools": ["SiriusAdapter", "AssayGeneratorMetabo"] + }, { + "value":"spectrast", + "tools": ["SpectraSTSearchAdapter"] + } + ], + "r_executable": [{ + "value":"R" + }], + "rscript_executable": [{ + "value":"Rscript" + }], + "java_executable": [{ + "value":"java" + }], + "log": [{ + "value":"log.txt" + }], + "tempDirectory": [{ + "value":"$TMP_DIR" + }], + "temp_data_directory": [{ + "value":"$TMP_DIR" + }], + "algorithm:Preprocessing:tmp_dir": [{ + "value":"$TMP_DIR" + }], + "no_progress": [{ + "value": true + }], + "#": "only used in LuciphorAdapter at the moment, inconsistency will be fixed", + "num_threads": [{ + "value":"${GALAXY_SLOTS:-1}" + }], + "threads": [{ + "value": "${GALAXY_SLOTS:-1}" + }], + "#": "hardcode the outer loop threads for OpenSwathWorkflow", + "outer_loop_threads": [{ + "value": "1", + "tools": ["OpenSwathWorkflow"] + }], + "separator": [{ + "value": ",", + "tools": ["IDMassAccuracy"] + }], + "#": "overwrite/add Galaxy xml attributes of some parameters (names need to start with param_)", + + "#": "test is not a hardcoded value since we need to set it in the tool tests", + "test": [{ + "CTD:type": "text", + "XML:type": "hidden" + }], + + "#": "overwrite CTD attributes of some parameters (some are not possible, e.g. type)", + + "#": "for some tools the user needs to select the desired output type since detection by extension makes no sense for galaxy tools", + "out_type": [{ + "CTD:required": true, + "CTD:advanced": false + }], + + "#": "SeedListGenerator with consensusXML input needs a dynamic number of outputs that depends on the content of the input, so we remove this options at the moment because its hard or impossible to implement in Galaxy, https://github.com/OpenMS/OpenMS/issues/4404 .. see also in parameter", + "#": "FileInfo, MapStatistics, SequenceCoverageCalculator wo -out just writes to stdout. not wanted here", + "#": "MascotAdapter: https://github.com/OpenMS/OpenMS/pull/4553", + "#": "Phosphoscoring: https://github.com/OpenMS/OpenMS/pull/4569", + "#": "MzMLSplitter output prefix https://github.com/OpenMS/OpenMS/issues/4404", + "#": "IDRipper: blacklist out (is doing the same as the output-prefix out-path)", + "out": [{ + "CTD:is_list": false, + "tools": ["SeedListGenerator"] + }, { + "CTD:required": true, + "tools": ["FileInfo", "MapStatistics", "SequenceCoverageCalculator"] + }, { + "CTD:restrictions": "idXML", + "tools": ["MascotAdapter"] + }, { + "CTD:restrictions": "idXML", + "tools": ["PhosphoScoring"] + }, { + "CTD:type": "output-prefix", + "CTD:required": "true", + "CTD:restrictions": "mzml", + "tools": ["MzMLSplitter"] + }, { + "value": "@", + "tools": ["IDRipper"] + }], + + "#": "https://github.com/OpenMS/OpenMS/pull/4552 .. if this is removed the test will need out_cm_FLAG", + "out_cm": [{ + "CTD:required": true, + "tools": ["MetaboliteAdductDecharger"] + }], + + + "#": "https://github.com/OpenMS/OpenMS/pull/4451", + "out_xquestxml": [{ + "CTD:restrictions": "xquest.xml", + "tools": ["OpenPepXL", "OpenPepXLLF"] + }], + "out_xquest_specxml": [{ + "CTD:restrictions": "spec.xml", + "tools": ["OpenPepXL", "OpenPepXLLF"] + }], + + "#": "Try to remove xml data type whereever possible", + "#": "XTandem Adapter output is called .xml in OMS -> use Galaxy's bioml", + "xml_out": [{ + "CTD:restrictions": "bioml", + "tools": ["XTandemAdapter"] + }], + + "#": "IDFileConverter remove xml", + "#": "OpenSwathWorkflow make in single file input and all outputs non-optional", + "#": "XFDR does not need xml .. redundant with xquest.xml TODO check if list is up to date with each new release", + "#": "SpectraSTSearchAdapter does not need xml .. redundant with pep.xml TODO check if list is up to date with each new release", + "#": "SeedListGenerator https://github.com/OpenMS/OpenMS/issues/4404 .. see also out parameter", + "in": [{ + "CTD:restrictions": "pepXML,protXML,mascotXML,omssaXML,bioml,psms,tsv,idXML,mzid,xquest.xml", + "tools": ["IDFileConverter"] + }, { + "CTD:is_list": false, + "tools": ["OpenSwathWorkflow"] + }, { + "CTD:restrictions": "idXML,mzid,xquest.xml", + "tools": ["XFDR"] + }, { + "CTD:restrictions": "txt,tsv,pep.xml,pepXML,html", + "tools": ["SpectraSTSearchAdapter"] + }, { + "CTD:restrictions": "mzML,idXML,featureXML", + "tools": ["SeedListGenerator"] + }], + + "#": "parameter is a flag but restrictions in wrong order: TODO https://github.com/OpenMS/OpenMS/pull/4455", + "# masstrace_snr_filtering": [{ + "CTD:restrictions": "true,false", + "tools": ["MassTraceExtractor"] + }], + + "#": "IDMapper has in and spectra:in params, in is used in out as format_source", + "#": "which does not work in Galaxy: https://github.com/galaxyproject/galaxy/pull/9493", + "spectra:in": [{ + "CTD:name": "_in", + "tools": ["IDMapper"] + }], + + "#": "output-prefix", + "out_path": [{ + "CTD:type": "output-prefix", + "CTD:required": "true", + "CTD:restrictions": "idXML", + "tools": ["IDRipper"] + }], + "outputDirectory": [{ + "CTD:type": "output-prefix", + "CTD:required": "true", + "CTD:restrictions": "mzml", + "tools": ["OpenSwathFileSplitter"] + }] +} |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d macros.xml --- a/macros.xml Fri May 17 10:07:55 2019 -0400 +++ b/macros.xml Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| @@ -1,11 +1,21 @@ <?xml version='1.0' encoding='UTF-8'?> +<!-- CTD2Galaxy depends on this file and on the stdio, advanced_options macros! + You can edit this file to add your own macros, if you so desire, or you can + add additional macro files using the m/macros parameter --> <macros> + <token name="@TOOL_VERSION@">2.5</token> + <token name="@GALAXY_VERSION@">0</token> <xml name="requirements"> <requirements> - <requirement type="package" version="2.3">openms</requirement> - <requirement type="package" version="15.12.15.2">xtandem</requirement> - <requirement type="package" version="1.0">fido</requirement> - <requirement type="package" version="2017.07.21">msgf_plus</requirement> + <requirement type="package" version="@TOOL_VERSION@">openms</requirement> + <requirement type="package" version="@TOOL_VERSION@">openms-thirdparty</requirement> + <!-- makeblastdb for OMSSAAdapter --> + <requirement type="package" version="2.9.0">blast</requirement> + <requirement type="package" version="8.0.192">openjdk</requirement> + <!--<requirement type="package" version="5.0.0">tpp</requirement>--> + <!-- for realpath (used e.g. in LuciphorAdapter) --> + <!--<requirement type="package" version="8.25">coreutils</requirement>--> + <requirement type="package" version="1.4">ctdopts</requirement> <yield/> </requirements> </xml> @@ -21,8 +31,8 @@ <citation type="doi">doi:10.1186/1471-2105-9-163</citation> </citations> </xml> - <xml name="advanced_options"> - <conditional name="adv_opts"> + <xml name="adv_opts_macro"> + <conditional name="adv_opts_cond"> <param name="adv_opts_selector" type="select" label="Advanced Options"> <option value="basic" selected="True">Hide Advanced Options</option> <option value="advanced">Show Advanced Options</option> @@ -33,4 +43,87 @@ </when> </conditional> </xml> -</macros> + + <!-- sanitizers and validators --> + <xml name="list_string_val"> + <validator type="regex" message="parameter must not start with $">^[^$]</validator> + + <validator type="regex" message="a space separated list of string is needed (strings that contain spaces can be quoted with ")">^ *((?:\"[^\"]*\" +)|(?:[^ \"]+ +))*((?:\"[^\"]*\")|(?:[^ \"]+)) *$</validator> + </xml> + <xml name="list_string_san"> + <sanitizer> + <valid initial="string.printable"> +<!-- <remove value="'"/>--> +<!-- <remove value="\"/>--><!-- otherwise the user could quote the final quote --> +<!-- <remove value="`"/>--> +<!-- <remove value="$"/>--> +<!-- <remove value="<"/>--> + <!--<remove value="&"/> removed for MascotAdapterOnline -Mascot_server:export_params which is a URL POST string which can contain & .. could be mapped to & but there is still a & --> + <!--<remove value=";"/>--> +<!-- <remove value="#"/>--> + </valid> + </sanitizer> + </xml> + <xml name="list_float_valsan"> + <validator type="regex" message="a space separated list of float values is required">^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$</validator> + <yield/> + <sanitizer> + <valid initial="string.digits"> + <add value=" "/> + <add value="."/> + <add value="E"/> + <add value="e"/> + <add value="+"/> + <add value="-"/> + </valid> + </sanitizer> + </xml> + <xml name="list_integer_valsan"> + <validator type="regex" message="a space separated list of integer values is required">^ *[+-]?[0-9]+( *[+-]?[0-9]+)* *$</validator> + <yield/> + <sanitizer> + <valid initial="string.digits"> + <add value=" "/> + <add value="+"/> + <add value="-"/> + </valid> + </sanitizer> + </xml> + + <!-- helper function to quote space separated strings --> + <token name="@QUOTE_FOO@"> +#def quote(s): + #set $s = [ _ for _ in $s.split(" ") if _ != "" ] + #set $q = False + #for $i, $p in enumerate($s): + #if $p == "": + #continue + #end if + #if $p.startswith('"'): + #set $q = True + #end if +## #if p.startswith('-'): +## #set p = "\\" + p +## #elif p.startswith('"-'): +## #set p = "\\" + p[1:] +## #end if + #if not $q: + #set $s[i] = '"%s"' % p + #end if + #if $p.endswith('"'): + #set $q = False + #end if + #end for + #return " ".join($s) +#end def + </token> + +<token name="@EXT_FOO@"><![CDATA[#def oms2gxyext(o) + #set m={'txt': 'txt', 'tsv': 'tabular', 'bioml': 'xml', 'consensusXML': 'consensusxml', 'csv': 'csv', 'dta': 'dta', 'dta2d': 'dta2d', 'edta': 'edta', 'fa': 'fasta', 'fas': 'fasta', 'fasta': 'fasta', 'FASTA': 'fasta', 'featureXML': 'featurexml', 'featurexml': 'featurexml', 'html': 'html', 'HTML': 'html', 'idXML': 'idxml', 'json': 'json', 'kroenik': 'kroenik', 'mascotXML': 'mascotxml', 'mgf': 'mgf', 'mrm': 'mrm', 'ms': 'sirius.ms', 'ms2': 'ms2', 'msp': 'msp', 'mzData': 'mzdata', 'mzid': 'mzid', 'mzML': 'mzml', 'mzml': 'mzml', 'mzq': 'mzq', 'mzTab': 'mztab', 'mzXML': 'mzxml', 'novor': 'txt', 'obo': 'obo', 'omssaXML': 'idxml', 'osw': 'osw', 'OSW': 'osw', 'params': 'txt', 'paramXML': 'paramxml', 'peplist': 'peplist', 'pep.xml': 'pepxml', 'pepXML': 'pepxml', 'png': 'png', 'PNG': 'png', 'protXML': 'protxml', 'psms': 'psms', 'pqp': 'pqp', 'qcML': 'qcml', 'spec.xml': 'spec.xml', 'splib': 'splib', 'sqMass': 'sqmass', 'tandem.xml': 'tandem', 'trafoXML': 'trafoxml', 'traML': 'traml', 'TraML': 'traml', 'tab': 'tabular', 'raw': 'thermo.raw', 'xls': 'tsv', 'XML': 'xml', 'xml': 'xml', 'xquest.xml': 'xquest.xml', 'xsd': 'xml'} + #return m[o] +#end def +#def gxy2omsext(g) + #set m={'txt': 'txt', 'tabular': 'tsv', 'xml': 'bioml', 'consensusxml': 'consensusXML', 'csv': 'csv', 'dta': 'dta', 'dta2d': 'dta2d', 'edta': 'edta', 'fasta': 'fa', 'featurexml': 'featureXML', 'html': 'html', 'idxml': 'idXML', 'json': 'json', 'kroenik': 'kroenik', 'mascotxml': 'mascotXML', 'mgf': 'mgf', 'mrm': 'mrm', 'sirius.ms': 'ms', 'ms2': 'ms2', 'msp': 'msp', 'mzdata': 'mzData', 'mzid': 'mzid', 'mzml': 'mzML', 'mzq': 'mzq', 'mztab': 'mzTab', 'mzxml': 'mzXML', 'obo': 'obo', 'osw': 'osw', 'paramxml': 'paramXML', 'peff': 'fasta', 'peplist': 'peplist', 'pepxml': 'pep.xml', 'png': 'png', 'protxml': 'protXML', 'psms': 'psms', 'pqp': 'pqp', 'qcml': 'qcML', 'spec.xml': 'spec.xml', 'splib': 'splib', 'sqmass': 'sqMass', 'tandem': 'tandem.xml', 'trafoxml': 'trafoXML', 'traml': 'traML', 'thermo.raw': 'raw', 'tsv': 'xls', 'xquest.xml': 'xquest.xml'} + #return m[g] +#end def +]]></token></macros> |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d macros_autotest.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros_autotest.xml Wed Sep 09 19:57:32 2020 +0000 |
| b |
| b'@@ -0,0 +1,26318 @@\n+<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n+<macros>\n+ <xml name="autotest_AccurateMassSearch">\n+ <test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="ConsensusMapNormalizer_input.consensusXML"/>\n+ <output name="out" file="AccurateMassSearch_1_output.tmp.mzTab" compare="sim_size" delta="5700" ftype="mztab"/>\n+ <param name="positive_adducts" value="CHEMISTRY/PositiveAdducts.tsv" ftype="tabular"/>\n+ <param name="negative_adducts" value="CHEMISTRY/NegativeAdducts.tsv" ftype="tabular"/>\n+ <section name="db">\n+ <param name="mapping" value="CHEMISTRY/HMDBMappingFile.tsv" ftype="tabular"/>\n+ <param name="struct" value="CHEMISTRY/HMDB2StructMapping.tsv" ftype="tabular"/>\n+ </section>\n+ <section name="algorithm">\n+ <param name="mass_error_value" value="5.0"/>\n+ <param name="mass_error_unit" value="ppm"/>\n+ <param name="ionization_mode" value="positive"/>\n+ <param name="isotopic_similarity" value="false"/>\n+ <param name="use_feature_adducts" value="false"/>\n+ <param name="keep_unidentified_masses" value="false"/>\n+ <section name="mzTab">\n+ <param name="exportIsotopeIntensities" value="false"/>\n+ </section>\n+ </section>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test expect_num_outputs="3">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="AccurateMassSearch_2_input.featureXML"/>\n+ <output name="out" file="AccurateMassSearch_2_output.tmp.mzTab" compare="sim_size" delta="5700" ftype="mztab"/>\n+ <output name="out_annotation" file="AccurateMassSearch_2_output.tmp.featureXML" compare="sim_size" delta="5700" ftype="featurexml"/>\n+ <param name="positive_adducts" value="AMS_PositiveAdducts.tsv" ftype="tabular"/>\n+ <param name="negative_adducts" value="AMS_NegativeAdducts.tsv" ftype="tabular"/>\n+ <section name="db">\n+ <param name="mapping" value="AMS_test_Mapping.tsv" ftype="tabular"/>\n+ <param name="struct" value="AMS_test_Struct.tsv" ftype="tabular"/>\n+ </section>\n+ <section name="algorithm">\n+ <param name="mass_error_value" value="5.0"/>\n+ <param name="mass_error_unit" value="ppm"/>\n+ <param name="ionization_mode" value="positive"/>\n+ <param name="isotopic_similarity" value="false"/>\n+ <param name="use_feature_adducts" value="false"/>\n+ <param name="keep_unidentified_masses" value="false"/>\n+ <section name="mzTab">\n+ <param name="exportIsotopeIntensities" value="true"/>\n+ </section>\n+ </section>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_annotation_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test expect_num_outputs="3">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="AccurateMassSearch_2_input.featureXML"/>\n+ <output name="out" file="AccurateMassSearch_3_output.tmp.mzTab" compare="sim_size" delta="5700" ftype="mztab"/>\n+ <output name="out_annotation" file="AccurateMassSearch_2_output.tmp.featureXML" compare="sim_size" delta="5700" ftype="featurexml"/>\n+ <param name'..b' <param name="minscore" value="0.0"/>\n+ <param name="binsize" value="0.1"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_idXML_FLAG,out_mzIdentML_FLAG,out_xquest_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ </xml>\n+ <xml name="autotest_XMLValidator">\n+</xml>\n+ <xml name="autotest_XTandemAdapter">\n+ <test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="spectra.mzML"/>\n+ <output name="out" file="XTandemAdapter_1_out.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+ <param name="database" value="proteins.fasta"/>\n+ <param name="default_config_file" value="CHEMISTRY/XTandem_default_input.xml"/>\n+ <param name="ignore_adapter_param" value="false"/>\n+ <param name="precursor_mass_tolerance" value="5.0"/>\n+ <param name="fragment_mass_tolerance" value="0.3"/>\n+ <param name="precursor_error_units" value="ppm"/>\n+ <param name="fragment_error_units" value="Da"/>\n+ <param name="max_precursor_charge" value="0"/>\n+ <param name="no_isotope_error" value="false"/>\n+ <param name="fixed_modifications" value=""/>\n+ <param name="variable_modifications" value="Oxidation (M)"/>\n+ <param name="minimum_fragment_mz" value="150.0"/>\n+ <param name="enzyme" value="Trypsin"/>\n+ <param name="missed_cleavages" value="1"/>\n+ <param name="semi_cleavage" value="false"/>\n+ <param name="output_results" value="all"/>\n+ <param name="max_valid_expect" value="0.1"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="spectra.mzML"/>\n+ <output name="out" file="XTandemAdapter_2_out.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+ <param name="database" value="proteins.fasta"/>\n+ <param name="default_config_file" value="CHEMISTRY/XTandem_default_input.xml"/>\n+ <param name="ignore_adapter_param" value="false"/>\n+ <param name="precursor_mass_tolerance" value="5.0"/>\n+ <param name="fragment_mass_tolerance" value="0.3"/>\n+ <param name="precursor_error_units" value="ppm"/>\n+ <param name="fragment_error_units" value="Da"/>\n+ <param name="max_precursor_charge" value="0"/>\n+ <param name="no_isotope_error" value="false"/>\n+ <param name="fixed_modifications" value=""/>\n+ <param name="variable_modifications" value="Oxidation (M)"/>\n+ <param name="minimum_fragment_mz" value="150.0"/>\n+ <param name="enzyme" value="Trypsin"/>\n+ <param name="missed_cleavages" value="1"/>\n+ <param name="semi_cleavage" value="false"/>\n+ <param name="output_results" value="valid"/>\n+ <param name="max_valid_expect" value="1e-14"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ </xml>\n+<xml name="autotest_IDDecoyProbability"/><xml name="autotest_PTPredict"/><xml name="autotest_OpenSwathDIAPreScoring"/><xml name="autotest_Digestor"/><xml name="autotest_InclusionExclusionListCreator"/><xml name="autotest_InspectAdapter"/><xml name="autotest_DigestorMotif"/><xml name="autotest_RTPredict"/><xml name="autotest_ProteomicsLFQ"/></macros>\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d macros_discarded_auto.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros_discarded_auto.xml Wed Sep 09 19:57:32 2020 +0000 |
| b |
| b'@@ -0,0 +1,378 @@\n+\n+<xml name="manutest_OpenSwathFileSplitter">\n+<test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="OpenSwathWorkflow_1_input.mzML"/>\n+ <output_collection name="adv_opts_cond.outputDirectory" count=""/>\n+ <output name="out_qc" file="OpenSwathFileSplitter_1.json" compare="sim_size" delta="5700" ftype="json"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_qc_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+</test></xml>\n+<xml name="manutest_IDRipper">\n+<test expect_num_outputs="1">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="IDRipper_1_input.idXML"/>\n+ <output_collection name="out_path" count=""/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test><test expect_num_outputs="1">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="IDRipper_2_input.idXML"/>\n+ <output_collection name="out_path" count=""/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test><test expect_num_outputs="1">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="IDRipper_3_output.idXML"/>\n+ <output_collection name="out_path" count=""/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test></xml>\n+<xml name="manutest_MzMLSplitter">\n+<test expect_num_outputs="1">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="FileFilter_1_input.mzML"/>\n+ <output_collection name="out" count=""/>\n+ <param name="parts" value="2"/>\n+ <param name="size" value="0"/>\n+ <param name="unit" value="MB"/>\n+ <param name="no_chrom" value="false"/>\n+ <param name="no_spec" value="false"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test><test expect_num_outputs="1">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="FileFilter_1_input.mzML"/>\n+ <output_collection name="out" count=""/>\n+ <param name="parts" value="1"/>\n+ <param name="size" value="40"/>\n+ <param name="unit" value="KB"/>\n+ <param name="no_chrom" value="false"/>\n+ <param name="no_spec" value="false"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ '..b'" value="2"/>\n+ <param name="precursor_tolerance" value="20.0"/>\n+ <param name="precursor_tolerance_units" value="ppm"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="MaRaClusterAdapter_1_in_1.mzML,MaRaClusterAdapter_1_in_2.mzML"/>\n+ <param name="id_in" value="MaRaClusterAdapter_1_in_3.idXML"/>\n+ <output name="out" file="MaRaClusterAdapter_2_out_1.tmp.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+ <param name="pcut" value="-10.0"/>\n+ <param name="min_cluster_size" value="1"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test></xml>\n+<xml name="manutest_NovorAdapter">\n+<test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="in" value="NovorAdapter_in.mzML"/>\n+ <output name="out" file="NovorAdapter_1_out.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+ <param name="enzyme" value="Trypsin"/>\n+ <param name="fragmentation" value="CID"/>\n+ <param name="massAnalyzer" value="Trap"/>\n+ <param name="fragment_mass_tolerance" value="0.5"/>\n+ <param name="precursor_mass_tolerance" value="15.0"/>\n+ <param name="precursor_error_units" value="ppm"/>\n+ <param name="variable_modifications" value="Acetyl (K)"/>\n+ <param name="fixed_modifications" value="Carbamidomethyl (C)"/>\n+ <param name="forbiddenResidues" value="I"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test></xml>\n+<xml name="manutest_SpectraSTSearchAdapter">\n+<test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="use_isotopically_averaged_mass" value="false"/>\n+ <param name="use_all_charge_states" value="false"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="spectra_files" value="SpectrastAdapter_1_hack.mzML"/>\n+ <param name="output_files_type" value="pep.xml"/>\n+ <output_collection name="output_files" count="1"/>\n+ <param name="library_file" value="testLib.splib" ftype="splib"/>\n+ <param name="sequence_database_type" value="AA"/>\n+ <param name="precursor_mz_tolerance" value="3.0"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test><test expect_num_outputs="2">\n+ <conditional name="adv_opts_cond">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="use_isotopically_averaged_mass" value="false"/>\n+ <param name="use_all_charge_states" value="false"/>\n+ <param name="force" value="false"/>\n+ <param name="test" value="true"/>\n+ </conditional>\n+ <param name="spectra_files" value="SpectrastAdapter_1_hack.mzML"/>\n+ <param name="output_files_type" value="tsv"/>\n+ <output_collection name="output_files" count="1"/>\n+ <param name="library_file" value="testLib.splib" ftype="splib"/>\n+ <param name="sequence_database_type" value="AA"/>\n+ <param name="precursor_mz_tolerance" value="3.0"/>\n+ <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+ <output name="ctd_out" ftype="xml">\n+ <assert_contents>\n+ <is_valid_xml/>\n+ </assert_contents>\n+ </output>\n+ </test></xml>\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d macros_test.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros_test.xml Wed Sep 09 19:57:32 2020 +0000 |
| b |
| b'@@ -0,0 +1,549 @@\n+<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n+<macros>\n+ \n+<!-- a copy of a FileConverter test without the advanced options used\n+ in order to check if this works (all other tests enable advanced) -->\n+<xml name="manutest_FileConverter">\n+<test expect_num_outputs="1">\n+ <param name="in" value="FileConverter_1_input.mzData"/>\n+ <output name="out" file="FileConverter_1_output.mzML" compare="sim_size" delta="5700" ftype="mzml"/>\n+ <param name="out_type" value="mzML"/>\n+</test>\n+</xml>\n+<!-- tests contributed by the galaxyproteomics community -->\n+<xml name="manutest_ClusterMassTracesByPrecursor">\n+ <test expect_num_outputs="1"><!-- test with arbitarily chosen consensusXML -->\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="in_ms1" ftype="consensusxml" value="ConsensusMapNormalizer_input.consensusXML"/>\n+ <param name="in_swath" ftype="consensusxml" value="ConsensusMapNormalizer_input.consensusXML"/>\n+ <output name="out" ftype="mzml" value="ClusterMassTracesByPrecursor.mzml"/>\n+ </test>\n+</xml>\n+<xml name="manutest_ClusterMassTraces"> \n+ <test expect_num_outputs="1"><!-- test with arbitarily chosen consensusXML -->\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="in" ftype="consensusxml" value="ConsensusMapNormalizer_input.consensusXML"/>\n+ <output name="out" ftype="mzml" value="ClusterMassTraces.mzml"/>\n+ </test>\n+</xml>\n+<xml name="manutest_CVInspector">\n+ <!-- test with https://raw.githubusercontent.com/HUPO-PSI/mzIdentML/master/cv/XLMOD.obo listed here https://www.psidev.info/groups/controlled-vocabularies, mapping file from share/OpenMS/MAPPING/ms-mapping.xml, see currently not working, see: https://github.com/OpenMS/OpenMS/pull/4425 -->\n+ <test expect_num_outputs="1">\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="cv_files" ftype="obo" value="CHEMISTRY/XLMOD.obo"/>\n+ <param name="cv_names" value="XLMOD"/>\n+ <param name="mapping_file" value="MAPPING/ms-mapping.xml"/>\n+ <param name="OPTIONAL_OUTPUTS" value="html_FLAG"/>\n+ <output name="html" ftype="html" value="CVInspector.html"/>\n+ </test>\n+</xml>\n+<xml name="manutest_DeMeanderize">\n+ <test expect_num_outputs="1"><!-- test with the output of a MSsimulator output generated by a test below -->\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="in" ftype="mzml" value="MSsimulator_MALDI.mzml"/>\n+\t<output name="out" ftype="mzml" value="DeMeanderize.mzml"/>\n+ </test>\n+</xml>\n+<xml name="manutest_Digestor">\n+ <test expect_num_outputs="1"><!-- just using some random fasta, in contrast to DigestorMotif idXML/fasta output possible, testing for the non-default fasta option -->\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="in" ftype="fasta" value="random.fa"/>\n+ <output name="out" ftype="fasta" value="Digestor.fasta"/>\n+ <param name="out_type" value="fasta"/>\n+ </test>\n+</xml>\n+<xml name="manutest_EICExtractor">\n+ <test expect_num_outputs="1"><!-- just using some random test data -->\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="in" value="spectra.mzML"/>\n+ <param name="pos" ftype="edta" value="FileConverter_10_input.edta"/>\n+ <output name="out" ftype="csv" value="EICExtractor.csv" lines_diff="2"/>\n+ </test>\n+</xml>\n+<xml name="manutest_ERPairFinder">\n+ <!-- TODO -->\n+</xml>\n+<xml name="manutest_FeatureFinderIsotopeWavelet">\n+ <test expect_num_outputs="1"><!--just use the input of another FeatureFinder -->\n+ <param name="adv_opts_selector" value="advanced"/>\n+ <param name="adv_opts_cond|test" value="true"/>\n+ <param name="in" value="FeatureFin'..b'e="manutest_PTPredict"/><xml name="manutest_XTandemAdapter"/><xml name="manutest_FeatureFinderMetabo"/><xml name="manutest_CruxAdapter"/><xml name="manutest_OpenSwathConfidenceScoring"/><xml name="manutest_PrecursorIonSelector"/><xml name="manutest_ConsensusMapNormalizer"/><xml name="manutest_RTPredict"/><xml name="manutest_PercolatorAdapter"/><xml name="manutest_ProteinInference"/><xml name="manutest_OpenSwathDecoyGenerator"/><xml name="manutest_TextExporter"/><xml name="manutest_FeatureFinderCentroided"/><xml name="manutest_DecoyDatabase"/><xml name="manutest_SpectraFilterWindowMower"/><xml name="manutest_NoiseFilterGaussian"/><xml name="manutest_MaRaClusterAdapter"/><xml name="manutest_ConsensusID"/><xml name="manutest_FileFilter"/><xml name="manutest_InclusionExclusionListCreator"/><xml name="manutest_FeatureLinkerUnlabeledKD"/><xml name="manutest_NovorAdapter"/><xml name="manutest_OpenPepXL"/><xml name="manutest_SeedListGenerator"/><xml name="manutest_FalseDiscoveryRate"/><xml name="manutest_OpenPepXLLF"/><xml name="manutest_SpecLibSearcher"/><xml name="manutest_IDConflictResolver"/><xml name="manutest_MRMMapper"/><xml name="manutest_OMSSAAdapter"/><xml name="manutest_ExternalCalibration"/><xml name="manutest_PeakPickerHiRes"/><xml name="manutest_MascotAdapter"/><xml name="manutest_FeatureFinderMultiplex"/><xml name="manutest_MetaProSIP"/><xml name="manutest_FuzzyDiff"/><xml name="manutest_TargetedFileConverter"/><xml name="manutest_MapAlignerIdentification"/><xml name="manutest_IDRTCalibration"/><xml name="manutest_MRMTransitionGroupPicker"/><xml name="manutest_RTModel"/><xml name="manutest_OpenSwathAssayGenerator"/><xml name="manutest_IDSplitter"/><xml name="manutest_OpenSwathMzMLFileCacher"/><xml name="manutest_HighResPrecursorMassCorrector"/><xml name="manutest_OpenSwathChromatogramExtractor"/><xml name="manutest_OpenSwathAnalyzer"/><xml name="manutest_FeatureLinkerUnlabeled"/><xml name="manutest_CompNovoCID"/><xml name="manutest_DTAExtractor"/><xml name="manutest_FileInfo"/><xml name="manutest_MapAlignerSpectrum"/><xml name="manutest_IsobaricAnalyzer"/><xml name="manutest_LowMemPeakPickerHiRes"/><xml name="manutest_MyriMatchAdapter"/><xml name="manutest_InternalCalibration"/><xml name="manutest_TOFCalibration"/><xml name="manutest_MSGFPlusAdapter"/><xml name="manutest_PTModel"/><xml name="manutest_InspectAdapter"/><xml name="manutest_SpectraFilterSqrtMower"/><xml name="manutest_SimpleSearchEngine"/><xml name="manutest_PeakPickerWavelet"/><xml name="manutest_MapNormalizer"/><xml name="manutest_IDScoreSwitcher"/><xml name="manutest_FeatureLinkerLabeled"/><xml name="manutest_MapRTTransformer"/><xml name="manutest_RNPxlSearch"/><xml name="manutest_PrecursorMassCorrector"/><xml name="manutest_MapAlignerPoseClustering"/><xml name="manutest_MzTabExporter"/><xml name="manutest_BaselineFilter"/><xml name="manutest_FeatureFinderMRM"/><xml name="manutest_MascotAdapterOnline"/><xml name="manutest_DatabaseFilter"/><xml name="manutest_ProteinResolver"/><xml name="manutest_OpenSwathFeatureXMLToTSV"/><xml name="manutest_FidoAdapter"/><xml name="manutest_AccurateMassSearch"/><xml name="manutest_CompNovo"/><xml name="manutest_LowMemPeakPickerHiResRandomAccess"/><xml name="manutest_OpenSwathRTNormalizer"/><xml name="manutest_PeptideIndexer"/><xml name="manutest_CometAdapter"/><xml name="manutest_NoiseFilterSGolay"/><xml name="manutest_MSFraggerAdapter"/><xml name="manutest_SpectraSTSearchAdapter"/><xml name="manutest_SequestAdapter"/><xml name="manutest_FeatureFinder"/><xml name="manutest_LuciphorAdapter"/><xml name="manutest_GNPSExport"/><xml name="manutest_Epifany"/><xml name="manutest_NucleicAcidSearchEngine"/><xml name="manutest_QualityControl"/><xml name="manutest_FeatureFinderMetaboIdent"/><xml name="manutest_RNAMassCalculator"/><xml name="manutest_MapAlignerTreeGuided"/><xml name="manutest_IDMassAccuracy"/><xml name="manutest_ProteomicsLFQ"/><xml name="manutest_IDDecoyProbability"/><xml name="manutest_DigestorMotif"/></macros>\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d prepare_test_data.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/prepare_test_data.sh Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| b'@@ -0,0 +1,1196 @@\n+export COMET_BINARY="comet"\n+export CRUX_BINARY="crux"\n+export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"\n+export FIDO_BINARY="Fido"\n+export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"\n+export MARACLUSTER_BINARY="/tmp/openms-stuff//OpenMS2.5.0-git/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"\n+export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"\n+export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"\n+export MYRIMATCH_BINARY="myrimatch"\n+export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"\n+export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"\n+export PERCOLATOR_BINARY="percolator"\n+export SIRIUS_BINARY="$(which sirius)"\n+export SPECTRAST_BINARY="spectrast"\n+export XTANDEM_BINARY="xtandem"\n+export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"\n+FuzzyDiff -test -ini FuzzyDiff.ini -in1 FuzzyDiff_3_in1.featureXML -in2 FuzzyDiff_3_in2.featureXML > UTILS_FuzzyDiff_3.stdout 2> UTILS_FuzzyDiff_3.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'UTILS_FuzzyDiff_3 failed\'; >&2 echo -e "stderr:\\n$(cat UTILS_FuzzyDiff_3.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat UTILS_FuzzyDiff_3.stdout)";fi\n+IDMerger -test -in IDMerger_1_input1.idXML IDMerger_1_input2.idXML -out IDMerger_1_output.tmp -annotate_file_origin > TOPP_IDMerger_1.stdout 2> TOPP_IDMerger_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_1.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_1.stdout)";fi\n+IDMerger -test -pepxml_protxml -in IDMerger_2_input1.idXML IDMerger_2_input2.idXML -out IDMerger_2_output.tmp > TOPP_IDMerger_2.stdout 2> TOPP_IDMerger_2.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_2 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_2.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_2.stdout)";fi\n+IDMerger -test -in IDMerger_1_input1.idXML IDMerger_1_input1.idXML -out IDMerger_3_output.tmp > TOPP_IDMerger_3.stdout 2> TOPP_IDMerger_3.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_3 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_3.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_3.stdout)";fi\n+IDMerger -test -in degenerated_empty.idXML degenerated_empty.idXML -out IDMerger_4_output.tmp > TOPP_IDMerger_4.stdout 2> TOPP_IDMerger_4.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_4 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_4.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_4.stdout)";fi\n+IDMerger -test -in IDMerger_2_input1.idXML -add_to IDMerger_5_input1.idXML -out IDMerger_5_output.tmp > TOPP_IDMerger_5.stdout 2> TOPP_IDMerger_5.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_5 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_5.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_5.stdout)";fi\n+BaselineFilter -test -in BaselineFilter_input.mzML -out BaselineFilter.tmp -struc_elem_length 1.5 > TOPP_BaselineFilter_1.stdout 2> TOPP_BaselineFilter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_BaselineFilter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_BaselineFilter_1.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_BaselineFilter_1.stdout)";fi\n+ConsensusMapNormalizer -test -in ConsensusMapNormalizer_input.consensusXML -out ConsensusMapNormalizer.tmp > TOPP_ConsensusMapNormalizer_1.stdout 2> TOPP_ConsensusMapNormalizer_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_ConsensusMapNormalizer_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_ConsensusMapNormalizer_1.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_ConsensusMapNormalizer_1.stdout)";fi\n+MapNormalizer -test -in MapNormalizer_input.mzML -out MapNormalizer.tmp > TOPP_MapNormalizer_1.stdout 2> TOPP_MapNormalizer_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_MapNormalizer_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_MapNormalizer_1.stderr | sed \'s/^/'..b'S_AssayGeneratorMetabo_9 failed\'; >&2 echo -e "stderr:\\n$(cat UTILS_AssayGeneratorMetabo_9.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat UTILS_AssayGeneratorMetabo_9.stdout)";fi\n+AssayGeneratorMetabo -test -executable "${SIRIUS_BINARY}" -in AssayGeneratorMetabo_input.mzML -in_id AssayGeneratorMetabo_ams_input.featureXML -out AssayGeneratorMetabo_ams_sirius_restrict_output.tmp.tsv -fragment_annotation sirius -use_exact_mass -transition_threshold 3.0 -min_transitions 2 -max_transitions 3 -min_fragment_mz 100 -max_fragment_mz 900 -preprocessing:filter_by_num_masstraces 1 -preprocessing:precursor_mz_tolerance 10 -preprocessing:precursor_mz_tolerance_unit ppm -preprocessing:feature_only -sirius:profile qtof -sirius:compound_timeout 100 > UTILS_AssayGeneratorMetabo_10.stdout 2> UTILS_AssayGeneratorMetabo_10.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'UTILS_AssayGeneratorMetabo_10 failed\'; >&2 echo -e "stderr:\\n$(cat UTILS_AssayGeneratorMetabo_10.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat UTILS_AssayGeneratorMetabo_10.stdout)";fi\n+SiriusAdapter -test -executable "${SIRIUS_BINARY}" -in SiriusAdapter_2_input.mzML -in_featureinfo SiriusAdapter_2_input.featureXML -out_sirius SiriusAdapter_4_output.tmp -out_fingerid SiriusAdapter_4_foutput.tmp -sirius:auto_charge -sirius:profile qtof -sirius:database all > TOPP_SiriusAdapter_4.stdout 2> TOPP_SiriusAdapter_4.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SiriusAdapter_4 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SiriusAdapter_4.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_SiriusAdapter_4.stdout)";fi\n+NovorAdapter -test -java_memory 512 -executable "${NOVOR_BINARY}" -in NovorAdapter_in.mzML -out NovorAdapter_1_out.tmp -variable_modifications "Acetyl (K)" -fixed_modifications "Carbamidomethyl (C)" -forbiddenResidues "I" > TOPP_NovorAdapter_1.stdout 2> TOPP_NovorAdapter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_NovorAdapter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_NovorAdapter_1.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_NovorAdapter_1.stdout)";fi\n+FileConverter -test -force_TPP_compatibility -in spectra_spectrast.mzXML -out SpectrastAdapter_1_hack.mzML > TOPP_SpectrastSearchAdapter_0_prepare.stdout 2> TOPP_SpectrastSearchAdapter_0_prepare.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SpectrastSearchAdapter_0_prepare failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SpectrastSearchAdapter_0_prepare.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_SpectrastSearchAdapter_0_prepare.stdout)";fi\n+SpectraSTSearchAdapter -test -library_file testLib.splib -spectra_files SpectrastAdapter_1_hack.mzML -output_files SpectrastAdapter_1_out1.tmp.pep.xml -executable "${SPECTRAST_BINARY}" > TOPP_SpectrastSearchAdapter_1.stdout 2> TOPP_SpectrastSearchAdapter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SpectrastSearchAdapter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SpectrastSearchAdapter_1.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_SpectrastSearchAdapter_1.stdout)";fi\n+SpectraSTSearchAdapter -test -library_file testLib.splib -spectra_files SpectrastAdapter_1_hack.mzML -output_files SpectrastAdapter_1_out1.tmp.pep.tsv -executable "${SPECTRAST_BINARY}" > TOPP_SpectrastSearchAdapter_2.stdout 2> TOPP_SpectrastSearchAdapter_2.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SpectrastSearchAdapter_2 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SpectrastSearchAdapter_2.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_SpectrastSearchAdapter_2.stdout)";fi\n+LuciphorAdapter -test -in LuciphorAdapter_1_input.mzML -java_memory 1024 -id LuciphorAdapter_1_input.idXML -out LuciphorAdapter_1_output.tmp -executable "${LUCIPHOR_BINARY}" -min_num_psms_model 1 > TOPP_LuciphorAdapter_1.stdout 2> TOPP_LuciphorAdapter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_LuciphorAdapter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_LuciphorAdapter_1.stderr | sed \'s/^/ /\')"; echo -e "stdout:\\n$(cat TOPP_LuciphorAdapter_1.stdout)";fi\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d prepare_test_data_manual.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/prepare_test_data_manual.sh Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| b'@@ -0,0 +1,155 @@\n+MSSimulator -test -in DecoyDatabase_1.fasta -out MSsimulator.mzml -algorithm:RandomNumberGenerators:biological reproducible -algorithm:RandomNumberGenerators:technical reproducible > MSSimulator_1.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MSSimulator_1 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+MSSimulator -test -in DecoyDatabase_1.fasta -out MSsimulator_MALDI.mzml -algorithm:RandomNumberGenerators:biological reproducible -algorithm:RandomNumberGenerators:technical reproducible -algorithm:MSSim:Global:ionization_type MALDI > MSSimulator_2.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MSSimulator_2 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+ClusterMassTracesByPrecursor -test -in_ms1 ConsensusMapNormalizer_input.consensusXML -in_swath ConsensusMapNormalizer_input.consensusXML -out ClusterMassTracesByPrecursor.mzml > ClusterMassTracesByPrecursor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'ClusterMassTracesByPrecursor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+ClusterMassTraces -test -in ConsensusMapNormalizer_input.consensusXML -out ClusterMassTraces.mzml > ClusterMassTraces.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'ClusterMassTraces failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+CVInspector -test -cv_files CHEMISTRY/XLMOD.obo -cv_names XLMOD -mapping_file MAPPING/ms-mapping.xml -html CVInspector.html > CVInspector.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'CVInspector failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+DeMeanderize -test -in MSsimulator_MALDI.mzml -out DeMeanderize.mzml > DeMeanderize.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'DeMeanderize failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO DigestorMotif\n+\n+Digestor -test -in random.fa -out Digestor.fasta -out_type fasta > Digestor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'Digestor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+EICExtractor -test -in spectra.mzML -pos FileConverter_10_input.edta -out EICExtractor.csv > EICExtractor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'EICExtractor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+#TODO ERPairFinder\n+\n+FeatureFinderIsotopeWavelet -test -in FeatureFinderCentroided_1_input.mzML -out FeatureFinderIsotopeWavelet.featureXML > FeatureFinderIsotopeWavelet.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'FeatureFinderIsotopeWavelet failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+\n+FFEval -test -in FeatureFinderCentroided_1_output.featureXML -truth FeatureFinderCentroided_1_output.featureXML -out FFEval.featureXML -out_roc FFEval_roc.csv > FFEval.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'FFEval failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO? deprecated IDDecoyProbability\n+\n+IDExtractor -test -in MSGFPlusAdapter_1_out.idXML -best_hits -number_of_peptides 1 -out IDExtractor.idXML > IDExtractor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'IDExtractor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+LabeledEval -test -in FeatureLinkerLabeled_1_input.featureXML -truth FeatureLinkerLabeled_1_output.consensusXML> LabeledEval.txt > LabeledEval.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'LabeledEval failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+MapStatistics -test -in SiriusAdapter_3_input.featureXML -out MapStatistics.txt > MapStatistics_1.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MapStatistics_1 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+MapStatistics -test -in ConsensusXMLFile_1.consensusXML -out MapStatistics2.txt > MapStatistics_2.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MapStatistics_2 failed\'; '..b'\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO out should be tsv, but needs https://github.com/OpenMS/OpenMS/pull/4533\n+RTEvaluation -in PeptideIndexer_1.idXML -out RTEvaluation.csv > RTEvaluation.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'RTEvaluation failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+ln -fs RTEvaluation.csv RTEvaluation.tsv > .stdout 2> stderr\n+\n+SemanticValidator -test -in FileFilter_1_input.mzML -mapping_file MAPPING/ms-mapping.xml > SemanticValidator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SemanticValidator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+IDFilter -in PeptideIndexer_1.idXML -best:strict -out SequenceCoverageCalculator_1.idXML > IDFilter.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'IDFilter failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+SequenceCoverageCalculator -test -in_database PeptideIndexer_1.fasta -in_peptides SequenceCoverageCalculator_1.idXML -out SequenceCoverageCalculator.txt > SequenceCoverageCalculator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SequenceCoverageCalculator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO SpecLibCreator\n+\n+SpectraFilterBernNorm -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterBernNorm.mzML > SpectraFilterBernNorm.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterBernNorm failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterMarkerMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterMarkerMower.mzML > SpectraFilterMarkerMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterMarkerMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterNLargest -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterNLargest.mzML > SpectraFilterNLargest.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterNLargest failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterNormalizer -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterNormalizer.mzML > SpectraFilterNormalizer.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterNormalizer failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterParentPeakMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterParentPeakMower.mzML > SpectraFilterParentPeakMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterParentPeakMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterScaler -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterScaler.mzML > SpectraFilterScaler.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterScaler failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterThresholdMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterThresholdMower.mzML > SpectraFilterThresholdMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterThresholdMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraMerger -test -in NovorAdapter_in.mzML -out SpectraMerger_1.mzML > SpectraMerger.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraMerger failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO SvmTheoreticalSpectrumGeneratorTrainer\n+\n+TransformationEvaluation -test -in FileInfo_16_input.trafoXML -out TransformationEvaluation.trafoXML > TransformationEvaluation.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TransformationEvaluation failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+XMLValidator -test -in FileFilter_1_input.mzML > XMLValidator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'XMLValidator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d readme.md --- a/readme.md Fri May 17 10:07:55 2019 -0400 +++ b/readme.md Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| b'@@ -8,175 +8,127 @@\n More informations are available at:\n \n * https://github.com/OpenMS/OpenMS\n- * http://open-ms.sourceforge.net\n+ * https://www.openms.de/\n+\n+The wrappers for these tools and most of their tests are automatically\n+generated using the `generate.sh` script. The generation of the tools is\n+based on the CTDConverter (https://github.com/WorkflowConversion/CTDConverter)\n+which can be fine tuned via the `hardcoded_params.json` file. This file allows\n+to blacklist and hardcode parameters and to modify or set arbitrary\n+CTD/XML attributes.\n+\n+Note that, due to its size, the test data is excluded from this repository. In\n+order to generate the test data on call `test-data.sh`.\n+\n+Manual updates should only be done to\n+\n+- the `@GALAXY_VERSION@"` token in `macros.xml`\n+- and the manually contributed tests in `macros_test.xml` (The goal is that all\n+ tools that do not have an automatically generated test are covered here)\n+- the `hardcoded_params.json` files\n+\n+In a few cases patches may be acceptable.\n+\n+Installation\n+============\n+\n+The Galaxy OpenMS tools can be installed from the toolshed. While most tools\n+will work out of the box some need attention since requirements can not be\n+fulfilled via Conda:\n+\n+Not yet in Conda are:\n+\n+- SpectraST (http://tools.proteomecenter.org/wiki/index.php?title=SpectraST)\n+- MaRaCluster (https://github.com/statisticalbiotechnology/maracluster)\n+\n+Binaries for these tools can easily be obtained via: \n+\n+```\n+VERSION=....\n+git git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git OpenMS$VERSION.0-git\n+git submodule init OpenMS$VERSION.0-git\n+git submodule update OpenMS$VERSION.0-git\n+```\n+\n+They are located in `OpenMS$VERSION-git/THIRDPARTY/`. \n \n+Not in Conda due to licencing restrictions:\n+\n+- Mascot http://www.matrixscience.com/\n+- MSFragger https://github.com/Nesvilab/MSFragger\n+- Novor http://www.rapidnovor.org/novor\n+\n+There are multiple ways to enable the Galaxy tools to use these binaries. \n+\n+- Just copy them to the `bin` path within Galaxy\'s conda environment\n+- Put them in any other path that that is included in PATH\n+- Edit the corresponding tools: In the command line part search for the parameters `-executable`, `-maracluster_executable`, or `-mascot_directory` and edit them appropriately.\n+\n+Working\n+=======\n+\n+The tools work by:\n+\n+Preprocessing:\n+\n+- For each input / output data set parameter a directory is crated (named by\n+ the parameter)\n+- For input data set parameters the links to the actual location of the data\n+ sets are created\n+\n+Main:\n+\n+- The galaxy wrapper create two json config files: one containing the\n+ parameters and the values chosen by the user and the other the values of\n+ hardcoded parameters.\n+- With `OpenMSTool -write_ctd ./` a CTD (names OpenMSTool.ctd) file is\n+ generated that contains the default values.\n+- A call to `fill_ctd.py` fills in the values from the json config files into\n+ the CTD file\n+- The actual tool is called `OpenMSTool -ini OpenMSTool.ctd` and also all input\n+ and output parameters are given on the command line.\n+\n+Postprocessing:\n+\n+- output data sets are moved to the final locations\n+\n+Note: The reason for handling data sets on the command line (and not specifying\n+them in the CTD file) is mainly that all files in Galaxy have the extension\n+`.dat` and OpenMS tools require an appropriate extension. But this may change\n+in the future.\n \n Generating OpenMS wrappers\n ==========================\n \n- * install OpenMS (you can do this automatically through Conda)\n- * create a folder called CTD\n- * if you installed openms as a binary in a specific directory, execute the following command in the `openms/bin` directory:\n- \n- ```bash\n- for binary in `ls`; do ./$binary -write_ctd /PATH/TO/YOUR/CTD; done;\n- ```\n- \n- * if there is no binary release (e.g. as with version 2.2), download and unpack the Conda package, find the `bin` folder and create a list of the tools as follow:\n- \n- ```bash\n- ls '..b'yrimatch\' wrappers/MyriMatchAdapter.xml\n- sed -i \'13 a\\-omssa_executable omssa\' wrappers/OMSSAAdapter.xml\n- sed -i \'13 a\\-xtandem_executable xtandem\' wrappers/XTandemAdapter.xml\n- ```\n- \n- * For some tools, additional work has to be done. In `MSGFPlusAdapter.xml` the following is needed in the command section at the beginning (check your file to know what to copy where):\n- \n- ```\n- <command><![CDATA[\n-\n- ## check input file type\n- #set $in_type = $param_in.ext\n+2. The CI testing framework of OpenMS contains command lines and test data \n+ (https://github.com/OpenMS/OpenMS/tree/develop/src/tests/topp). These tests\n+ are described in two CMake files.\n \n- ## create the symlinks to set the proper file extension, since msgf uses them to choose how to handle the input files\n- ln -s \'$param_in\' \'param_in.${in_type}\' &&\n- ln -s \'$param_database\' param_database.fasta &&\n- ## find location of the MSGFPlus.jar file of the msgf_plus conda package\n- MSGF_JAR=\\$(msgf_plus -get_jar_path) &&\n+ - From these CMake files Galaxy tests are auto generated and stored in `macros_autotest.xml`\n+ - The command lines are stored in `prepare_test_data.sh` for regeneration of test data\n+\n+More details can be found in the comments of the shell script.\n+\n+Open problems\n+=============\n \n- MSGFPlusAdapter\n- -executable \\$MSGF_JAR\n- #if $param_in:\n- -in \'param_in.${in_type}\'\n- #end if\n- #if $param_out:\n- -out $param_out\n- #end if\n- #if $param_mzid_out:\n- -mzid_out $param_mzid_out\n- #end if\n- #if $param_database:\n- -database param_database.fasta\n- #end if\n- \n- [...]\n- ]]>\n- ```\n- \n- * In Xtandem Converter and probably in others:\n- \n- ```\n- #if str($param_missed_cleavages) != \'\':\n- ```\n- This is because integers needs to be compared as string otherwise `0` becomes `false`.\n- \n- * In `MetaProSIP.xml` add `R` as a requirement:\n- \n- ```\n- <expand macro="requirements">\n- <requirement type="package" version="3.3.1">r-base</requirement>\n- </expand>\n- ```\n- \n- * In `IDFileConverter.xml` the following is needed in the command section at the beginning (check your file to know what to copy where):\n- \n- ```\n- <command><![CDATA[\n- \n- ## check input file type\n- #set $in_type = $param_in.ext\n+Some tools stall in CI testing using `--biocontainers` which is why the OpenMS\n+tools are currently listed in `.tt_biocontainer_skip`. This is\n \n- ## create the symlinks to set the proper file extension, since IDFileConverter uses them to choose how to handle the input files\n- ln -s \'$param_in\' \'param_in.${in_type}\' &&\n-\n- IDFileConverter\n-\n- #if $param_in:\n- -in \'param_in.${in_type}\'\n- #end if\n-\n- [...]\n- ]]>\n- ```\n+- AssayGeneratorMetabo and SiriusAdapter (both depend on sirius)\n+- OMSSAAdapter\n \n- * In `IDFileConverter.xml` and `FileConverter.xml` add `auto_format="true"` to the output, e.g.:\n- \n- - `<data name="param_out" auto_format="true"/>`\n- - `<data name="param_out" metadata_source="param_in" auto_format="true"/>`\n- \n- * To add an example test case to `DecoyDatabase.xml` add the following after the output section. If standard settings change you might have to adjust the options and/or the test files.\n- \n- ```\n- <tests>\n- <test>\n- <param name="param_in" value="DecoyDatabase_input.fasta"/>\n- <output name="param_out" file="DecoyDatabase_output.fasta"/>\n- </test>\n- </tests>\n- ```\n- \n- * Additionally cause of lacking dependencies, the following adapters have been removed in `SKIP_TOOLS_FILES.txt` as well:\n- * OMSSAAdapter\n- * MyrimatchAdapter\n- \n- * Additionally cause of a problematic parameter (-model_directory), the following adapter has been removed:\n- * PepNovoAdapter\n-\n+Using `docker -t` seems to solve the problem (see\n+https://github.com/galaxyproject/galaxy/issues/10153).\n \n Licence (MIT)\n =============\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d test-data.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data.sh Wed Sep 09 19:57:32 2020 +0000 |
| [ |
| b'@@ -0,0 +1,255 @@\n+#!/usr/bin/env bash\n+\n+VERSION=2.5\n+FILETYPES="filetypes.txt"\n+CONDAPKG="https://anaconda.org/bioconda/openms/2.5.0/download/linux-64/openms-2.5.0-h463af6b_1.tar.bz2"\n+\n+# import the magic\n+. ./generate-foo.sh\n+\n+# install conda\n+if [ -z "$tmp" ]; then\n+\ttmp=$(mktemp -d)\n+\tcreated="yes"\n+fi\n+\n+export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"\n+export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"\n+export OPENMSENV="$tmp/OpenMS$VERSION-env"\n+export CTDCONVERTER="$tmp/CTDConverter"\n+\n+\n+if type conda > /dev/null; then \n+\ttrue\n+else\n+\twget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh\n+\tbash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"\n+\tsource "$tmp/miniconda/bin/activate"\n+fi\n+eval "$(conda shell.bash hook)"\n+\n+if [[ -z "$1" ]]; then\n+\tautotests="/dev/null"\n+else\n+\tautotests="$1"\n+fi\n+\n+###############################################################################\n+## get \n+## - conda environment (for executing the binaries) and \n+## - the git clone of OpenMS (for generating the tests)\n+###############################################################################\n+\n+echo "Clone OpenMS $VERSION sources"\n+if [[ ! -d $OPENMSGIT ]]; then\n+\tgit clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT\n+\tcd $OPENMSGIT\n+\tgit submodule init\n+\tgit submodule update\n+\tcd -\n+else\n+\tcd $OPENMSGIT\n+\t\tgit pull origin release/$VERSION.0\n+\tcd -\n+fi\n+\n+echo "Create OpenMS $VERSION conda env"\n+# TODO currently add lxml (needed by CTDConverter)\n+# TODO for some reason a to recent openjdk is used\n+if conda env list | grep "$OPENMSENV"; then\n+\ttrue\n+else\n+\tconda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION openjdk=8.0.192 ctdopts=1.4 lxml\n+# chmod -R u-w $OPENMSENV \n+fi\n+###############################################################################\n+## get the \n+## - conda package (for easy access and listing of the OpenMS binaries), \n+###############################################################################\n+echo "Download OpenMS $VERSION package $CONDAPKG"\n+\n+if [[ ! -d $OPENMSPKG ]]; then\n+\tmkdir $OPENMSPKG\n+\twget -q -P $OPENMSPKG/ "$CONDAPKG"\n+\ttar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/\n+\trm $OPENMSPKG/"$(basename $CONDAPKG)"\n+fi\n+\n+###############################################################################\n+## Get python libaries for CTD -> Galaxy conversion\n+## TODO fix to main repo OR conda packkage if PRs are merged \n+###############################################################################\n+echo "Clone CTDConverter"\n+if [[ ! -d $CTDCONVERTER ]]; then\n+\t#git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter\n+\tgit clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER\n+else\n+\tcd $CTDCONVERTER\n+\tgit pull origin topic/cdata\n+\tcd -\n+fi\n+export PYTHONPATH=$(pwd)/CTDopts\n+\n+###############################################################################\n+## copy all the test data files to test-data\n+## most of it (outputs) will be overwritten later, but its needed for\n+## prepare_test_data\n+###############################################################################\n+echo "Get test data"\n+cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/\n+cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/\n+cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/\n+cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/\n+if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then \n+\twget -q https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML && mv MetaboliteSpectralDB.mzML test-data/\n+fi\n+ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt\n+ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv\n+\n+if [ ! -d test-data/pepno'..b'e results for the tests and run it\n+###############################################################################\n+echo "Create test shell script"\n+\n+echo -n "" > prepare_test_data.sh\n+echo \'export COMET_BINARY="comet"\' >> prepare_test_data.sh\n+echo \'export CRUX_BINARY="crux"\' >> prepare_test_data.sh\n+echo \'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"\' >> prepare_test_data.sh\n+echo \'export FIDO_BINARY="Fido"\' >> prepare_test_data.sh\n+echo \'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"\' >> prepare_test_data.sh\n+\n+echo \'export MARACLUSTER_BINARY="\'"$OPENMSGIT"\'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"\'>> prepare_test_data.sh\n+echo \'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"\'>> prepare_test_data.sh\n+echo \'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"\' >> prepare_test_data.sh\n+echo \'export MYRIMATCH_BINARY="myrimatch"\'>> prepare_test_data.sh\n+echo \'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"\' >> prepare_test_data.sh\n+echo \'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"\'>> prepare_test_data.sh\n+echo \'export PERCOLATOR_BINARY="percolator"\'>> prepare_test_data.sh\n+echo \'export SIRIUS_BINARY="$(which sirius)"\' >> prepare_test_data.sh\n+echo \'export SPECTRAST_BINARY="spectrast"\' >> prepare_test_data.sh\n+echo \'export XTANDEM_BINARY="xtandem"\' >> prepare_test_data.sh\n+echo \'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"\' >> prepare_test_data.sh\n+\n+prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh\n+\n+# prepare_test_data > tmp_test_data.sh\n+# # remove calls not needed for the tools listed in any .list file\n+# echo LIST $LIST\n+# if [ ! -z "$LIST" ]; then\n+# \tREX=$(echo $LIST | sed \'s/ /\\n/g\' | sed \'s@.*/\\([^/]\\+\\).xml$@\\1@\' | tr \'\\n\' \'|\' | sed \'s/|$//\')\n+# else\n+# \tREX=".*"\n+# fi\n+# echo REX $REX\n+# cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh\n+# rm tmp_test_data.sh\n+\n+echo "Execute test shell script"\n+chmod u+x prepare_test_data.sh\n+cd ./test-data || exit\n+../prepare_test_data.sh\n+cd - || exit\n+\n+\n+###############################################################################\n+## create/update test data for the manually generated tests\n+## - run convert once with the manual tests only and \n+## - update test-data (needs to run 2x)\n+###############################################################################\n+echo "Execute test shell script for manually curated tests"\n+chmod u+x prepare_test_data_manual.sh\n+\n+cd ./test-data || exit\n+../prepare_test_data_manual.sh\n+cd - || exit\n+\n+\n+# # # exit\n+\n+###############################################################################\n+## auto generate tests\n+###############################################################################\n+\n+echo "Write test macros to "$autotests\n+echo "<macros>" > $autotests\n+for i in $(ls *xml |grep -v macros)\n+do\n+\tb=$(basename "$i" .xml)\n+\tget_tests2 "$b" >> $autotests \n+done\n+echo "</macros>" >> $autotests\n+\n+echo "Create test data links"\n+link_tmp_files\n+\n+# tests for tools using output_prefix parameters can not be auto generated\n+# hence we output the tests for manual curation in macros_test.xml\n+# and remove them from the autotests\n+if [[ ! -z "$1" ]]; then\n+\techo "" > macros_discarded_auto.xml\n+\tfor i in OpenSwathFileSplitter IDRipper MzMLSplitter\n+\tdo\n+\t\techo "<xml name=\\"manutest_$i\\">" >> macros_discarded_auto.xml\n+\t\txmlstarlet sel -t -c "/macros/xml[@name=\'autotest_$i\']/test" macros_autotest.xml >> macros_discarded_auto.xml\n+\t\techo "</xml>" >> macros_discarded_auto.xml\n+\t\txmlstarlet ed -d "/macros/xml[@name=\'autotest_$i\']/test" macros_autotest.xml > tmp\n+\t\tmv tmp macros_autotest.xml\n+\tdone\n+\t>&2 "discarded autogenerated macros for curation in macros_discarded_auto.xml"\n+fi\n+conda deactivate\n+\n+## remove broken symlinks in test-data\n+find test-data/ -xtype l -delete\n+\n+# if [ ! -z "$created" ]; then\n+# \techo "Removing temporary directory"\n+# \trm -rf "$tmp"\n+# fi\n' |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d tool-data/pepnovo_models.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/pepnovo_models.loc.sample Wed Sep 09 19:57:32 2020 +0000 |
| b |
| @@ -0,0 +1,23 @@ +# This is a sample file distributed with Galaxy that enables tools +# to use pepnovo models +# The file has three tab separated columns: name, value, and path. +# The idea is that there are a number of models in a directory: +# - each model directory has a unique name (columns 2 and 0) +# - each model can contain a set of models (column 1) +# +# The following example works fo the default models from +# http://proteomics.ucsd.edu/Software/PepNovo.html (just remove the comment +# chars and replace DIR_TO_PEPNOVO_MODELS) + +#default_models CID_IT_TRYP DIR_TO_PEPNOVO_MODELS +#default_models LTQ_COMP DIR_TO_PEPNOVO_MODELS +#default_models DBC4_PEAK DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG5 DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG6 DIR_TO_PEPNOVO_MODELS +#default_models ITDNV_PEAK DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_SCORE DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG3 DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_DNVPART DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG4 DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_DB DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_CSP DIR_TO_PEPNOVO_MODELS |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d tool.conf --- a/tool.conf Fri May 17 10:07:55 2019 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,160 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<toolbox> - <section id="section-id-Peptidepropertyprediction" name="Peptide property prediction"> - <tool file="openms/PTModel.xml"/> - <tool file="openms/PTPredict.xml"/> - <tool file="openms/RTModel.xml"/> - <tool file="openms/RTPredict.xml"/> - </section> - <section id="section-id-TargetedExperiments" name="Targeted Experiments"> - <tool file="openms/InclusionExclusionListCreator.xml"/> - <tool file="openms/MRMMapper.xml"/> - <tool file="openms/OpenSwathAnalyzer.xml"/> - <tool file="openms/OpenSwathAssayGenerator.xml"/> - <tool file="openms/OpenSwathChromatogramExtractor.xml"/> - <tool file="openms/OpenSwathConfidenceScoring.xml"/> - <tool file="openms/OpenSwathDecoyGenerator.xml"/> - <tool file="openms/OpenSwathDIAPreScoring.xml"/> - <tool file="openms/OpenSwathFeatureXMLToTSV.xml"/> - <tool file="openms/OpenSwathFileSplitter.xml"/> - <tool file="openms/OpenSwathMzMLFileCacher.xml"/> - <tool file="openms/OpenSwathRewriteToFeatureXML.xml"/> - <tool file="openms/OpenSwathRTNormalizer.xml"/> - <tool file="openms/PrecursorIonSelector.xml"/> - <tool file="openms/TargetedFileConverter.xml"/> - </section> - <section id="section-id-Utilities" name="Utilities"> - <tool file="openms/AccurateMassSearch.xml"/> - <tool file="openms/CVInspector.xml"/> - <tool file="openms/DatabaseFilter.xml"/> - <tool file="openms/DecoyDatabase.xml"/> - <tool file="openms/DeMeanderize.xml"/> - <tool file="openms/Digestor.xml"/> - <tool file="openms/DigestorMotif.xml"/> - <tool file="openms/ERPairFinder.xml"/> - <tool file="openms/FFEval.xml"/> - <tool file="openms/FuzzyDiff.xml"/> - <tool file="openms/IDDecoyProbability.xml"/> - <tool file="openms/IDExtractor.xml"/> - <tool file="openms/IDMassAccuracy.xml"/> - <tool file="openms/IDScoreSwitcher.xml"/> - <tool file="openms/IDSplitter.xml"/> - <tool file="openms/LabeledEval.xml"/> - <tool file="openms/LowMemPeakPickerHiRes.xml"/> - <tool file="openms/LowMemPeakPickerHiRes_RandomAccess.xml"/> - <tool file="openms/MassCalculator.xml"/> - <tool file="openms/MetaboliteSpectralMatcher.xml"/> - <tool file="openms/MetaProSIP.xml"/> - <tool file="openms/MRMPairFinder.xml"/> - <tool file="openms/MRMTransitionGroupPicker.xml"/> - <tool file="openms/MSSimulator.xml"/> - <tool file="openms/MultiplexResolver.xml"/> - <tool file="openms/MzMLSplitter.xml"/> - <tool file="openms/OpenSwathWorkflow.xml"/> - <tool file="openms/QCCalculator.xml"/> - <tool file="openms/QCEmbedder.xml"/> - <tool file="openms/QCExporter.xml"/> - <tool file="openms/QCExtractor.xml"/> - <tool file="openms/QCImporter.xml"/> - <tool file="openms/QCMerger.xml"/> - <tool file="openms/QCShrinker.xml"/> - <tool file="openms/RNPxl.xml"/> - <tool file="openms/RNPxlSearch.xml"/> - <tool file="openms/RNPxlXICFilter.xml"/> - <tool file="openms/RTEvaluation.xml"/> - <tool file="openms/SemanticValidator.xml"/> - <tool file="openms/SequenceCoverageCalculator.xml"/> - <tool file="openms/SimpleSearchEngine.xml"/> - <tool file="openms/SpecLibCreator.xml"/> - <tool file="openms/SpectraSTSearchAdapter.xml"/> - <tool file="openms/SvmTheoreticalSpectrumGeneratorTrainer.xml"/> - <tool file="openms/TICCalculator.xml"/> - <tool file="openms/TopPerc.xml"/> - <tool file="openms/TransformationEvaluation.xml"/> - <tool file="openms/XMLValidator.xml"/> - </section> - <section id="section-id-MapAlignment" name="Map Alignment"> - <tool file="openms/ConsensusMapNormalizer.xml"/> - <tool file="openms/FeatureLinkerLabeled.xml"/> - <tool file="openms/FeatureLinkerUnlabeled.xml"/> - <tool file="openms/FeatureLinkerUnlabeledKD.xml"/> - <tool file="openms/FeatureLinkerUnlabeledQT.xml"/> - <tool file="openms/MapRTTransformer.xml"/> - </section> - <section id="section-id-IDProcessing" name="ID Processing"> - <tool file="openms/ConsensusID.xml"/> - <tool file="openms/FalseDiscoveryRate.xml"/> - <tool file="openms/FidoAdapter.xml"/> - <tool file="openms/IDConflictResolver.xml"/> - <tool file="openms/IDFileConverter.xml"/> - <tool file="openms/IDFilter.xml"/> - <tool file="openms/IDMapper.xml"/> - <tool file="openms/IDPosteriorErrorProbability.xml"/> - <tool file="openms/IDRTCalibration.xml"/> - <tool file="openms/LuciphorAdapter.xml"/> - <tool file="openms/PeptideIndexer.xml"/> - <tool file="openms/PhosphoScoring.xml"/> - </section> - <section id="section-id-Signalprocessingandpreprocessing" name="Signal processing and preprocessing"> - <tool file="openms/BaselineFilter.xml"/> - <tool file="openms/ExternalCalibration.xml"/> - <tool file="openms/HighResPrecursorMassCorrector.xml"/> - <tool file="openms/InternalCalibration.xml"/> - <tool file="openms/MapNormalizer.xml"/> - <tool file="openms/MassTraceExtractor.xml"/> - <tool file="openms/NoiseFilterGaussian.xml"/> - <tool file="openms/NoiseFilterSGolay.xml"/> - <tool file="openms/PeakPickerHiRes.xml"/> - <tool file="openms/PeakPickerIterative.xml"/> - <tool file="openms/PeakPickerWavelet.xml"/> - <tool file="openms/PrecursorMassCorrector.xml"/> - <tool file="openms/SpectraMerger.xml"/> - <tool file="openms/TOFCalibration.xml"/> - </section> - <section id="section-id-Identification" name="Identification"> - <tool file="openms/CompNovoCID.xml"/> - <tool file="openms/CompNovo.xml"/> - <tool file="openms/InspectAdapter.xml"/> - <tool file="openms/MascotAdapter.xml"/> - <tool file="openms/MascotAdapterOnline.xml"/> - <tool file="openms/MSGFPlusAdapter.xml"/> - <tool file="openms/ProteinInference.xml"/> - <tool file="openms/SpectraFilterBernNorm.xml"/> - <tool file="openms/SpectraFilterMarkerMower.xml"/> - <tool file="openms/SpectraFilterNLargest.xml"/> - <tool file="openms/SpectraFilterNormalizer.xml"/> - <tool file="openms/SpectraFilterParentPeakMower.xml"/> - <tool file="openms/SpectraFilterScaler.xml"/> - <tool file="openms/SpectraFilterSqrtMower.xml"/> - <tool file="openms/SpectraFilterThresholdMower.xml"/> - <tool file="openms/SpectraFilterWindowMower.xml"/> - <tool file="openms/XTandemAdapter.xml"/> - </section> - <section id="section-id-FileHandling" name="File Handling"> - <tool file="openms/DTAExtractor.xml"/> - <tool file="openms/FileConverter.xml"/> - <tool file="openms/FileFilter.xml"/> - <tool file="openms/FileInfo.xml"/> - <tool file="openms/FileMerger.xml"/> - <tool file="openms/IDMerger.xml"/> - <tool file="openms/IDRipper.xml"/> - <tool file="openms/MapStatistics.xml"/> - <tool file="openms/MzTabExporter.xml"/> - <tool file="openms/TextExporter.xml"/> - </section> - <section id="section-id-Quantitation" name="Quantitation"> - <tool file="openms/AdditiveSeries.xml"/> - <tool file="openms/Decharger.xml"/> - <tool file="openms/EICExtractor.xml"/> - <tool file="openms/FeatureFinderCentroided.xml"/> - <tool file="openms/FeatureFinderIdentification.xml"/> - <tool file="openms/FeatureFinderIsotopeWavelet.xml"/> - <tool file="openms/FeatureFinderMetabo.xml"/> - <tool file="openms/FeatureFinderMRM.xml"/> - <tool file="openms/FeatureFinderMultiplex.xml"/> - <tool file="openms/FeatureFinderSuperHirn.xml"/> - <tool file="openms/IsobaricAnalyzer.xml"/> - <tool file="openms/ProteinQuantifier.xml"/> - <tool file="openms/ProteinResolver.xml"/> - </section> -</toolbox> |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Wed Sep 09 19:57:32 2020 +0000 |
| b |
| @@ -0,0 +1,7 @@ +<tables> + <!-- Locations of pepnovo models --> + <table name="pepnovo_models" comment_char="#" allow_duplicate_entries="False"> + <columns>name,value,path</columns> + <file path="tool-data/pepnovo_models.loc" /> + </table> +</tables> |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d tool_data_table_conf.xml.test --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.test Wed Sep 09 19:57:32 2020 +0000 |
| b |
| @@ -0,0 +1,7 @@ +<tables> + <!-- Locations of all eggnog_mapper data --> + <table name="pepnovo_models" comment_char="#"> + <columns>name,value,path</columns> + <file path="${__HERE__}/test-data/pepnovo_models.loc" /> + </table> +</tables> |
| b |
| diff -r a604e5e1bb06 -r 94b81345543d tools_blacklist.txt --- a/tools_blacklist.txt Fri May 17 10:07:55 2019 -0400 +++ b/tools_blacklist.txt Wed Sep 09 19:57:32 2020 +0000 |
| b |
| @@ -1,9 +1,24 @@ -OMSSAAdapter -MyriMatchAdapter -PepNovoAdapter -SeedListGenerator -SpecLibSearcher -MapAlignerIdentification -MapAlignerPoseClustering -MapAlignerSpectrum -MapAlignerRTTransformer +# seems not possible for 2.5 https://github.com/OpenMS/OpenMS/issues/4426 +DigestorMotif + +# deprecated https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Documentation/release/latest/html/UTILS_IDDecoyProbability.html +# https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Documentation/release/latest/html/a16242.html +IDDecoyProbability + +# personal communication with author V. Bafna: +# "InsPect is no longer maintained as there are many better tools including MS-GF+" +InspectAdapter + +# licence? see https://github.com/bioconda/bioconda-recipes/issues/18953 +#MSFraggerAdapter + +# seems currently unusable https://github.com/OpenMS/OpenMS/pull/4443 +OpenSwathDIAPreScoring + +# https://github.com/OpenMS/OpenMS/issues/4550#issuecomment-594065727 +ProteomicsLFQ + +# https://github.com/OpenMS/OpenMS/issues/4401 +InclusionExclusionListCreator +RTPredict +PTPredict |