Repository 'openms_openswathfilesplitter'
hg clone https://toolshed.g2.bx.psu.edu/repos/galaxyp/openms_openswathfilesplitter

Changeset 9:0ab17cc5cf0c (2020-09-09)
Previous changeset 8:8b814eafce2c (2019-05-17) Next changeset 10:2d856cba192b (2020-09-24)
Commit message:
"planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 020906fb54bde7fc143c356f41975c378a741315"
modified:
OpenSwathFileSplitter.xml
filetypes.txt
macros.xml
readme.md
tools_blacklist.txt
added:
OMSSAAdapter.patch
PepNovoAdapter.patch
fill_ctd.py
fill_ctd_clargs.py
generate-foo.sh
generate.sh
hardcoded_params.json
macros_autotest.xml
macros_discarded_auto.xml
macros_test.xml
prepare_test_data.sh
prepare_test_data_manual.sh
test-data.sh
tool-data/pepnovo_models.loc.sample
tool_data_table_conf.xml.sample
tool_data_table_conf.xml.test
removed:
tool.conf
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c OMSSAAdapter.patch
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/OMSSAAdapter.patch Wed Sep 09 19:55:24 2020 +0000
[
@@ -0,0 +1,10 @@
+--- OMSSAAdapter.xml 2020-06-16 15:51:40.315400730 +0200
++++ /tmp/OMSSAAdapter.xml 2020-06-16 15:50:23.536086074 +0200
+@@ -22,6 +22,7 @@
+ mkdir database &&
+ ln -s '$database' 'database/${re.sub("[^\w\-_]", "_", $database.element_identifier)}.$gxy2omsext($database.ext)' &&

++makeblastdb -dbtype prot -in 'database/${re.sub("[^\w\-_]", "_", $database.element_identifier)}.$gxy2omsext($database.ext)' &&
+ ## Main program call

+ set -o pipefail &&
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c OpenSwathFileSplitter.xml
--- a/OpenSwathFileSplitter.xml Fri May 17 09:58:11 2019 -0400
+++ b/OpenSwathFileSplitter.xml Wed Sep 09 19:55:24 2020 +0000
[
@@ -1,57 +1,88 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <!--This is a configuration file for the integration of a tools into Galaxy (https://galaxyproject.org/). This file was automatically generated using CTDConverter.-->
 <!--Proposed Tool Section: [Targeted Experiments]-->
-<tool id="OpenSwathFileSplitter" name="OpenSwathFileSplitter" version="2.3.0">
+<tool id="OpenSwathFileSplitter" name="OpenSwathFileSplitter" version="@TOOL_VERSION@+galaxy@GALAXY_VERSION@" profile="20.05">
   <description>Splits SWATH files into n files, each containing one window.</description>
   <macros>
     <token name="@EXECUTABLE@">OpenSwathFileSplitter</token>
     <import>macros.xml</import>
+    <import>macros_autotest.xml</import>
+    <import>macros_test.xml</import>
   </macros>
-  <expand macro="references"/>
+  <expand macro="requirements"/>
   <expand macro="stdio"/>
-  <expand macro="requirements"/>
-  <command detect_errors="aggressive"><![CDATA[OpenSwathFileSplitter
+  <command detect_errors="exit_code"><![CDATA[@QUOTE_FOO@
+@EXT_FOO@
+#import re
 
--in
-  #for token in $param_in:
-    $token
-  #end for
-#if $adv_opts.adv_opts_selector=='advanced':
-    #if $adv_opts.param_outputDirectory:
-  -outputDirectory     "$adv_opts.param_outputDirectory"
-#end if
-    #if $adv_opts.param_force:
-  -force
-#end if
+## Preprocessing
+mkdir in &&
+ln -s '$in' 'in/${re.sub("[^\w\-_]", "_", $in.element_identifier)}.$gxy2omsext($in.ext)' &&
+mkdir adv_opts_cond.outputDirectory &&
+#if "out_qc_FLAG" in str($OPTIONAL_OUTPUTS).split(',')
+  mkdir out_qc &&
 #end if
-&gt; $param_stdout
-]]></command>
+
+## Main program call
+
+set -o pipefail &&
+@EXECUTABLE@ -write_ctd ./ &&
+python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hardcoded_json' &&
+@EXECUTABLE@ -ini @EXECUTABLE@.ctd
+-in
+'in/${re.sub("[^\w\-_]", "_", $in.element_identifier)}.$gxy2omsext($in.ext)'
+-outputDirectory
+adv_opts_cond.outputDirectory/
+#if "out_qc_FLAG" in str($OPTIONAL_OUTPUTS).split(',')
+  -out_qc
+  'out_qc/output.${gxy2omsext("json")}'
+#end if
+#if len(str($OPTIONAL_OUTPUTS).split(',')) == 0
+  | tee '$stdout'
+#end if
+
+## Postprocessing
+#if "out_qc_FLAG" in str($OPTIONAL_OUTPUTS).split(',')
+  && mv 'out_qc/output.${gxy2omsext("json")}' '$out_qc'
+#end if
+#if "ctd_out_FLAG" in $OPTIONAL_OUTPUTS
+  && mv '@EXECUTABLE@.ctd' '$ctd_out'
+#end if]]></command>
+  <configfiles>
+    <inputs name="args_json" data_style="paths"/>
+    <configfile name="hardcoded_json"><![CDATA[{"log": "log.txt", "threads": "\${GALAXY_SLOTS:-1}", "no_progress": true}]]></configfile>
+  </configfiles>
   <inputs>
-    <param name="param_in" type="data" format="mzml,mzxml" multiple="true" optional="False" size="30" label="Input file (SWATH/DIA file)" help="(-in) ">
-      <sanitizer>
-        <valid initial="string.printable">
-          <remove value="'"/>
-          <remove value="&quot;"/>
-        </valid>
-      </sanitizer>
+    <param name="in" argument="-in" type="data" format="mzml,mzxml" optional="false" label="Input file (SWATH/DIA file)" help=" select mzml,mzxml data sets(s)"/>
+    <expand macro="adv_opts_macro">
+      <param name="force" argument="-force" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Overwrite tool specific checks" help=""/>
+      <param name="test" argument="-test" type="hidden" optional="true" value="False" label="Enables the test mode (needed for internal use only)" help="">
+        <expand macro="list_string_san"/>
+      </param>
+    </expand>
+    <param name="OPTIONAL_OUTPUTS" type="select" multiple="true" label="Optional outputs" optional="true">
+      <option value="out_qc_FLAG">out_qc (Enables the test mode (needed for internal use only))</option>
+      <option value="ctd_out_FLAG">Output used ctd (ini) configuration file</option>
     </param>
-    <expand macro="advanced_options">
-      <param name="param_outputDirectory" type="text" size="30" value="./" label="Output path to store the split files" help="(-outputDirectory) ">
-        <sanitizer>
-          <valid initial="string.printable">
-            <remove value="'"/>
-            <remove value="&quot;"/>
-          </valid>
-        </sanitizer>
-      </param>
-      <param name="param_force" display="radio" type="boolean" truevalue="-force" falsevalue="" checked="false" optional="True" label="Overwrite tool specific checks" help="(-force) "/>
-    </expand>
   </inputs>
   <outputs>
-    <data name="param_stdout" format="txt" label="Output from stdout"/>
+    <collection type="list" name="adv_opts_cond.outputDirectory" label="${tool.name} on ${on_string}: outputDirectory">
+      <discover_datasets directory="adv_opts_cond.outputDirectory" format="mzml" pattern="_?(?P&lt;designation&gt;.*)\.[^.]*"/>
+    </collection>
+    <data name="out_qc" label="${tool.name} on ${on_string}: out_qc" format="json">
+      <filter>OPTIONAL_OUTPUTS is not None and "out_qc_FLAG" in OPTIONAL_OUTPUTS</filter>
+    </data>
+    <data name="ctd_out" format="xml" label="${tool.name} on ${on_string}: ctd">
+      <filter>OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS</filter>
+    </data>
   </outputs>
-  <help>Splits SWATH files into n files, each containing one window.
+  <tests>
+    <expand macro="autotest_OpenSwathFileSplitter"/>
+    <expand macro="manutest_OpenSwathFileSplitter"/>
+  </tests>
+  <help><![CDATA[Splits SWATH files into n files, each containing one window.
 
 
-For more information, visit https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Documentation/release/2.3.0/html/UTILS_OpenSwathFileSplitter.html</help>
+For more information, visit http://www.openms.de/documentation/UTILS_OpenSwathFileSplitter.html]]></help>
+  <expand macro="references"/>
 </tool>
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c PepNovoAdapter.patch
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/PepNovoAdapter.patch Wed Sep 09 19:55:24 2020 +0000
b
@@ -0,0 +1,35 @@
+--- PepNovoAdapter.xml 2020-05-12 15:55:24.712831518 +0200
++++ /tmp/PepNovoAdapter.xml 2020-05-12 15:36:31.267276757 +0200
+@@ -42,8 +42,13 @@
+   </configfiles>
+   <inputs>
+     <param name="in" argument="-in" type="data" format="mzml" optional="false" label="input file" help=" select mzml data sets(s)"/>
+-    <param name="model_directory" argument="-model_directory" type="text" optional="false" value="" label="Name of the directory where the model files are kept" help="">
+-      <expand macro="list_string_san"/>
++    <param name="model_directory" argument="-model_directory" type="select" optional="false" label="Name of the directory where the model files are kept" help="">
++        <options from_data_table="pepnovo_models">
++            <column name="name" index="0"/>
++            <column name="value" index="2"/>
++            <filter type="unique_value" name="unique_set" column="0"/>
++            <validator type="no_options" message="No model directory available"/>
++        </options>
+     </param>
+     <param name="correct_pm" argument="-correct_pm" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Find optimal precursor mass and charge values" help=""/>
+     <param name="use_spectrum_charge" argument="-use_spectrum_charge" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Do not correct charge" help=""/>
+@@ -51,8 +56,14 @@
+     <param name="no_quality_filter" argument="-no_quality_filter" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Do not remove low quality spectra" help=""/>
+     <param name="fragment_tolerance" argument="-fragment_tolerance" type="float" optional="true" value="-1.0" label="The fragment tolerance (between 0 and 0.75 Da" help="Set to -1.0 to use model's default setting)"/>
+     <param name="pm_tolerance" argument="-pm_tolerance" type="float" optional="true" value="-1.0" label="The precursor mass tolerance (between 0 and 5.0 Da" help="Set to -1.0 to use model's default setting)"/>
+-    <param name="model" argument="-model" type="text" optional="true" value="CID_IT_TRYP" label="Name of the model that should be used" help="">
+-      <expand macro="list_string_san"/>
++    <param name="model" argument="-model" type="select" label="Name of the model that should be used" help="">
++        <options from_data_table="pepnovo_models">
++            <column name="name" index="1"/>
++            <column name="value" index="1"/>
++            <filter type="param_value" ref="model_directory" column="2"/>
++            <filter type="unique_value" column="1"/>
++            <validator type="no_options" message="No model available"/>
++        </options>
+     </param>
+     <param name="digest" argument="-digest" display="radio" type="select" optional="false" label="Enzyme used for digestion (default TRYPSIN)" help="">
+       <option value="TRYPSIN" selected="true">TRYPSIN</option>
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c filetypes.txt
--- a/filetypes.txt Fri May 17 09:58:11 2019 -0400
+++ b/filetypes.txt Wed Sep 09 19:55:24 2020 +0000
[
@@ -1,29 +1,85 @@
-# CTD type    # Galaxy type     # Long Galaxy data type                    # Mimetype
-csv           tabular           galaxy.datatypes.tabular:Tabular
-fasta         fasta             galaxy.datatypes.sequence:Fasta
-FASTA         fasta             galaxy.datatypes.sequence:Fasta
-ini           txt               galaxy.datatypes.data:Text
-txt           txt               galaxy.datatypes.data:Text
-options       txt               galaxy.datatypes.data:Text
-grid          grid              galaxy.datatypes.data:Grid
-html          html              galaxy.datatypes.text:Html                  text/html
-HTML          html              galaxy.datatypes.text:Html                  text/html
-TraML         traml             galaxy.datatypes.proteomics:TraML           application/xml
-traML         traml             galaxy.datatypes.proteomics:TraML           application/xml
-XML           xml               galaxy.datatypes.xml:GenericXml             application/xml
-consensusXML  consensusxml      galaxy.datatypes.proteomics:ConsensusXML    application/xml
-edta          tabular           galaxy.datatypes.tabular:Tabular 
-featureXML    featurexml        galaxy.datatypes.proteomics:FeatureXML      application/xml
-idXML         idxml             galaxy.datatypes.proteomics:IdXML           application/xml
-mzML          mzml              galaxy.datatypes.proteomics:MzML            application/xml
-mzXML         mzxml             galaxy.datatypes.proteomics:MzXML           application/xml
-pepXML        pepxml            galaxy.datatypes.proteomics:PepXml          application/xml
-qcML          qcml              galaxy.datatypes.xml:GenericXml             application/xml
-trafoXML      trafoxml          galaxy.datatypes.xml:GenericXml             application/xml
-tsv           tabular           galaxy.datatypes.tabular:Tabular
-xsd           txt               galaxy.datatypes.data:Text
-mzq           mzq               galaxy.datatypes.proteomics:MzQuantML       application/xml
-msp           msp               galaxy.datatypes.proteomics:Msp
-mzid          mzid              galaxy.datatypes.proteomics:MzIdentML       application/xml
-png           png               galaxy.datatypes.images:Png                 image/png
-mgf           mgf               galaxy.datatypes.proteomics:Mgf
+# CTD type    # Galaxy type
+# the following lines need to be at the top in order to ensure 
+# correct translation Galaxy->CTD type for the ambiguous cases
+# (should only be relevant for the autogenerated tests [which 
+# do not set the ftype of the inputs])
+txt           txt
+tsv           tabular
+
+##analysisXML
+# XTandemAdapter output is named xml in OMS (which is to unspecific) and bioml in Galaxy .. so this is renamed via hardcoded parameters 
+bioml         xml
+consensusXML  consensusxml
+# TODO csv is problematic, since csv often actually means tsv .. but not always
+csv           csv
+##dat
+dta           dta
+dta2d         dta2d
+edta          edta
+fa            fasta
+fas           fasta
+fasta         fasta
+FASTA         fasta
+featureXML    featurexml
+featurexml    featurexml
+# fid
+html          html
+HTML          html
+idXML         idxml
+##ini         txt
+json          json
+kroenik       kroenik
+mascotXML     mascotxml
+mgf           mgf
+mrm           mrm
+ms            sirius.ms
+ms2           ms2
+msp           msp
+mzData        mzdata
+mzid          mzid
+# important to have mzML first, since LuciphorAdapter is case sensitive https://github.com/OpenMS/OpenMS/issues/4444
+mzML          mzml
+mzml          mzml
+mzq           mzq
+mzTab         mztab
+mzXML         mzxml
+novor         txt
+obo           obo
+# I guess this is the idXML output of omssa
+omssaXML      idxml
+osw           osw
+OSW           osw
+params        txt
+paramXML      paramxml
+fasta         peff
+peplist       peplist
+# TODO pep.xml should be removed with OMS 2.6 https://github.com/OpenMS/OpenMS/pull/4541
+pep.xml       pepxml
+pepXML        pepxml
+png           png
+PNG           png
+protXML       protxml
+psms          psms
+# TODO implement or use
+# psq
+pqp           pqp
+qcML          qcml
+spec.xml      spec.xml
+splib         splib
+sqMass        sqmass
+tandem.xml    tandem
+trafoXML      trafoxml
+traML         traml
+TraML         traml
+tab           tabular
+## MOVED TO TOP txt           txt
+raw           thermo.raw
+## xls: SpectraSTSearchAdapter https://github.com/OpenMS/OpenMS/pull/4419
+xls           tsv
+XML           xml
+xml           xml
+xquest.xml    xquest.xml
+xsd           xml
+
+# TODO needs to be implemented, needs to be below xml in order that Galaxy->OMS mapping gives xml
+# cachedMzML    xml
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c fill_ctd.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/fill_ctd.py Wed Sep 09 19:55:24 2020 +0000
[
@@ -0,0 +1,190 @@
+import collections
+import json
+import operator
+import os
+import re
+import subprocess
+import sys
+from functools import reduce  # forward compatibility for Python 3
+
+from CTDopts.CTDopts import (
+    _Choices,
+    _InFile,
+    _Null,
+    _NumericRange,
+    CTDModel
+)
+
+
+def getFromDict(dataDict, mapList):
+    return reduce(operator.getitem, mapList, dataDict)
+
+
+def setInDict(dataDict, mapList, value):
+    getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value
+
+
+def mergeDicts(d, e):
+    """
+    insert values from the dict e into dict d
+    no values of d are overwritten
+    """
+    for k, v in e.items():
+        if (k in d and isinstance(d[k], dict) and isinstance(e[k], collections.abc.Mapping)):
+            mergeDicts(d[k], e[k])
+        elif k not in d and not isinstance(e[k], collections.abc.Mapping):
+            d[k] = e[k]
+        else:
+            sys.stderr.write("fill_ctd.py: could not merge key %s for %s in %s" % (k, d, e))
+            sys.exit(1)
+
+
+def _json_object_hook_noenvlookup(d):
+    return _json_object_hook(d, envlookup=False)
+
+
+def _json_object_hook(d, envlookup=True):
+    """
+    wee helper to transform the json written by galaxy
+    while loading
+    - True/False (bool objects) -> "true"/"false" (lowercase string)
+    - data inputs with multiple and optional true give [None] if no file is given -> []
+    - None -> "" (empty string)
+    - replace bash expressions (if envlookup is True):
+      - environment variables (need to consist capital letters and _) by their value
+      - expressions
+    """
+    for k in d.keys():
+        # if type(d[k]) is bool:
+        #     d[k] = str(d[k]).lower()
+        # else
+        if type(d[k]) is list and len(d[k]) == 1 and d[k][0] is None:
+            d[k] = []
+        elif d[k] is None:
+            d[k] = ""
+        elif envlookup and type(d[k]) is str and d[k].startswith("$"):
+            m = re.fullmatch(r"\$([A-Z_]+)", d[k])
+            if m:
+                d[k] = os.environ.get(m.group(1), "")
+                continue
+            m = re.fullmatch(r"\$(\{[A-Z_]+):-(.*)\}", d[k])
+            if m:
+                d[k] = os.environ.get(m.group(1), m.group(2))
+                continue
+
+            try:
+                p = subprocess.run("echo %s" % d[k], shell=True, check=True, stdout=subprocess.PIPE, encoding="utf8")
+                d[k] = p.stdout.strip()
+            except subprocess.CalledProcessError:
+                sys.stderr.write("fill_ctd error: Could not evaluate %s" % d[k])
+                continue
+    return d
+
+
+def qstring2list(qs):
+    """
+    transform a space separated string that is quoted by " into a list
+    """
+    lst = list()
+    qs = qs.split(" ")
+    quoted = False
+    for p in qs:
+        if p == "":
+            continue
+        if p.startswith('"') and p.endswith('"'):
+            lst.append(p[1:-1])
+        elif p.startswith('"'):
+            quoted = True
+            lst.append(p[1:] + " ")
+        elif p.endswith('"'):
+            quoted = False
+            lst[-1] += p[:-1]
+        else:
+            if quoted:
+                lst[-1] += p + " "
+            else:
+                lst.append(p)
+    return lst
+
+
+def fix_underscores(args):
+    if type(args) is dict:
+        for k in list(args.keys()):
+            v = args[k]
+            if type(v) is dict:
+                fix_underscores(args[k])
+            if k.startswith("_"):
+                args[k[1:]] = v
+                del args[k]
+    elif type(args) is list:
+        for i, v in enumerate(args):
+            if type(v) is dict:
+                fix_underscores(args[i])
+
+
+input_ctd = sys.argv[1]
+
+# load user specified parameters from json
+with open(sys.argv[2]) as fh:
+    args = json.load(fh, object_hook=_json_object_hook_noenvlookup)
+
+# load hardcoded parameters from json
+with open(sys.argv[3]) as fh:
+    hc_args = json.load(fh, object_hook=_json_object_hook)
+
+# insert the hc_args into the args
+mergeDicts(args, hc_args)
+
+if "adv_opts_cond" in args:
+    args.update(args["adv_opts_cond"])
+    del args["adv_opts_cond"]
+
+# IDMapper has in and spectra:in params, in is used in out as format_source",
+# which does not work in Galaxy: https://github.com/galaxyproject/galaxy/pull/9493"
+# therefore hardcoded params change the name of spectra:in to spectra:_in
+# which is corrected here again
+# TODO remove once PR is in and adapt profile accordingly
+fix_underscores(args)
+
+model = CTDModel(from_file=input_ctd)
+
+# transform values from json that correspond to
+# - old style booleans (string + restrictions) -> transformed to a str
+# - unrestricted ITEMLIST which are represented as strings
+#   ("=quoted and space separated) in Galaxy -> transform to lists
+# - optional data input parameters that have defaults and for which no
+#   value is given -> overwritte with the default
+for p in model.get_parameters():
+
+    # check if the parameter is in the arguments from the galaxy tool
+    # (from the json file(s)), since advanced parameters are absent
+    # if the conditional is set to basic parameters
+    try:
+        getFromDict(args, p.get_lineage(name_only=True))
+    except KeyError:
+        # few tools use dashes in parameters which are automatically replaced
+        # by underscores by Galaxy. in these cases the dictionary needs to be
+        # updated
+        # TODO might be removed later https://github.com/OpenMS/OpenMS/pull/4529
+        try:
+            lineage = [_.replace("-", "_") for _ in p.get_lineage(name_only=True)]
+            val = getFromDict(args, lineage)
+        except KeyError:
+            continue
+        else:
+            setInDict(args, lineage, val)
+
+    if p.type is str and type(p.restrictions) is _Choices and set(p.restrictions.choices) == set(["true", "false"]):
+        v = getFromDict(args, p.get_lineage(name_only=True))
+        setInDict(args, p.get_lineage(name_only=True), str(v).lower())
+
+    elif p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange):
+        v = getFromDict(args, p.get_lineage(name_only=True))
+        if type(v) is str:
+            setInDict(args, p.get_lineage(name_only=True), qstring2list(v))
+    elif p.type is _InFile and not (p.default is None or type(p.default) is _Null):
+        v = getFromDict(args, p.get_lineage(name_only=True))
+        if v in [[], ""]:
+            setInDict(args, p.get_lineage(name_only=True), p.default)
+
+model.write_ctd(input_ctd, arg_dict=args)
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c fill_ctd_clargs.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/fill_ctd_clargs.py Wed Sep 09 19:55:24 2020 +0000
b
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser
+from io import StringIO
+
+from CTDopts.CTDopts import (
+    CTDModel,
+    ModelTypeError,
+    Parameters
+)
+
+if __name__ == "__main__":
+    # note add_help=False since otherwise arguments starting with -h will
+    # trigger an error (despite allow_abbreviate)
+    parser = ArgumentParser(prog="fill_ctd_clargs",
+                            description="fill command line arguments"
+                            "into a CTD file and write the CTD file to",
+                            add_help=False, allow_abbrev=False)
+    parser.add_argument("--ctd", dest="ctd", help="input ctd file",
+                        metavar='CTD', default=None, required=True)
+    args, cliargs = parser.parse_known_args()
+    # load CTDModel
+    model = None
+    try:
+        model = CTDModel(from_file=args.ctd)
+    except ModelTypeError:
+        pass
+    try:
+        model = Parameters(from_file=args.ctd)
+    except ModelTypeError:
+        pass
+    assert model is not None, "Could not parse %s, seems to be no CTD/PARAMS" % (args.ctd)
+
+    # get a dictionary of the ctd arguments where the values of the parameters
+    # given on the command line are overwritten
+    margs = model.parse_cl_args(cl_args=cliargs, ignore_required=True)
+
+    # write the ctd with the values taken from the dictionary
+    out = StringIO()
+    ctd_tree = model.write_ctd(out, margs)
+    print(out.getvalue())
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c generate-foo.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/generate-foo.sh Wed Sep 09 19:55:24 2020 +0000
[
b'@@ -0,0 +1,217 @@\n+#!/usr/bin/env bash\n+\n+# parse test definitions from OpenMS sources for a tool with a given id\n+function get_tests2 {\n+\tid=$1\n+\t>&2 echo "generate tests for $id"\n+\techo \'<xml name="autotest_\'"$id"\'">\'\n+\n+\t# get the tests from the CMakeLists.txt\n+\t# 1st remove some tests\n+\t# - Filefilter with empty select_palarity value (empty is not in the list of allowed options)\n+\t# - MassTraceExtractor with outdated ini file leading to wrong parameters https://github.com/OpenMS/OpenMS/issues/4386\n+\t# - OpenSwathMzMLFileCacher with -convert_back argumen https://github.com/OpenMS/OpenMS/issues/4399\n+    # - IDRipper PATH gets empty causing problems. TODO But overall the option needs to be handled differentlt\n+\t# - several tools with duplicated input (leads to conflict when linking)\n+\t# - TOFCalibration inputs we extension (also in prepare_test_data) https://github.com/OpenMS/OpenMS/pull/4525\n+\t# - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456)\n+\t# - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed)\n+\t# - some input files are originally in a subdir (degenerated cases/), but not in test-data\n+\t# - SeedListGenerator: https://github.com/OpenMS/OpenMS/issues/4404\n+\t# - OpenSwathAnalyzer 9/10: cachedMzML (not supported yet)\n+\tCMAKE=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake  |\n+\t\tsed \'s@${DATA_DIR_SHARE}/@@g\' |\n+\t\tgrep -v \'OpenSwathMzMLFileCacher .*-convert_back\' |\n+\t    \tsed \'s/${TMP_RIP_PATH}/""/\' |\n+\t\tsed \'s@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @\' |\n+\t\tgrep -v "MaRaClusterAdapter.*-consensus_out"|\n+ \t\tgrep -v "FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta " |\n+\t\tsed \'s@degenerate_cases/@@g\' |\n+\t\tgrep -v \'TOPP_SeedListGenerator_3"\' | \n+\t\tegrep -v \'TOPP_OpenSwathAnalyzer_test_3"|TOPP_OpenSwathAnalyzer_test_4"\')\n+\n+\n+# \t\tgrep -v \'FileFilter.*-spectra:select_polarity ""\' |\n+# \t\tgrep -v \'MassTraceExtractor_2.ini \' |\n+# \t\tgrep -v "FileMerger_6_input2.mzML.*FileMerger_6_input2.mzML" |\n+# \t\tgrep -v "IDMerger_1_input1.idXML.*IDMerger_1_input1.idXML" |\n+# \t\tgrep -v "degenerated_empty.idXML.*degenerated_empty.idXML" |\n+# \t\tgrep -v "FeatureLinkerUnlabeledKD_1_output.consensusXML.*FeatureLinkerUnlabeledKD_1_output.consensusXML" |\n+# \t\tgrep -v "FeatureLinkerUnlabeledQT_1_output.consensusXML.*FeatureLinkerUnlabeledQT_1_output.consensusXML" |\n+\n+\t# 1st part is a dirty hack to join lines containing a single function call, e.g.\n+\t# addtest(....\n+\t#         ....)\n+\techo "$CMAKE" | sed \'s/#.*//; s/^\\s*//; s/\\s*$//\' | grep -v "^#" | grep -v "^$"  | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | \n+\t\tgrep -iE "add_test\\(\\"(TOPP|UTILS)_.*/$id " | egrep -v "_prepare\\"|_convert|WRITEINI|WRITECTD|INVALIDVALUE"  | while read -r line\n+\tdo\n+\t\tline=$(echo "$line" | sed \'s/add_test("\\([^"]\\+\\)"/\\1/; s/)$//; s/\\${TOPP_BIN_PATH}\\///g;s/\\${DATA_DIR_TOPP}\\///g; s#THIRDPARTY/##g\')\n+\t\t# >&2 echo $line\n+\t\ttest_id=$(echo "$line" | cut -d" " -f 1)\n+\t\ttool_id=$(echo "$line" | cut -d" " -f 2)\n+\t\tif [[ $test_id =~ _out_?[0-9]? ]]; then\n+\t\t\t>&2 echo "    skip $test_id $line"\n+\t\t\tcontinue\n+\t\tfi\n+\t\tif [[ ${id,,} != ${tool_id,,} ]]; then\n+\t\t\t>&2 echo "    skip $test_id ($id != $tool_id) $line"\n+\t\t\tcontinue\n+\t\tfi\n+\n+\t\t#remove tests with set_tests_properties(....PROPERTIES WILL_FAIL 1)\n+\t\tif grep -lq "$test_id"\'\\".* PROPERTIES WILL_FAIL 1\' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then\n+\t\t\t>&2 echo "    skip failing "$test_id\n+\t\t\tcontinue\n+\t\tfi\n+\t\ttes="  <test>\\n"\n+\t\tline=$(fix_tmp_files "$line")\n+\t\tline=$(unique_files "$line")\n+\t\t# >&2 echo $line\n+\t\t#if there is an ini file then we use this to generate the test\n+\t\t#otherwise the ctd file is used\n+\t\t#other command line parameters are inserted later into this xml\n+\t\tif grep -lq "\\'..b'f to the expected file.\n+#problem: the extension of the tmp files is unusable for test generation.\n+#unfortunately the extensions used in the DIFF lines are not always usable for the CLI\n+#(e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt)\n+#this function replaces the tmp file by the expected file. \n+function fix_tmp_files {\n+#\t>&2 echo "FIX $line"\n+\tret=""\n+\tfor a in $@; do\n+\t\tif [[ ! $a =~ .tmp$ ]]; then\n+\t\t\tret="$ret $a"\n+\t\t\tcontinue\n+\t\tfi\n+#\t\t>&2 echo "    a "$a\n+\t\tg=$(cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | grep \'\\${DIFF}.*\'"$a")\n+#\t\t>&2 echo "    g "$g\n+\t\tin1=$(sed \'s/.*-in1 \\([^ ]\\+\\).*/\\1/\' <<<$g)\n+\t\t# >&2 echo "    in1 "$in1\n+\t\tif [[  "$a" != "$in1" ]]; then\n+\t\t\tret="$ret $a"\n+\t\t\tcontinue\n+\t\tfi\n+\t\tin2=$(sed \'s/.*-in2 \\([^ ]\\+\\).*/\\1/\' <<<$g)\n+\t\tin2=$(basename $in2 | sed \'s/)$//\')\n+\t\t# >&2 echo "    in2 "$in2\n+\t\tif [[ -f "test-data/$in2" ]]; then\n+\t\t\tln -fs "$in1" "test-data/$in2"\n+\t\t\tret="$ret $in2"\n+\t\telse\n+\t\t\tret="$ret $a"\n+\t\tfi\n+\tdone\n+#\t>&2 echo "--> $ret"\n+\techo "$ret"\n+}\n+\n+function link_tmp_files {\n+    # note this also considers commented lines (starting with a #)\n+    # because of tests where the diff command is commented and we\n+    # still want to use the extension of these files\n+    cat $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed \'s/^\\s*//; s/\\s*$//\' | grep -v "^$"  | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | grep "\\${DIFF}" | while read -r line\n+    do\n+        in1=$(sed \'s/.*-in1 \\([^ ]\\+\\).*/\\1/\' <<<$line)\n+        in1=$(basename $in1 | sed \'s/)$//\')\n+        in2=$(sed \'s/.*-in2 \\([^ ]\\+\\).*/\\1/\' <<<$line)\n+        in2=$(basename $in2 | sed \'s/)$//\')\n+        if [[ "$in1" == "$in2" ]]; then\n+            >&2 echo "not linking equal $in1 $in2"\n+            continue\n+        fi\n+        ln -f -s $in1 test-data/$in2\n+    done\n+    for i in test-data/*.tmp\n+    do\n+        if [ ! -e test-data/$(basename $i .tmp) ]; then\n+            ln -s $(basename $i) test-data/$(basename $i .tmp)\n+            #ln -s $(basename $i) test-data/$(basename $i .tmp)\n+        else\n+            ln -fs $(basename $i) test-data/$(basename $i .tmp)\n+        fi\n+    done\n+}\n+\n+\n+\n+# parse data preparation calls from OpenMS sources for a tool with a given id\n+function prepare_test_data {\n+# \tid=$1\n+# | egrep -i "$id\\_.*[0-9]+(_prepare\\"|_convert)?"\n+\tcat $OPENMSGIT/src/tests/topp/CMakeLists.txt  $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake | sed \'s/#.*$//\'| sed \'s/^\\s*//; s/\\s*$//\' | grep -v "^$"  | awk \'{printf("%s@NEWLINE@", $0)}\' | sed \'s/)@NEWLINE@/)\\n/g\' | sed \'s/@NEWLINE@/ /g\' | \n+\t\tsed \'s/degenerate_cases\\///\' | \n+\t\tegrep -v "WRITEINI|WRITECTD|INVALIDVALUE|DIFF" | \n+\t\tgrep add_test | \n+\t\tegrep "TOPP|UTILS" |\n+\t\tsed \'s@${DATA_DIR_SHARE}/@@g;\'|\n+\t\tsed \'s@${TMP_RIP_PATH}@dummy2.tmp@g\'|\n+\t\tsed \'s@TOFCalibration_ref_masses @TOFCalibration_ref_masses.txt @g; s@TOFCalibration_const @TOFCalibration_const.csv @\'| \n+\twhile read line\n+\tdo\n+\t\ttest_id=$(echo "$line" | sed \'s/add_test(//; s/"//g;  s/)[^)]*$//; s/\\${TOPP_BIN_PATH}\\///g;s/\\${DATA_DIR_TOPP}\\///g; s#THIRDPARTY/##g\' | cut -d" " -f1)\n+\n+\t\tif grep -lq "$test_id"\'\\".* PROPERTIES WILL_FAIL 1\' $OPENMSGIT/src/tests/topp/CMakeLists.txt $OPENMSGIT/src/tests/topp/THIRDPARTY/third_party_tests.cmake; then\n+\t\t\t>&2 echo "    skip failing "$test_id\n+\t\t\tcontinue\n+\t\tfi\n+\n+\t\tline=$(echo "$line" | sed \'s/add_test("//; s/)[^)]*$//; s/\\${TOPP_BIN_PATH}\\///g;s/\\${DATA_DIR_TOPP}\\///g; s#THIRDPARTY/##g\' | cut -d" " -f2-)\n+\t\t# line="$(fix_tmp_files $line)"\n+\t\techo "$line > $test_id.stdout 2> $test_id.stderr"\n+\t\techo "if [[ \\"\\$?\\" -ne \\"0\\" ]]; then >&2 echo \'$test_id failed\'; >&2 echo -e \\"stderr:\\n\\$(cat $test_id.stderr | sed \'s/^/    /\')\\"; echo -e \\"stdout:\\n\\$(cat $test_id.stdout)\\";fi"\t\n+    done\n+}\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c generate.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/generate.sh Wed Sep 09 19:55:24 2020 +0000
[
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# VERSION=2.3
+# CONDAPKG=https://anaconda.org/bioconda/openms/2.3.0/download/linux-64/openms-2.3.0-py27h932d754_3.tar.bz2
+# VERSION=2.4
+# CONDAPKG=https://anaconda.org/bioconda/openms/2.4.0/download/linux-64/openms-2.4.0-py27h574aadf_1.tar.bz2
+
+VERSION=2.5
+FILETYPES="filetypes.txt"
+PROFILE="20.05"
+## FILETYPES_RE=$(grep -v "^#" $FILETYPES | grep -v "^$" | cut -f 1 -d" " | tr '\n' '|' | sed 's/|$//'| sed 's/|/\\|/g')
+
+export tmp=$(mktemp -d)
+
+export CTDCONVERTER="$tmp/CTDConverter"
+
+export PYTHONPATH="$(pwd)/CTDopts/"
+###############################################################################
+## reset old data
+###############################################################################
+# rm $(ls *xml |grep -v macros)
+# rm -rf ctd
+# mkdir -p ctd
+# echo "" > prepare_test_data.sh
+
+###############################################################################
+## generate tests
+## also creates 
+## - conda environment (for executing the binaries) and 
+## - the git clone of OpenMS (for generating the tests)
+## - ctd files
+###############################################################################
+bash ./test-data.sh ./macros_autotest.xml
+
+###############################################################################
+## get the 
+## - conda package (for easy access and listing of the OpenMS binaries), 
+###############################################################################
+# if [ ! -d $OPENMSPKG ]; then
+#  mkdir $OPENMSPKG/
+#  wget -P $OPENMSPKG/ "$CONDAPKG"
+#  tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C OpenMS$VERSION-pkg/
+#   rm $OPENMSPKG/"$(basename $CONDAPKG)"
+# fi
+
+###############################################################################
+## Get python libaries for CTD -> Galaxy conversion
+## TODO fix to main repo OR conda packkage if PRs are merged 
+###############################################################################
+# if [ ! -d CTDopts ]; then
+#  # git clone https://github.com/genericworkflownodes/CTDopts CTDopts
+#  git clone -b topic/no-1-2x https://github.com/bernt-matthias/CTDopts CTDopts
+# fi
+if [ ! -d $CTDCONVERTER ]; then
+ #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
+ git clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
+fi
+# export PYTHONPATH=$(pwd)/CTDopts
+###############################################################################
+## conversion ctd->xml 
+###############################################################################
+source $(dirname $(which conda))/../etc/profile.d/conda.sh
+conda activate $tmp/OpenMS$VERSION-env
+python $CTDCONVERTER/convert.py galaxy -i ctd/*ctd -o ./ -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf  -p hardcoded_params.json --test-macros macros_autotest.xml --test-macros-prefix autotest_  --test-macros macros_test.xml --test-macros-prefix manutest_ --tool-version $VERSION --tool-profile $PROFILE > convert.out 2> convert.err
+if [[ "$?" -ne "0" ]]; then >&2 echo 'CTD -> XML conversion failed'; >&2 echo -e "stderr:\n$(cat convert.err)"; fi
+conda deactivate
+
+patch PepNovoAdapter.xml < PepNovoAdapter.patch
+patch OMSSAAdapter.xml < OMSSAAdapter.patch
+
+# #-b version log debug test in_type executable pepnovo_executable param_model_directory rt_concat_trafo_out param_id_pool
+
+# for i in A-E F-H I-L M-N O-P Q-Z
+# do
+#  planemo t [$i]*xml --galaxy_branch release_20.05 --galaxy_python_version 3.7 --test_output $i.html --test_output_json $i.json &
+# done
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c hardcoded_params.json
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hardcoded_params.json Wed Sep 09 19:55:24 2020 +0000
[
@@ -0,0 +1,247 @@
+{
+ "#": "blacklist parameters",
+
+ "version": [{"value": "@"}],
+ "debug": [{"value": "@"}],
+ "algorithm:debug": [{"value": "@"}],
+ "java_memory": [{"value": "@"}],
+ "java_permgen": [{"value": "@"}],
+ "#": "type of input is always determined from the file extension ",
+ "in_type": [{"value": "@"}],
+
+ "#": "tool specific blacklist parameters",
+
+ "convert_back": [{"value": "@", "tools": ["OpenSwathMzMLFileCacher"]}],
+ "NET_executable": [{
+ "value": "@", 
+ "tools": ["FileConverter"]
+ }],
+
+
+ "params_file": [{"value": "@", "tools": ["SpectraSTSearchAdapter"]}],
+
+ "#": "TODO not usable in 2.5 https://github.com/OpenMS/OpenMS/issues/4456, corresponding test currently disabled",
+     "consensus_out": [{"value": "@", "tools": ["MaRaClusterAdapter"]}],
+ "#": "TODO would need treatment as prefix-output",
+ "output_directory": [{"value": "@", "tools": ["MaRaClusterAdapter"]}],
+
+ "#": "hardcode parameter values",
+
+ "comet_version": [{
+ "value":"2016.01 rev. 3"
+ }],
+ "comet_executable": [{
+ "value":"comet"
+ }],
+ "crux_executable": [{
+ "value": "crux"
+ }],
+ "fido_executable": [{
+ "value":"Fido"
+ }],
+ "fidocp_executable": [{
+ "value":"FidoChooseParameters"
+ }],
+ "maracluster_executable": [{
+ "value":"/home/berntm/projects/tools-galaxyp/tools/openms/OpenMS2.5.0-git/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"
+ }],
+ "mascot_directory": [{
+ "value":"TODO"
+ }],
+ "myrimatch_executable": [{
+ "value":"myrimatch"
+ }],
+ "omssa_executable": [{
+ "value":"$(dirname $(realpath $(which omssacl)))/omssacl"
+ }],
+ "ThermoRaw_executable": [{
+ "value": "ThermoRawFileParser.exe", 
+ "tools": ["FileConverter"]
+ }],
+ "pepnovo_executable": [{
+ "value":"pepnovo"
+ }],
+ "percolator_executable": [{
+ "value":"percolator"
+ }],
+ "xtandem_executable": [{
+ "value":"xtandem"
+ }],
+ "executable": [
+ {
+ "value":"$(dirname $(realpath $(which luciphor2)))/luciphor2.jar",
+ "tools": ["LuciphorAdapter"]
+ }, {
+ "value":"/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar",
+ "tools": ["MSFraggerAdapter"]
+ }, {
+ "value":"$(msgf_plus -get_jar_path)",
+ "tools": ["MSGFPlusAdapter"]
+ }, {
+ "value": "/home/berntm/Downloads/novor/lib/novor.jar",
+ "tools": ["NovorAdapter"]
+ }, {
+ "value":"$(which sirius)",
+ "tools": ["SiriusAdapter", "AssayGeneratorMetabo"]
+ }, {
+ "value":"spectrast",
+ "tools": ["SpectraSTSearchAdapter"]
+ }
+ ],
+ "r_executable": [{
+ "value":"R"
+ }],
+ "rscript_executable": [{
+ "value":"Rscript"
+ }],
+ "java_executable": [{
+ "value":"java"
+ }],
+ "log": [{
+ "value":"log.txt"
+ }],
+ "tempDirectory": [{
+ "value":"$TMP_DIR"
+ }],
+ "temp_data_directory": [{
+ "value":"$TMP_DIR"
+ }],
+ "algorithm:Preprocessing:tmp_dir": [{
+ "value":"$TMP_DIR"
+ }],
+ "no_progress": [{
+ "value": true
+ }],
+ "#": "only used in LuciphorAdapter at the moment, inconsistency will be fixed",
+ "num_threads": [{
+ "value":"${GALAXY_SLOTS:-1}"
+ }],
+ "threads": [{
+ "value": "${GALAXY_SLOTS:-1}"
+ }],
+ "#": "hardcode the outer loop threads for OpenSwathWorkflow",
+ "outer_loop_threads": [{
+ "value": "1", 
+ "tools": ["OpenSwathWorkflow"]
+ }],
+ "separator": [{
+ "value": ",",
+ "tools": ["IDMassAccuracy"]
+ }],
+ "#": "overwrite/add Galaxy xml attributes of some parameters (names need to start with param_)",
+
+ "#": "test is not a hardcoded value since we need to set it in the tool tests", 
+ "test": [{
+ "CTD:type": "text",
+ "XML:type": "hidden"
+ }],
+
+ "#": "overwrite CTD attributes of some parameters (some are not possible, e.g. type)",
+
+ "#": "for some tools the user needs to select the desired output type since detection by extension makes no sense for galaxy tools",
+ "out_type": [{
+     "CTD:required": true,
+     "CTD:advanced": false
+ }],
+
+ "#": "SeedListGenerator with consensusXML input needs a dynamic number of outputs that depends on the content of the input, so we remove this options at the moment because its hard or impossible to implement in Galaxy, https://github.com/OpenMS/OpenMS/issues/4404 .. see also in parameter",
+ "#": "FileInfo, MapStatistics, SequenceCoverageCalculator wo -out just writes to stdout. not wanted here",
+ "#": "MascotAdapter: https://github.com/OpenMS/OpenMS/pull/4553",
+ "#": "Phosphoscoring: https://github.com/OpenMS/OpenMS/pull/4569",
+ "#": "MzMLSplitter output prefix https://github.com/OpenMS/OpenMS/issues/4404",
+ "#": "IDRipper: blacklist out (is doing the same as the output-prefix out-path)",
+ "out": [{
+ "CTD:is_list": false, 
+ "tools": ["SeedListGenerator"]
+ }, {
+   "CTD:required": true,
+ "tools": ["FileInfo", "MapStatistics", "SequenceCoverageCalculator"]
+ }, {
+ "CTD:restrictions": "idXML",
+ "tools": ["MascotAdapter"]
+ }, {
+ "CTD:restrictions": "idXML",
+ "tools": ["PhosphoScoring"]
+ }, {
+ "CTD:type": "output-prefix", 
+ "CTD:required": "true",
+ "CTD:restrictions": "mzml",
+ "tools": ["MzMLSplitter"]
+ }, {
+ "value": "@", 
+ "tools": ["IDRipper"]
+ }],
+
+ "#": "https://github.com/OpenMS/OpenMS/pull/4552 .. if this is removed the test will need out_cm_FLAG",
+ "out_cm": [{
+ "CTD:required": true, 
+ "tools": ["MetaboliteAdductDecharger"]
+ }],
+
+
+ "#": "https://github.com/OpenMS/OpenMS/pull/4451", 
+ "out_xquestxml": [{
+ "CTD:restrictions": "xquest.xml",
+ "tools": ["OpenPepXL", "OpenPepXLLF"]
+ }],
+ "out_xquest_specxml": [{
+ "CTD:restrictions": "spec.xml",
+ "tools": ["OpenPepXL", "OpenPepXLLF"]
+ }],
+
+ "#": "Try to remove xml data type whereever possible",
+ "#": "XTandem Adapter output is called .xml in OMS -> use Galaxy's bioml",
+ "xml_out": [{
+ "CTD:restrictions": "bioml",
+ "tools": ["XTandemAdapter"]
+ }],
+
+ "#": "IDFileConverter remove xml",
+ "#": "OpenSwathWorkflow make in single file input and all outputs non-optional",
+        "#": "XFDR does not need xml .. redundant with xquest.xml TODO check if list is up to date with each new release",
+        "#": "SpectraSTSearchAdapter does not need xml .. redundant with pep.xml TODO check if list is up to date with each new release",
+ "#": "SeedListGenerator https://github.com/OpenMS/OpenMS/issues/4404 .. see also out parameter",
+ "in": [{
+ "CTD:restrictions": "pepXML,protXML,mascotXML,omssaXML,bioml,psms,tsv,idXML,mzid,xquest.xml",
+ "tools": ["IDFileConverter"]
+ }, {
+ "CTD:is_list": false, 
+ "tools": ["OpenSwathWorkflow"]
+ }, {
+ "CTD:restrictions": "idXML,mzid,xquest.xml",
+ "tools": ["XFDR"]
+ }, {
+ "CTD:restrictions": "txt,tsv,pep.xml,pepXML,html",
+ "tools": ["SpectraSTSearchAdapter"]
+ }, {
+ "CTD:restrictions": "mzML,idXML,featureXML",
+ "tools": ["SeedListGenerator"]
+ }],
+
+ "#": "parameter is a flag but restrictions in wrong order: TODO https://github.com/OpenMS/OpenMS/pull/4455",
+ "# masstrace_snr_filtering": [{
+ "CTD:restrictions": "true,false",
+ "tools": ["MassTraceExtractor"]
+ }],
+
+ "#": "IDMapper has in and spectra:in params, in is used in out as format_source",
+ "#": "which does not work in Galaxy: https://github.com/galaxyproject/galaxy/pull/9493", 
+ "spectra:in": [{
+ "CTD:name": "_in", 
+ "tools": ["IDMapper"]
+ }],
+
+ "#": "output-prefix",
+ "out_path": [{
+ "CTD:type": "output-prefix", 
+ "CTD:required": "true",
+ "CTD:restrictions": "idXML",
+ "tools": ["IDRipper"]
+ }],
+ "outputDirectory": [{
+ "CTD:type": "output-prefix", 
+ "CTD:required": "true",
+ "CTD:restrictions": "mzml",
+ "tools": ["OpenSwathFileSplitter"]
+ }]
+}
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c macros.xml
--- a/macros.xml Fri May 17 09:58:11 2019 -0400
+++ b/macros.xml Wed Sep 09 19:55:24 2020 +0000
[
@@ -1,11 +1,21 @@
 <?xml version='1.0' encoding='UTF-8'?>
+<!-- CTD2Galaxy depends on this file and on the stdio, advanced_options macros!
+     You can edit this file to add your own macros, if you so desire, or you can
+     add additional macro files using the m/macros parameter -->
 <macros>
+  <token name="@TOOL_VERSION@">2.5</token>
+  <token name="@GALAXY_VERSION@">0</token>
   <xml name="requirements">
     <requirements>
-      <requirement type="package" version="2.3">openms</requirement>
-      <requirement type="package" version="15.12.15.2">xtandem</requirement>
-      <requirement type="package" version="1.0">fido</requirement>
-      <requirement type="package" version="2017.07.21">msgf_plus</requirement>
+      <requirement type="package" version="@TOOL_VERSION@">openms</requirement>
+      <requirement type="package" version="@TOOL_VERSION@">openms-thirdparty</requirement>
+      <!-- makeblastdb for OMSSAAdapter -->
+      <requirement type="package" version="2.9.0">blast</requirement>
+      <requirement type="package" version="8.0.192">openjdk</requirement>
+      <!--<requirement type="package" version="5.0.0">tpp</requirement>-->
+      <!-- for realpath (used e.g. in LuciphorAdapter) -->
+   <!--<requirement type="package" version="8.25">coreutils</requirement>-->
+   <requirement type="package" version="1.4">ctdopts</requirement>
       <yield/>
     </requirements>
   </xml>
@@ -21,8 +31,8 @@
       <citation type="doi">doi:10.1186/1471-2105-9-163</citation>
     </citations>
   </xml>
-  <xml name="advanced_options">
-    <conditional name="adv_opts">
+  <xml name="adv_opts_macro">
+    <conditional name="adv_opts_cond">
       <param name="adv_opts_selector" type="select" label="Advanced Options">
         <option value="basic" selected="True">Hide Advanced Options</option>
         <option value="advanced">Show Advanced Options</option>
@@ -33,4 +43,87 @@
       </when>
     </conditional>
   </xml>
-</macros>
+
+  <!-- sanitizers and validators -->
+  <xml name="list_string_val">
+    <validator type="regex" message="parameter must not start with $">^[^$]</validator>
+    
+    <validator type="regex" message="a space separated list of string is needed (strings that contain spaces can be quoted with &quot;)">^ *((?:\"[^\"]*\" +)|(?:[^ \"]+ +))*((?:\"[^\"]*\")|(?:[^ \"]+)) *$</validator>
+  </xml>
+  <xml name="list_string_san">
+    <sanitizer>
+      <valid initial="string.printable">
+<!--        <remove value="'"/>-->
+<!--        <remove value="\"/>--><!-- otherwise the user could quote the final quote -->
+<!--        <remove value="`"/>-->
+<!--        <remove value="$"/>-->
+<!--        <remove value="&lt;"/>-->
+ <!--<remove value="&amp;"/> removed for MascotAdapterOnline -Mascot_server:export_params which is a URL POST string which can contain & .. could be mapped to &amp; but there is still a & -->
+        <!--<remove value=";"/>-->
+<!--        <remove value="#"/>-->
+      </valid>
+    </sanitizer>
+  </xml>
+  <xml name="list_float_valsan">
+    <validator type="regex" message="a space separated list of float values is required">^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$</validator>
+    <yield/>
+    <sanitizer>
+      <valid initial="string.digits">
+        <add value=" "/>
+        <add value="."/>
+        <add value="E"/>
+        <add value="e"/>
+        <add value="+"/>
+        <add value="-"/>
+      </valid>
+    </sanitizer>
+  </xml>
+  <xml name="list_integer_valsan">
+    <validator type="regex" message="a space separated list of integer values is required">^ *[+-]?[0-9]+( *[+-]?[0-9]+)* *$</validator>
+    <yield/>
+    <sanitizer>
+      <valid initial="string.digits">
+        <add value=" "/>
+        <add value="+"/>
+        <add value="-"/>
+      </valid>
+    </sanitizer>
+  </xml>
+
+  <!-- helper function to quote space separated strings -->
+  <token name="@QUOTE_FOO@">
+#def quote(s):
+    #set $s = [ _ for _ in $s.split(" ") if _ != "" ]
+    #set $q = False
+    #for $i, $p in enumerate($s):
+        #if $p == "":
+            #continue
+        #end if
+        #if $p.startswith('"'):
+            #set $q = True
+        #end if
+##        #if p.startswith('-'):
+##            #set p = "\\" + p
+##        #elif p.startswith('"-'):
+##            #set p = "\\" + p[1:]
+##        #end if
+        #if not $q:
+            #set $s[i] = '"%s"' % p
+        #end if
+        #if $p.endswith('"'):
+            #set $q = False
+        #end if
+    #end for
+    #return " ".join($s)
+#end def
+  </token>
+
+<token name="@EXT_FOO@"><![CDATA[#def oms2gxyext(o)
+    #set m={'txt': 'txt', 'tsv': 'tabular', 'bioml': 'xml', 'consensusXML': 'consensusxml', 'csv': 'csv', 'dta': 'dta', 'dta2d': 'dta2d', 'edta': 'edta', 'fa': 'fasta', 'fas': 'fasta', 'fasta': 'fasta', 'FASTA': 'fasta', 'featureXML': 'featurexml', 'featurexml': 'featurexml', 'html': 'html', 'HTML': 'html', 'idXML': 'idxml', 'json': 'json', 'kroenik': 'kroenik', 'mascotXML': 'mascotxml', 'mgf': 'mgf', 'mrm': 'mrm', 'ms': 'sirius.ms', 'ms2': 'ms2', 'msp': 'msp', 'mzData': 'mzdata', 'mzid': 'mzid', 'mzML': 'mzml', 'mzml': 'mzml', 'mzq': 'mzq', 'mzTab': 'mztab', 'mzXML': 'mzxml', 'novor': 'txt', 'obo': 'obo', 'omssaXML': 'idxml', 'osw': 'osw', 'OSW': 'osw', 'params': 'txt', 'paramXML': 'paramxml', 'peplist': 'peplist', 'pep.xml': 'pepxml', 'pepXML': 'pepxml', 'png': 'png', 'PNG': 'png', 'protXML': 'protxml', 'psms': 'psms', 'pqp': 'pqp', 'qcML': 'qcml', 'spec.xml': 'spec.xml', 'splib': 'splib', 'sqMass': 'sqmass', 'tandem.xml': 'tandem', 'trafoXML': 'trafoxml', 'traML': 'traml', 'TraML': 'traml', 'tab': 'tabular', 'raw': 'thermo.raw', 'xls': 'tsv', 'XML': 'xml', 'xml': 'xml', 'xquest.xml': 'xquest.xml', 'xsd': 'xml'}
+    #return m[o]
+#end def
+#def gxy2omsext(g)
+    #set m={'txt': 'txt', 'tabular': 'tsv', 'xml': 'bioml', 'consensusxml': 'consensusXML', 'csv': 'csv', 'dta': 'dta', 'dta2d': 'dta2d', 'edta': 'edta', 'fasta': 'fa', 'featurexml': 'featureXML', 'html': 'html', 'idxml': 'idXML', 'json': 'json', 'kroenik': 'kroenik', 'mascotxml': 'mascotXML', 'mgf': 'mgf', 'mrm': 'mrm', 'sirius.ms': 'ms', 'ms2': 'ms2', 'msp': 'msp', 'mzdata': 'mzData', 'mzid': 'mzid', 'mzml': 'mzML', 'mzq': 'mzq', 'mztab': 'mzTab', 'mzxml': 'mzXML', 'obo': 'obo', 'osw': 'osw', 'paramxml': 'paramXML', 'peff': 'fasta', 'peplist': 'peplist', 'pepxml': 'pep.xml', 'png': 'png', 'protxml': 'protXML', 'psms': 'psms', 'pqp': 'pqp', 'qcml': 'qcML', 'spec.xml': 'spec.xml', 'splib': 'splib', 'sqmass': 'sqMass', 'tandem': 'tandem.xml', 'trafoxml': 'trafoXML', 'traml': 'traML', 'thermo.raw': 'raw', 'tsv': 'xls', 'xquest.xml': 'xquest.xml'}
+    #return m[g]
+#end def
+]]></token></macros>
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c macros_autotest.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/macros_autotest.xml Wed Sep 09 19:55:24 2020 +0000
b
b'@@ -0,0 +1,26318 @@\n+<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n+<macros>\n+  <xml name="autotest_AccurateMassSearch">\n+    <test expect_num_outputs="2">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="ConsensusMapNormalizer_input.consensusXML"/>\n+      <output name="out" file="AccurateMassSearch_1_output.tmp.mzTab" compare="sim_size" delta="5700" ftype="mztab"/>\n+      <param name="positive_adducts" value="CHEMISTRY/PositiveAdducts.tsv" ftype="tabular"/>\n+      <param name="negative_adducts" value="CHEMISTRY/NegativeAdducts.tsv" ftype="tabular"/>\n+      <section name="db">\n+        <param name="mapping" value="CHEMISTRY/HMDBMappingFile.tsv" ftype="tabular"/>\n+        <param name="struct" value="CHEMISTRY/HMDB2StructMapping.tsv" ftype="tabular"/>\n+      </section>\n+      <section name="algorithm">\n+        <param name="mass_error_value" value="5.0"/>\n+        <param name="mass_error_unit" value="ppm"/>\n+        <param name="ionization_mode" value="positive"/>\n+        <param name="isotopic_similarity" value="false"/>\n+        <param name="use_feature_adducts" value="false"/>\n+        <param name="keep_unidentified_masses" value="false"/>\n+        <section name="mzTab">\n+          <param name="exportIsotopeIntensities" value="false"/>\n+        </section>\n+      </section>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test>\n+    <test expect_num_outputs="3">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="AccurateMassSearch_2_input.featureXML"/>\n+      <output name="out" file="AccurateMassSearch_2_output.tmp.mzTab" compare="sim_size" delta="5700" ftype="mztab"/>\n+      <output name="out_annotation" file="AccurateMassSearch_2_output.tmp.featureXML" compare="sim_size" delta="5700" ftype="featurexml"/>\n+      <param name="positive_adducts" value="AMS_PositiveAdducts.tsv" ftype="tabular"/>\n+      <param name="negative_adducts" value="AMS_NegativeAdducts.tsv" ftype="tabular"/>\n+      <section name="db">\n+        <param name="mapping" value="AMS_test_Mapping.tsv" ftype="tabular"/>\n+        <param name="struct" value="AMS_test_Struct.tsv" ftype="tabular"/>\n+      </section>\n+      <section name="algorithm">\n+        <param name="mass_error_value" value="5.0"/>\n+        <param name="mass_error_unit" value="ppm"/>\n+        <param name="ionization_mode" value="positive"/>\n+        <param name="isotopic_similarity" value="false"/>\n+        <param name="use_feature_adducts" value="false"/>\n+        <param name="keep_unidentified_masses" value="false"/>\n+        <section name="mzTab">\n+          <param name="exportIsotopeIntensities" value="true"/>\n+        </section>\n+      </section>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_annotation_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test>\n+    <test expect_num_outputs="3">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="AccurateMassSearch_2_input.featureXML"/>\n+      <output name="out" file="AccurateMassSearch_3_output.tmp.mzTab" compare="sim_size" delta="5700" ftype="mztab"/>\n+      <output name="out_annotation" file="AccurateMassSearch_2_output.tmp.featureXML" compare="sim_size" delta="5700" ftype="featurexml"/>\n+      <param name'..b'      <param name="minscore" value="0.0"/>\n+      <param name="binsize" value="0.1"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_idXML_FLAG,out_mzIdentML_FLAG,out_xquest_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test>\n+  </xml>\n+  <xml name="autotest_XMLValidator">\n+</xml>\n+  <xml name="autotest_XTandemAdapter">\n+    <test expect_num_outputs="2">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="spectra.mzML"/>\n+      <output name="out" file="XTandemAdapter_1_out.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+      <param name="database" value="proteins.fasta"/>\n+      <param name="default_config_file" value="CHEMISTRY/XTandem_default_input.xml"/>\n+      <param name="ignore_adapter_param" value="false"/>\n+      <param name="precursor_mass_tolerance" value="5.0"/>\n+      <param name="fragment_mass_tolerance" value="0.3"/>\n+      <param name="precursor_error_units" value="ppm"/>\n+      <param name="fragment_error_units" value="Da"/>\n+      <param name="max_precursor_charge" value="0"/>\n+      <param name="no_isotope_error" value="false"/>\n+      <param name="fixed_modifications" value=""/>\n+      <param name="variable_modifications" value="Oxidation (M)"/>\n+      <param name="minimum_fragment_mz" value="150.0"/>\n+      <param name="enzyme" value="Trypsin"/>\n+      <param name="missed_cleavages" value="1"/>\n+      <param name="semi_cleavage" value="false"/>\n+      <param name="output_results" value="all"/>\n+      <param name="max_valid_expect" value="0.1"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test>\n+    <test expect_num_outputs="2">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="spectra.mzML"/>\n+      <output name="out" file="XTandemAdapter_2_out.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+      <param name="database" value="proteins.fasta"/>\n+      <param name="default_config_file" value="CHEMISTRY/XTandem_default_input.xml"/>\n+      <param name="ignore_adapter_param" value="false"/>\n+      <param name="precursor_mass_tolerance" value="5.0"/>\n+      <param name="fragment_mass_tolerance" value="0.3"/>\n+      <param name="precursor_error_units" value="ppm"/>\n+      <param name="fragment_error_units" value="Da"/>\n+      <param name="max_precursor_charge" value="0"/>\n+      <param name="no_isotope_error" value="false"/>\n+      <param name="fixed_modifications" value=""/>\n+      <param name="variable_modifications" value="Oxidation (M)"/>\n+      <param name="minimum_fragment_mz" value="150.0"/>\n+      <param name="enzyme" value="Trypsin"/>\n+      <param name="missed_cleavages" value="1"/>\n+      <param name="semi_cleavage" value="false"/>\n+      <param name="output_results" value="valid"/>\n+      <param name="max_valid_expect" value="1e-14"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test>\n+  </xml>\n+<xml name="autotest_IDDecoyProbability"/><xml name="autotest_PTPredict"/><xml name="autotest_OpenSwathDIAPreScoring"/><xml name="autotest_Digestor"/><xml name="autotest_InclusionExclusionListCreator"/><xml name="autotest_InspectAdapter"/><xml name="autotest_DigestorMotif"/><xml name="autotest_RTPredict"/><xml name="autotest_ProteomicsLFQ"/></macros>\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c macros_discarded_auto.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/macros_discarded_auto.xml Wed Sep 09 19:55:24 2020 +0000
b
b'@@ -0,0 +1,378 @@\n+\n+<xml name="manutest_OpenSwathFileSplitter">\n+<test expect_num_outputs="2">\n+  <conditional name="adv_opts_cond">\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="force" value="false"/>\n+    <param name="test" value="true"/>\n+  </conditional>\n+  <param name="in" value="OpenSwathWorkflow_1_input.mzML"/>\n+  <output_collection name="adv_opts_cond.outputDirectory" count=""/>\n+  <output name="out_qc" file="OpenSwathFileSplitter_1.json" compare="sim_size" delta="5700" ftype="json"/>\n+  <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_qc_FLAG"/>\n+  <output name="ctd_out" ftype="xml">\n+    <assert_contents>\n+      <is_valid_xml/>\n+    </assert_contents>\n+  </output>\n+</test></xml>\n+<xml name="manutest_IDRipper">\n+<test expect_num_outputs="1">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="IDRipper_1_input.idXML"/>\n+      <output_collection name="out_path" count=""/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test><test expect_num_outputs="1">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="IDRipper_2_input.idXML"/>\n+      <output_collection name="out_path" count=""/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test><test expect_num_outputs="1">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="IDRipper_3_output.idXML"/>\n+      <output_collection name="out_path" count=""/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test></xml>\n+<xml name="manutest_MzMLSplitter">\n+<test expect_num_outputs="1">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="FileFilter_1_input.mzML"/>\n+      <output_collection name="out" count=""/>\n+      <param name="parts" value="2"/>\n+      <param name="size" value="0"/>\n+      <param name="unit" value="MB"/>\n+      <param name="no_chrom" value="false"/>\n+      <param name="no_spec" value="false"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test><test expect_num_outputs="1">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="FileFilter_1_input.mzML"/>\n+      <output_collection name="out" count=""/>\n+      <param name="parts" value="1"/>\n+      <param name="size" value="40"/>\n+      <param name="unit" value="KB"/>\n+      <param name="no_chrom" value="false"/>\n+      <param name="no_spec" value="false"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+  '..b'" value="2"/>\n+        <param name="precursor_tolerance" value="20.0"/>\n+        <param name="precursor_tolerance_units" value="ppm"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="MaRaClusterAdapter_1_in_1.mzML,MaRaClusterAdapter_1_in_2.mzML"/>\n+      <param name="id_in" value="MaRaClusterAdapter_1_in_3.idXML"/>\n+      <output name="out" file="MaRaClusterAdapter_2_out_1.tmp.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+      <param name="pcut" value="-10.0"/>\n+      <param name="min_cluster_size" value="1"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG,out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test></xml>\n+<xml name="manutest_NovorAdapter">\n+<test expect_num_outputs="2">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="in" value="NovorAdapter_in.mzML"/>\n+      <output name="out" file="NovorAdapter_1_out.idXML" compare="sim_size" delta="5700" ftype="idxml"/>\n+      <param name="enzyme" value="Trypsin"/>\n+      <param name="fragmentation" value="CID"/>\n+      <param name="massAnalyzer" value="Trap"/>\n+      <param name="fragment_mass_tolerance" value="0.5"/>\n+      <param name="precursor_mass_tolerance" value="15.0"/>\n+      <param name="precursor_error_units" value="ppm"/>\n+      <param name="variable_modifications" value="Acetyl (K)"/>\n+      <param name="fixed_modifications" value="Carbamidomethyl (C)"/>\n+      <param name="forbiddenResidues" value="I"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test></xml>\n+<xml name="manutest_SpectraSTSearchAdapter">\n+<test expect_num_outputs="2">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="use_isotopically_averaged_mass" value="false"/>\n+        <param name="use_all_charge_states" value="false"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="spectra_files" value="SpectrastAdapter_1_hack.mzML"/>\n+      <param name="output_files_type" value="pep.xml"/>\n+      <output_collection name="output_files" count="1"/>\n+      <param name="library_file" value="testLib.splib" ftype="splib"/>\n+      <param name="sequence_database_type" value="AA"/>\n+      <param name="precursor_mz_tolerance" value="3.0"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test><test expect_num_outputs="2">\n+      <conditional name="adv_opts_cond">\n+        <param name="adv_opts_selector" value="advanced"/>\n+        <param name="use_isotopically_averaged_mass" value="false"/>\n+        <param name="use_all_charge_states" value="false"/>\n+        <param name="force" value="false"/>\n+        <param name="test" value="true"/>\n+      </conditional>\n+      <param name="spectra_files" value="SpectrastAdapter_1_hack.mzML"/>\n+      <param name="output_files_type" value="tsv"/>\n+      <output_collection name="output_files" count="1"/>\n+      <param name="library_file" value="testLib.splib" ftype="splib"/>\n+      <param name="sequence_database_type" value="AA"/>\n+      <param name="precursor_mz_tolerance" value="3.0"/>\n+      <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/>\n+      <output name="ctd_out" ftype="xml">\n+        <assert_contents>\n+          <is_valid_xml/>\n+        </assert_contents>\n+      </output>\n+    </test></xml>\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c macros_test.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/macros_test.xml Wed Sep 09 19:55:24 2020 +0000
b
b'@@ -0,0 +1,549 @@\n+<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n+<macros>\n+    \n+<!-- a copy of a FileConverter test without the advanced options used\n+     in order to check if this works (all other tests enable advanced) -->\n+<xml name="manutest_FileConverter">\n+<test expect_num_outputs="1">\n+  <param name="in" value="FileConverter_1_input.mzData"/>\n+  <output name="out" file="FileConverter_1_output.mzML" compare="sim_size" delta="5700" ftype="mzml"/>\n+  <param name="out_type" value="mzML"/>\n+</test>\n+</xml>\n+<!-- tests contributed by the galaxyproteomics community -->\n+<xml name="manutest_ClusterMassTracesByPrecursor">\n+  <test expect_num_outputs="1"><!-- test with arbitarily chosen consensusXML -->\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="in_ms1" ftype="consensusxml" value="ConsensusMapNormalizer_input.consensusXML"/>\n+    <param name="in_swath" ftype="consensusxml" value="ConsensusMapNormalizer_input.consensusXML"/>\n+    <output name="out" ftype="mzml" value="ClusterMassTracesByPrecursor.mzml"/>\n+  </test>\n+</xml>\n+<xml name="manutest_ClusterMassTraces"> \n+  <test expect_num_outputs="1"><!-- test with arbitarily chosen consensusXML -->\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="in" ftype="consensusxml" value="ConsensusMapNormalizer_input.consensusXML"/>\n+    <output name="out" ftype="mzml" value="ClusterMassTraces.mzml"/>\n+  </test>\n+</xml>\n+<xml name="manutest_CVInspector">\n+  <!-- test with https://raw.githubusercontent.com/HUPO-PSI/mzIdentML/master/cv/XLMOD.obo listed here https://www.psidev.info/groups/controlled-vocabularies, mapping file from share/OpenMS/MAPPING/ms-mapping.xml, see currently not working, see: https://github.com/OpenMS/OpenMS/pull/4425 -->\n+  <test expect_num_outputs="1">\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="cv_files" ftype="obo" value="CHEMISTRY/XLMOD.obo"/>\n+    <param name="cv_names" value="XLMOD"/>\n+    <param name="mapping_file" value="MAPPING/ms-mapping.xml"/>\n+    <param name="OPTIONAL_OUTPUTS" value="html_FLAG"/>\n+    <output name="html" ftype="html" value="CVInspector.html"/>\n+  </test>\n+</xml>\n+<xml name="manutest_DeMeanderize">\n+  <test expect_num_outputs="1"><!-- test with the output of a MSsimulator output generated by a test below -->\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="in" ftype="mzml" value="MSsimulator_MALDI.mzml"/>\n+\t<output name="out" ftype="mzml" value="DeMeanderize.mzml"/>\n+  </test>\n+</xml>\n+<xml name="manutest_Digestor">\n+  <test expect_num_outputs="1"><!-- just using some random fasta, in contrast to DigestorMotif idXML/fasta output possible, testing for the non-default fasta option -->\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="in" ftype="fasta" value="random.fa"/>\n+    <output name="out" ftype="fasta" value="Digestor.fasta"/>\n+    <param name="out_type" value="fasta"/>\n+  </test>\n+</xml>\n+<xml name="manutest_EICExtractor">\n+  <test expect_num_outputs="1"><!-- just using some random test data -->\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="in" value="spectra.mzML"/>\n+    <param name="pos" ftype="edta" value="FileConverter_10_input.edta"/>\n+    <output name="out" ftype="csv" value="EICExtractor.csv" lines_diff="2"/>\n+  </test>\n+</xml>\n+<xml name="manutest_ERPairFinder">\n+  <!-- TODO -->\n+</xml>\n+<xml name="manutest_FeatureFinderIsotopeWavelet">\n+  <test expect_num_outputs="1"><!--just use the input of another FeatureFinder -->\n+    <param name="adv_opts_selector" value="advanced"/>\n+    <param name="adv_opts_cond|test" value="true"/>\n+    <param name="in" value="FeatureFin'..b'e="manutest_PTPredict"/><xml name="manutest_XTandemAdapter"/><xml name="manutest_FeatureFinderMetabo"/><xml name="manutest_CruxAdapter"/><xml name="manutest_OpenSwathConfidenceScoring"/><xml name="manutest_PrecursorIonSelector"/><xml name="manutest_ConsensusMapNormalizer"/><xml name="manutest_RTPredict"/><xml name="manutest_PercolatorAdapter"/><xml name="manutest_ProteinInference"/><xml name="manutest_OpenSwathDecoyGenerator"/><xml name="manutest_TextExporter"/><xml name="manutest_FeatureFinderCentroided"/><xml name="manutest_DecoyDatabase"/><xml name="manutest_SpectraFilterWindowMower"/><xml name="manutest_NoiseFilterGaussian"/><xml name="manutest_MaRaClusterAdapter"/><xml name="manutest_ConsensusID"/><xml name="manutest_FileFilter"/><xml name="manutest_InclusionExclusionListCreator"/><xml name="manutest_FeatureLinkerUnlabeledKD"/><xml name="manutest_NovorAdapter"/><xml name="manutest_OpenPepXL"/><xml name="manutest_SeedListGenerator"/><xml name="manutest_FalseDiscoveryRate"/><xml name="manutest_OpenPepXLLF"/><xml name="manutest_SpecLibSearcher"/><xml name="manutest_IDConflictResolver"/><xml name="manutest_MRMMapper"/><xml name="manutest_OMSSAAdapter"/><xml name="manutest_ExternalCalibration"/><xml name="manutest_PeakPickerHiRes"/><xml name="manutest_MascotAdapter"/><xml name="manutest_FeatureFinderMultiplex"/><xml name="manutest_MetaProSIP"/><xml name="manutest_FuzzyDiff"/><xml name="manutest_TargetedFileConverter"/><xml name="manutest_MapAlignerIdentification"/><xml name="manutest_IDRTCalibration"/><xml name="manutest_MRMTransitionGroupPicker"/><xml name="manutest_RTModel"/><xml name="manutest_OpenSwathAssayGenerator"/><xml name="manutest_IDSplitter"/><xml name="manutest_OpenSwathMzMLFileCacher"/><xml name="manutest_HighResPrecursorMassCorrector"/><xml name="manutest_OpenSwathChromatogramExtractor"/><xml name="manutest_OpenSwathAnalyzer"/><xml name="manutest_FeatureLinkerUnlabeled"/><xml name="manutest_CompNovoCID"/><xml name="manutest_DTAExtractor"/><xml name="manutest_FileInfo"/><xml name="manutest_MapAlignerSpectrum"/><xml name="manutest_IsobaricAnalyzer"/><xml name="manutest_LowMemPeakPickerHiRes"/><xml name="manutest_MyriMatchAdapter"/><xml name="manutest_InternalCalibration"/><xml name="manutest_TOFCalibration"/><xml name="manutest_MSGFPlusAdapter"/><xml name="manutest_PTModel"/><xml name="manutest_InspectAdapter"/><xml name="manutest_SpectraFilterSqrtMower"/><xml name="manutest_SimpleSearchEngine"/><xml name="manutest_PeakPickerWavelet"/><xml name="manutest_MapNormalizer"/><xml name="manutest_IDScoreSwitcher"/><xml name="manutest_FeatureLinkerLabeled"/><xml name="manutest_MapRTTransformer"/><xml name="manutest_RNPxlSearch"/><xml name="manutest_PrecursorMassCorrector"/><xml name="manutest_MapAlignerPoseClustering"/><xml name="manutest_MzTabExporter"/><xml name="manutest_BaselineFilter"/><xml name="manutest_FeatureFinderMRM"/><xml name="manutest_MascotAdapterOnline"/><xml name="manutest_DatabaseFilter"/><xml name="manutest_ProteinResolver"/><xml name="manutest_OpenSwathFeatureXMLToTSV"/><xml name="manutest_FidoAdapter"/><xml name="manutest_AccurateMassSearch"/><xml name="manutest_CompNovo"/><xml name="manutest_LowMemPeakPickerHiResRandomAccess"/><xml name="manutest_OpenSwathRTNormalizer"/><xml name="manutest_PeptideIndexer"/><xml name="manutest_CometAdapter"/><xml name="manutest_NoiseFilterSGolay"/><xml name="manutest_MSFraggerAdapter"/><xml name="manutest_SpectraSTSearchAdapter"/><xml name="manutest_SequestAdapter"/><xml name="manutest_FeatureFinder"/><xml name="manutest_LuciphorAdapter"/><xml name="manutest_GNPSExport"/><xml name="manutest_Epifany"/><xml name="manutest_NucleicAcidSearchEngine"/><xml name="manutest_QualityControl"/><xml name="manutest_FeatureFinderMetaboIdent"/><xml name="manutest_RNAMassCalculator"/><xml name="manutest_MapAlignerTreeGuided"/><xml name="manutest_IDMassAccuracy"/><xml name="manutest_ProteomicsLFQ"/><xml name="manutest_IDDecoyProbability"/><xml name="manutest_DigestorMotif"/></macros>\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c prepare_test_data.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/prepare_test_data.sh Wed Sep 09 19:55:24 2020 +0000
[
b'@@ -0,0 +1,1196 @@\n+export COMET_BINARY="comet"\n+export CRUX_BINARY="crux"\n+export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"\n+export FIDO_BINARY="Fido"\n+export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"\n+export MARACLUSTER_BINARY="/tmp/openms-stuff//OpenMS2.5.0-git/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"\n+export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"\n+export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"\n+export MYRIMATCH_BINARY="myrimatch"\n+export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"\n+export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"\n+export PERCOLATOR_BINARY="percolator"\n+export SIRIUS_BINARY="$(which sirius)"\n+export SPECTRAST_BINARY="spectrast"\n+export XTANDEM_BINARY="xtandem"\n+export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"\n+FuzzyDiff -test -ini FuzzyDiff.ini -in1 FuzzyDiff_3_in1.featureXML -in2 FuzzyDiff_3_in2.featureXML > UTILS_FuzzyDiff_3.stdout 2> UTILS_FuzzyDiff_3.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'UTILS_FuzzyDiff_3 failed\'; >&2 echo -e "stderr:\\n$(cat UTILS_FuzzyDiff_3.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat UTILS_FuzzyDiff_3.stdout)";fi\n+IDMerger -test -in IDMerger_1_input1.idXML IDMerger_1_input2.idXML -out IDMerger_1_output.tmp -annotate_file_origin > TOPP_IDMerger_1.stdout 2> TOPP_IDMerger_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_1.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_1.stdout)";fi\n+IDMerger -test -pepxml_protxml -in IDMerger_2_input1.idXML IDMerger_2_input2.idXML -out IDMerger_2_output.tmp > TOPP_IDMerger_2.stdout 2> TOPP_IDMerger_2.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_2 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_2.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_2.stdout)";fi\n+IDMerger -test -in IDMerger_1_input1.idXML IDMerger_1_input1.idXML -out IDMerger_3_output.tmp > TOPP_IDMerger_3.stdout 2> TOPP_IDMerger_3.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_3 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_3.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_3.stdout)";fi\n+IDMerger -test -in degenerated_empty.idXML degenerated_empty.idXML -out IDMerger_4_output.tmp > TOPP_IDMerger_4.stdout 2> TOPP_IDMerger_4.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_4 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_4.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_4.stdout)";fi\n+IDMerger -test -in IDMerger_2_input1.idXML -add_to IDMerger_5_input1.idXML -out IDMerger_5_output.tmp > TOPP_IDMerger_5.stdout 2> TOPP_IDMerger_5.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_IDMerger_5 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_IDMerger_5.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_IDMerger_5.stdout)";fi\n+BaselineFilter -test -in BaselineFilter_input.mzML -out BaselineFilter.tmp -struc_elem_length 1.5 > TOPP_BaselineFilter_1.stdout 2> TOPP_BaselineFilter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_BaselineFilter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_BaselineFilter_1.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_BaselineFilter_1.stdout)";fi\n+ConsensusMapNormalizer -test -in ConsensusMapNormalizer_input.consensusXML -out ConsensusMapNormalizer.tmp > TOPP_ConsensusMapNormalizer_1.stdout 2> TOPP_ConsensusMapNormalizer_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_ConsensusMapNormalizer_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_ConsensusMapNormalizer_1.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_ConsensusMapNormalizer_1.stdout)";fi\n+MapNormalizer -test -in MapNormalizer_input.mzML -out MapNormalizer.tmp > TOPP_MapNormalizer_1.stdout 2> TOPP_MapNormalizer_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_MapNormalizer_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_MapNormalizer_1.stderr | sed \'s/^/'..b'S_AssayGeneratorMetabo_9 failed\'; >&2 echo -e "stderr:\\n$(cat UTILS_AssayGeneratorMetabo_9.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat UTILS_AssayGeneratorMetabo_9.stdout)";fi\n+AssayGeneratorMetabo -test -executable "${SIRIUS_BINARY}" -in AssayGeneratorMetabo_input.mzML -in_id AssayGeneratorMetabo_ams_input.featureXML -out AssayGeneratorMetabo_ams_sirius_restrict_output.tmp.tsv  -fragment_annotation sirius -use_exact_mass -transition_threshold 3.0 -min_transitions 2 -max_transitions 3 -min_fragment_mz 100 -max_fragment_mz 900 -preprocessing:filter_by_num_masstraces 1 -preprocessing:precursor_mz_tolerance 10 -preprocessing:precursor_mz_tolerance_unit ppm -preprocessing:feature_only -sirius:profile qtof -sirius:compound_timeout 100 > UTILS_AssayGeneratorMetabo_10.stdout 2> UTILS_AssayGeneratorMetabo_10.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'UTILS_AssayGeneratorMetabo_10 failed\'; >&2 echo -e "stderr:\\n$(cat UTILS_AssayGeneratorMetabo_10.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat UTILS_AssayGeneratorMetabo_10.stdout)";fi\n+SiriusAdapter -test -executable "${SIRIUS_BINARY}" -in SiriusAdapter_2_input.mzML -in_featureinfo SiriusAdapter_2_input.featureXML  -out_sirius SiriusAdapter_4_output.tmp -out_fingerid SiriusAdapter_4_foutput.tmp -sirius:auto_charge -sirius:profile qtof -sirius:database all > TOPP_SiriusAdapter_4.stdout 2> TOPP_SiriusAdapter_4.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SiriusAdapter_4 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SiriusAdapter_4.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_SiriusAdapter_4.stdout)";fi\n+NovorAdapter -test -java_memory 512 -executable "${NOVOR_BINARY}" -in NovorAdapter_in.mzML -out NovorAdapter_1_out.tmp -variable_modifications "Acetyl (K)" -fixed_modifications "Carbamidomethyl (C)" -forbiddenResidues "I" > TOPP_NovorAdapter_1.stdout 2> TOPP_NovorAdapter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_NovorAdapter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_NovorAdapter_1.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_NovorAdapter_1.stdout)";fi\n+FileConverter -test -force_TPP_compatibility -in spectra_spectrast.mzXML -out SpectrastAdapter_1_hack.mzML > TOPP_SpectrastSearchAdapter_0_prepare.stdout 2> TOPP_SpectrastSearchAdapter_0_prepare.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SpectrastSearchAdapter_0_prepare failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SpectrastSearchAdapter_0_prepare.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_SpectrastSearchAdapter_0_prepare.stdout)";fi\n+SpectraSTSearchAdapter -test -library_file testLib.splib -spectra_files SpectrastAdapter_1_hack.mzML -output_files SpectrastAdapter_1_out1.tmp.pep.xml -executable "${SPECTRAST_BINARY}" > TOPP_SpectrastSearchAdapter_1.stdout 2> TOPP_SpectrastSearchAdapter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SpectrastSearchAdapter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SpectrastSearchAdapter_1.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_SpectrastSearchAdapter_1.stdout)";fi\n+SpectraSTSearchAdapter -test -library_file testLib.splib -spectra_files SpectrastAdapter_1_hack.mzML -output_files SpectrastAdapter_1_out1.tmp.pep.tsv -executable "${SPECTRAST_BINARY}" > TOPP_SpectrastSearchAdapter_2.stdout 2> TOPP_SpectrastSearchAdapter_2.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_SpectrastSearchAdapter_2 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_SpectrastSearchAdapter_2.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_SpectrastSearchAdapter_2.stdout)";fi\n+LuciphorAdapter -test -in LuciphorAdapter_1_input.mzML  -java_memory 1024 -id LuciphorAdapter_1_input.idXML -out LuciphorAdapter_1_output.tmp  -executable "${LUCIPHOR_BINARY}" -min_num_psms_model 1 > TOPP_LuciphorAdapter_1.stdout 2> TOPP_LuciphorAdapter_1.stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TOPP_LuciphorAdapter_1 failed\'; >&2 echo -e "stderr:\\n$(cat TOPP_LuciphorAdapter_1.stderr | sed \'s/^/    /\')"; echo -e "stdout:\\n$(cat TOPP_LuciphorAdapter_1.stdout)";fi\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c prepare_test_data_manual.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/prepare_test_data_manual.sh Wed Sep 09 19:55:24 2020 +0000
[
b'@@ -0,0 +1,155 @@\n+MSSimulator -test -in DecoyDatabase_1.fasta -out MSsimulator.mzml -algorithm:RandomNumberGenerators:biological reproducible -algorithm:RandomNumberGenerators:technical reproducible > MSSimulator_1.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MSSimulator_1 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+MSSimulator -test -in DecoyDatabase_1.fasta -out MSsimulator_MALDI.mzml -algorithm:RandomNumberGenerators:biological reproducible -algorithm:RandomNumberGenerators:technical reproducible -algorithm:MSSim:Global:ionization_type MALDI > MSSimulator_2.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MSSimulator_2 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+ClusterMassTracesByPrecursor -test -in_ms1 ConsensusMapNormalizer_input.consensusXML -in_swath ConsensusMapNormalizer_input.consensusXML -out ClusterMassTracesByPrecursor.mzml > ClusterMassTracesByPrecursor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'ClusterMassTracesByPrecursor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+ClusterMassTraces -test -in ConsensusMapNormalizer_input.consensusXML -out ClusterMassTraces.mzml > ClusterMassTraces.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'ClusterMassTraces failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+CVInspector -test -cv_files CHEMISTRY/XLMOD.obo -cv_names XLMOD -mapping_file MAPPING/ms-mapping.xml -html CVInspector.html > CVInspector.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'CVInspector failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+DeMeanderize -test -in MSsimulator_MALDI.mzml -out DeMeanderize.mzml > DeMeanderize.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'DeMeanderize failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+# TODO DigestorMotif\n+\n+Digestor -test -in random.fa -out Digestor.fasta -out_type fasta > Digestor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'Digestor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+EICExtractor -test -in spectra.mzML -pos FileConverter_10_input.edta -out EICExtractor.csv > EICExtractor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'EICExtractor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+#TODO ERPairFinder\n+\n+FeatureFinderIsotopeWavelet -test -in FeatureFinderCentroided_1_input.mzML -out  FeatureFinderIsotopeWavelet.featureXML > FeatureFinderIsotopeWavelet.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'FeatureFinderIsotopeWavelet failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+\n+FFEval -test -in  FeatureFinderCentroided_1_output.featureXML -truth  FeatureFinderCentroided_1_output.featureXML -out  FFEval.featureXML -out_roc FFEval_roc.csv  > FFEval.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'FFEval failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+# TODO? deprecated IDDecoyProbability\n+\n+IDExtractor -test -in MSGFPlusAdapter_1_out.idXML -best_hits -number_of_peptides  1 -out  IDExtractor.idXML   > IDExtractor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'IDExtractor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+LabeledEval -test -in  FeatureLinkerLabeled_1_input.featureXML -truth  FeatureLinkerLabeled_1_output.consensusXML> LabeledEval.txt > LabeledEval.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'LabeledEval failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+MapStatistics -test -in SiriusAdapter_3_input.featureXML -out MapStatistics.txt > MapStatistics_1.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MapStatistics_1 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+MapStatistics -test -in ConsensusXMLFile_1.consensusXML -out MapStatistics2.txt > MapStatistics_2.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MapStatistics_2 failed\'; '..b'\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+# TODO out should be tsv, but needs https://github.com/OpenMS/OpenMS/pull/4533\n+RTEvaluation -in PeptideIndexer_1.idXML -out RTEvaluation.csv > RTEvaluation.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'RTEvaluation failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+ln -fs RTEvaluation.csv RTEvaluation.tsv > .stdout 2> stderr\n+\n+SemanticValidator -test -in FileFilter_1_input.mzML -mapping_file MAPPING/ms-mapping.xml > SemanticValidator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SemanticValidator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+IDFilter -in PeptideIndexer_1.idXML -best:strict -out SequenceCoverageCalculator_1.idXML > IDFilter.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'IDFilter failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+SequenceCoverageCalculator -test -in_database  PeptideIndexer_1.fasta -in_peptides  SequenceCoverageCalculator_1.idXML  -out  SequenceCoverageCalculator.txt > SequenceCoverageCalculator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SequenceCoverageCalculator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+# TODO SpecLibCreator\n+\n+SpectraFilterBernNorm -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterBernNorm.mzML > SpectraFilterBernNorm.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterBernNorm failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraFilterMarkerMower -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterMarkerMower.mzML > SpectraFilterMarkerMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterMarkerMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraFilterNLargest -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterNLargest.mzML > SpectraFilterNLargest.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterNLargest failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraFilterNormalizer -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterNormalizer.mzML > SpectraFilterNormalizer.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterNormalizer failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraFilterParentPeakMower -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterParentPeakMower.mzML > SpectraFilterParentPeakMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterParentPeakMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraFilterScaler -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterScaler.mzML > SpectraFilterScaler.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterScaler failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraFilterThresholdMower -test -in  SpectraFilterSqrtMower_1_input.mzML -out  SpectraFilterThresholdMower.mzML > SpectraFilterThresholdMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterThresholdMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+SpectraMerger -test -in NovorAdapter_in.mzML -out SpectraMerger_1.mzML > SpectraMerger.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraMerger failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+# TODO SvmTheoreticalSpectrumGeneratorTrainer\n+\n+TransformationEvaluation -test -in FileInfo_16_input.trafoXML -out TransformationEvaluation.trafoXML > TransformationEvaluation.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'TransformationEvaluation failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n+\n+XMLValidator -test -in FileFilter_1_input.mzML > XMLValidator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'XMLValidator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/    /\')"; fi\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c readme.md
--- a/readme.md Fri May 17 09:58:11 2019 -0400
+++ b/readme.md Wed Sep 09 19:55:24 2020 +0000
[
b'@@ -8,175 +8,127 @@\n More informations are available at:\n \n  * https://github.com/OpenMS/OpenMS\n- * http://open-ms.sourceforge.net\n+ * https://www.openms.de/\n+\n+The wrappers for these tools and most of their tests are automatically\n+generated using the `generate.sh` script. The generation of the tools is\n+based on the CTDConverter (https://github.com/WorkflowConversion/CTDConverter)\n+which can be fine tuned via the `hardcoded_params.json` file. This file allows\n+to blacklist and hardcode parameters and to modify or set arbitrary\n+CTD/XML attributes.\n+\n+Note that, due to its size, the test data is excluded from this repository. In\n+order to generate the test data on call `test-data.sh`.\n+\n+Manual updates should only be done to\n+\n+- the `@GALAXY_VERSION@"` token in `macros.xml`\n+- and the manually contributed tests in `macros_test.xml` (The goal is that all\n+  tools that do not have an automatically generated test are covered here)\n+- the `hardcoded_params.json` files\n+\n+In a few cases patches may be acceptable.\n+\n+Installation\n+============\n+\n+The Galaxy OpenMS tools can be installed from the toolshed. While most tools\n+will work out of the box some need attention since requirements can not be\n+fulfilled via Conda:\n+\n+Not yet in Conda are:\n+\n+- SpectraST (http://tools.proteomecenter.org/wiki/index.php?title=SpectraST)\n+- MaRaCluster (https://github.com/statisticalbiotechnology/maracluster)\n+\n+Binaries for these tools can easily be obtained via: \n+\n+```\n+VERSION=....\n+git git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git OpenMS$VERSION.0-git\n+git submodule init OpenMS$VERSION.0-git\n+git submodule update OpenMS$VERSION.0-git\n+```\n+\n+They are located in `OpenMS$VERSION-git/THIRDPARTY/`. \n \n+Not in Conda due to licencing restrictions:\n+\n+- Mascot http://www.matrixscience.com/\n+- MSFragger https://github.com/Nesvilab/MSFragger\n+- Novor http://www.rapidnovor.org/novor\n+\n+There are multiple ways to enable the Galaxy tools to use these binaries. \n+\n+- Just copy them to the `bin` path within Galaxy\'s conda environment\n+- Put them in any other path that that is included in PATH\n+- Edit the corresponding tools: In the command line part search for the parameters `-executable`, `-maracluster_executable`, or `-mascot_directory` and edit them appropriately.\n+\n+Working\n+=======\n+\n+The tools work by:\n+\n+Preprocessing:\n+\n+- For each input / output data set parameter a directory is crated (named by\n+  the parameter)\n+- For input data set parameters the links to the actual location of the data\n+  sets are created\n+\n+Main:\n+\n+- The galaxy wrapper create two json config files: one containing the\n+  parameters and the values chosen by the user and the other the values of\n+  hardcoded parameters.\n+- With `OpenMSTool -write_ctd ./` a CTD (names OpenMSTool.ctd) file is\n+  generated that contains the default values.\n+- A call to `fill_ctd.py` fills in the values from the json config files into\n+  the CTD file\n+- The actual tool is called `OpenMSTool -ini OpenMSTool.ctd` and also all input\n+  and output parameters are given on the command line.\n+\n+Postprocessing:\n+\n+- output data sets are moved to the final locations\n+\n+Note: The reason for handling data sets on the command line (and not specifying\n+them in the CTD file) is mainly that all files in Galaxy have the extension\n+`.dat` and OpenMS tools require an appropriate extension. But this may change\n+in the future.\n \n Generating OpenMS wrappers\n ==========================\n \n- * install OpenMS (you can do this automatically through Conda)\n- * create a folder called CTD\n- * if you installed openms as a binary in a specific directory, execute the following command in the `openms/bin` directory:\n-    \n-    ```bash\n-    for binary in `ls`; do ./$binary -write_ctd /PATH/TO/YOUR/CTD; done;\n-    ```\n-    \n- * if there is no binary release (e.g. as with version 2.2), download and unpack the Conda package, find the `bin` folder and create a list of the tools as follow:\n- \n-    ```bash\n-    ls '..b'yrimatch\' wrappers/MyriMatchAdapter.xml\n-    sed -i \'13 a\\-omssa_executable omssa\' wrappers/OMSSAAdapter.xml\n-    sed -i \'13 a\\-xtandem_executable xtandem\' wrappers/XTandemAdapter.xml\n-    ```\n-    \n- * For some tools, additional work has to be done. In `MSGFPlusAdapter.xml` the following is needed in the command section at the beginning (check your file to know what to copy where):\n- \n-   ```\n-    <command><![CDATA[\n-\n-    ## check input file type\n-    #set $in_type = $param_in.ext\n+2. The CI testing framework of OpenMS contains command lines and test data \n+   (https://github.com/OpenMS/OpenMS/tree/develop/src/tests/topp). These tests\n+   are described in two CMake files.\n \n-    ## create the symlinks to set the proper file extension, since msgf uses them to choose how to handle the input files\n-    ln -s \'$param_in\' \'param_in.${in_type}\' &&\n-    ln -s \'$param_database\' param_database.fasta &&\n-    ## find location of the MSGFPlus.jar file of the msgf_plus conda package\n-    MSGF_JAR=\\$(msgf_plus -get_jar_path) &&\n+   - From these CMake files Galaxy tests are auto generated and stored in `macros_autotest.xml`\n+   - The command lines are stored in `prepare_test_data.sh` for regeneration of test data\n+\n+More details can be found in the comments of the shell script.\n+\n+Open problems\n+=============\n \n-    MSGFPlusAdapter\n-    -executable \\$MSGF_JAR\n-    #if $param_in:\n-      -in \'param_in.${in_type}\'\n-    #end if\n-    #if $param_out:\n-      -out $param_out\n-    #end if\n-    #if $param_mzid_out:\n-      -mzid_out $param_mzid_out\n-    #end if\n-    #if $param_database:\n-      -database param_database.fasta\n-    #end if\n-    \n-    [...]\n-    ]]>\n-    ```\n- \n- * In Xtandem Converter and probably in others:\n- \n-    ```\n-    #if str($param_missed_cleavages) != \'\':\n-    ```\n-    This is because integers needs to be compared as string otherwise `0` becomes `false`.\n- \n- * In `MetaProSIP.xml` add `R` as a requirement:\n- \n-   ```\n-   <expand macro="requirements">\n-       <requirement type="package" version="3.3.1">r-base</requirement>\n-   </expand>\n-   ```\n-   \n- * In `IDFileConverter.xml` the following is needed in the command section at the beginning (check your file to know what to copy where):\n- \n-   ```\n-    <command><![CDATA[\n-   \n-      ## check input file type\n-      #set $in_type = $param_in.ext\n+Some tools stall in CI testing using `--biocontainers` which is why the OpenMS\n+tools are currently listed in `.tt_biocontainer_skip`. This is\n \n-      ## create the symlinks to set the proper file extension, since IDFileConverter uses them to choose how to handle the input files\n-      ln -s \'$param_in\' \'param_in.${in_type}\' &&\n-\n-      IDFileConverter\n-\n-      #if $param_in:\n-        -in \'param_in.${in_type}\'\n-      #end if\n-\n-        [...]\n-        ]]>\n-    ```\n+- AssayGeneratorMetabo and SiriusAdapter (both depend on sirius)\n+- OMSSAAdapter\n \n- * In `IDFileConverter.xml` and `FileConverter.xml` add `auto_format="true"` to the output, e.g.:\n- \n-   - `<data name="param_out" auto_format="true"/>`\n-   - `<data name="param_out" metadata_source="param_in" auto_format="true"/>`\n-        \n- * To add an example test case to `DecoyDatabase.xml` add the following after the output section. If standard settings change you might have to adjust the options and/or the test files.\n- \n-    ```\n-       <tests>\n-        <test>\n-            <param name="param_in" value="DecoyDatabase_input.fasta"/>\n-            <output name="param_out" file="DecoyDatabase_output.fasta"/>\n-        </test>\n-    </tests>\n-    ```\n-    \n- * Additionally cause of lacking dependencies, the following adapters have been removed in `SKIP_TOOLS_FILES.txt` as well:\n-    * OMSSAAdapter\n-    * MyrimatchAdapter\n-    \n- * Additionally cause of a problematic parameter (-model_directory), the following adapter has been removed:\n-    * PepNovoAdapter\n-\n+Using `docker -t` seems to solve the problem (see\n+https://github.com/galaxyproject/galaxy/issues/10153).\n \n Licence (MIT)\n =============\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c test-data.sh
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data.sh Wed Sep 09 19:55:24 2020 +0000
[
b'@@ -0,0 +1,255 @@\n+#!/usr/bin/env bash\n+\n+VERSION=2.5\n+FILETYPES="filetypes.txt"\n+CONDAPKG="https://anaconda.org/bioconda/openms/2.5.0/download/linux-64/openms-2.5.0-h463af6b_1.tar.bz2"\n+\n+# import the magic\n+. ./generate-foo.sh\n+\n+# install conda\n+if [ -z "$tmp" ]; then\n+\ttmp=$(mktemp -d)\n+\tcreated="yes"\n+fi\n+\n+export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"\n+export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"\n+export OPENMSENV="$tmp/OpenMS$VERSION-env"\n+export CTDCONVERTER="$tmp/CTDConverter"\n+\n+\n+if type conda > /dev/null; then  \n+\ttrue\n+else\n+\twget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh\n+\tbash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"\n+\tsource "$tmp/miniconda/bin/activate"\n+fi\n+eval "$(conda shell.bash hook)"\n+\n+if [[ -z "$1" ]]; then\n+\tautotests="/dev/null"\n+else\n+\tautotests="$1"\n+fi\n+\n+###############################################################################\n+## get \n+## - conda environment (for executing the binaries) and \n+## - the git clone of OpenMS (for generating the tests)\n+###############################################################################\n+\n+echo "Clone OpenMS $VERSION sources"\n+if [[ ! -d $OPENMSGIT ]]; then\n+\tgit clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT\n+\tcd $OPENMSGIT\n+\tgit submodule init\n+\tgit submodule update\n+\tcd -\n+else\n+\tcd $OPENMSGIT\n+\t\tgit pull origin release/$VERSION.0\n+\tcd -\n+fi\n+\n+echo "Create OpenMS $VERSION conda env"\n+# TODO currently add lxml (needed by CTDConverter)\n+# TODO for some reason a to recent openjdk is used\n+if conda env list | grep "$OPENMSENV"; then\n+\ttrue\n+else\n+\tconda create -y --quiet --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults -p $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION openjdk=8.0.192 ctdopts=1.4 lxml\n+# chmod -R u-w $OPENMSENV \n+fi\n+###############################################################################\n+## get the \n+## - conda package (for easy access and listing of the OpenMS binaries), \n+###############################################################################\n+echo "Download OpenMS $VERSION package $CONDAPKG"\n+\n+if [[ ! -d $OPENMSPKG ]]; then\n+\tmkdir $OPENMSPKG\n+\twget -q -P $OPENMSPKG/ "$CONDAPKG"\n+\ttar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/\n+\trm $OPENMSPKG/"$(basename $CONDAPKG)"\n+fi\n+\n+###############################################################################\n+## Get python libaries for CTD -> Galaxy conversion\n+## TODO fix to main repo OR conda packkage if PRs are merged \n+###############################################################################\n+echo "Clone CTDConverter"\n+if [[ ! -d $CTDCONVERTER ]]; then\n+\t#git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter\n+\tgit clone -b topic/cdata https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER\n+else\n+\tcd $CTDCONVERTER\n+\tgit pull origin topic/cdata\n+\tcd -\n+fi\n+export PYTHONPATH=$(pwd)/CTDopts\n+\n+###############################################################################\n+## copy all the test data files to test-data\n+## most of it (outputs) will be overwritten later, but its needed for\n+## prepare_test_data\n+###############################################################################\n+echo "Get test data"\n+cp $(find $OPENMSGIT/src/tests/topp/ -type f | grep -Ev "third_party_tests.cmake|CMakeLists.txt|check_ini") test-data/\n+cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/\n+cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/\n+cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/\n+if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then \n+\twget -q https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML && mv MetaboliteSpectralDB.mzML test-data/\n+fi\n+ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt\n+ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv\n+\n+if [ ! -d test-data/pepno'..b'e results for the tests and run it\n+###############################################################################\n+echo "Create test shell script"\n+\n+echo -n "" > prepare_test_data.sh\n+echo \'export COMET_BINARY="comet"\' >> prepare_test_data.sh\n+echo \'export CRUX_BINARY="crux"\' >> prepare_test_data.sh\n+echo \'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"\' >> prepare_test_data.sh\n+echo \'export FIDO_BINARY="Fido"\' >> prepare_test_data.sh\n+echo \'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"\' >> prepare_test_data.sh\n+\n+echo \'export MARACLUSTER_BINARY="\'"$OPENMSGIT"\'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"\'>> prepare_test_data.sh\n+echo \'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-20171106/MSFragger-20171106.jar"\'>> prepare_test_data.sh\n+echo \'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"\' >> prepare_test_data.sh\n+echo \'export MYRIMATCH_BINARY="myrimatch"\'>> prepare_test_data.sh\n+echo \'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"\' >> prepare_test_data.sh\n+echo \'export OMSSA_BINARY="$(dirname $(realpath $(which omssacl)))/omssacl"\'>> prepare_test_data.sh\n+echo \'export PERCOLATOR_BINARY="percolator"\'>> prepare_test_data.sh\n+echo \'export SIRIUS_BINARY="$(which sirius)"\' >> prepare_test_data.sh\n+echo \'export SPECTRAST_BINARY="spectrast"\' >> prepare_test_data.sh\n+echo \'export XTANDEM_BINARY="xtandem"\' >> prepare_test_data.sh\n+echo \'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"\' >> prepare_test_data.sh\n+\n+prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh\n+\n+# prepare_test_data > tmp_test_data.sh\n+# # remove calls not needed for the tools listed in any .list file\n+# echo LIST $LIST\n+# if [ ! -z "$LIST" ]; then\n+# \tREX=$(echo $LIST | sed \'s/ /\\n/g\' | sed \'s@.*/\\([^/]\\+\\).xml$@\\1@\' | tr \'\\n\' \'|\' | sed \'s/|$//\')\n+# else\n+# \tREX=".*"\n+# fi\n+# echo REX $REX\n+# cat tmp_test_data.sh | egrep "($REX)" >> prepare_test_data.sh\n+# rm tmp_test_data.sh\n+\n+echo "Execute test shell script"\n+chmod u+x prepare_test_data.sh\n+cd ./test-data || exit\n+../prepare_test_data.sh\n+cd - || exit\n+\n+\n+###############################################################################\n+## create/update test data for the manually generated tests\n+## - run convert once with the manual tests only and \n+## - update test-data (needs to run 2x)\n+###############################################################################\n+echo "Execute test shell script for manually curated tests"\n+chmod u+x prepare_test_data_manual.sh\n+\n+cd ./test-data || exit\n+../prepare_test_data_manual.sh\n+cd - || exit\n+\n+\n+# # # exit\n+\n+###############################################################################\n+## auto generate tests\n+###############################################################################\n+\n+echo "Write test macros to "$autotests\n+echo "<macros>" > $autotests\n+for i in $(ls *xml |grep -v macros)\n+do\n+\tb=$(basename "$i" .xml)\n+\tget_tests2 "$b" >> $autotests \n+done\n+echo "</macros>" >> $autotests\n+\n+echo "Create test data links"\n+link_tmp_files\n+\n+# tests for tools using output_prefix parameters can not be auto generated\n+# hence we output the tests for manual curation in macros_test.xml\n+# and remove them from the autotests\n+if [[ ! -z "$1" ]]; then\n+\techo "" > macros_discarded_auto.xml\n+\tfor i in OpenSwathFileSplitter IDRipper MzMLSplitter\n+\tdo\n+\t\techo "<xml name=\\"manutest_$i\\">" >>  macros_discarded_auto.xml\n+\t\txmlstarlet sel -t -c "/macros/xml[@name=\'autotest_$i\']/test" macros_autotest.xml >>  macros_discarded_auto.xml\n+\t\techo "</xml>"  >>  macros_discarded_auto.xml\n+\t\txmlstarlet ed -d "/macros/xml[@name=\'autotest_$i\']/test" macros_autotest.xml > tmp\n+\t\tmv tmp macros_autotest.xml\n+\tdone\n+\t>&2 "discarded autogenerated macros for curation in macros_discarded_auto.xml"\n+fi\n+conda deactivate\n+\n+## remove broken symlinks in test-data\n+find test-data/ -xtype l -delete\n+\n+# if [ ! -z "$created" ]; then\n+# \techo "Removing temporary directory"\n+# \trm -rf "$tmp"\n+# fi\n'
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c tool-data/pepnovo_models.loc.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/pepnovo_models.loc.sample Wed Sep 09 19:55:24 2020 +0000
b
@@ -0,0 +1,23 @@
+# This is a sample file distributed with Galaxy that enables tools
+# to use pepnovo models
+# The file has three tab separated columns: name, value, and path.
+# The idea is that there are a number of models in a directory:
+# - each model directory has a unique name (columns 2 and 0)
+# - each model can contain a set of models (column 1)
+#
+# The following example works fo the default models from 
+# http://proteomics.ucsd.edu/Software/PepNovo.html (just remove the comment
+# chars and replace DIR_TO_PEPNOVO_MODELS)
+
+#default_models CID_IT_TRYP DIR_TO_PEPNOVO_MODELS
+#default_models LTQ_COMP DIR_TO_PEPNOVO_MODELS
+#default_models DBC4_PEAK DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_TAG5 DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_TAG6 DIR_TO_PEPNOVO_MODELS
+#default_models ITDNV_PEAK DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_SCORE DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_TAG3 DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_DNVPART DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_TAG4 DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_DB DIR_TO_PEPNOVO_MODELS
+#default_models CID_IT_TRYP_CSP DIR_TO_PEPNOVO_MODELS
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c tool.conf
--- a/tool.conf Fri May 17 09:58:11 2019 -0400
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
b
@@ -1,160 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<toolbox>
-  <section id="section-id-Peptidepropertyprediction" name="Peptide property prediction">
-    <tool file="openms/PTModel.xml"/>
-    <tool file="openms/PTPredict.xml"/>
-    <tool file="openms/RTModel.xml"/>
-    <tool file="openms/RTPredict.xml"/>
-  </section>
-  <section id="section-id-TargetedExperiments" name="Targeted Experiments">
-    <tool file="openms/InclusionExclusionListCreator.xml"/>
-    <tool file="openms/MRMMapper.xml"/>
-    <tool file="openms/OpenSwathAnalyzer.xml"/>
-    <tool file="openms/OpenSwathAssayGenerator.xml"/>
-    <tool file="openms/OpenSwathChromatogramExtractor.xml"/>
-    <tool file="openms/OpenSwathConfidenceScoring.xml"/>
-    <tool file="openms/OpenSwathDecoyGenerator.xml"/>
-    <tool file="openms/OpenSwathDIAPreScoring.xml"/>
-    <tool file="openms/OpenSwathFeatureXMLToTSV.xml"/>
-    <tool file="openms/OpenSwathFileSplitter.xml"/>
-    <tool file="openms/OpenSwathMzMLFileCacher.xml"/>
-    <tool file="openms/OpenSwathRewriteToFeatureXML.xml"/>
-    <tool file="openms/OpenSwathRTNormalizer.xml"/>
-    <tool file="openms/PrecursorIonSelector.xml"/>
-    <tool file="openms/TargetedFileConverter.xml"/>
-  </section>
-  <section id="section-id-Utilities" name="Utilities">
-    <tool file="openms/AccurateMassSearch.xml"/>
-    <tool file="openms/CVInspector.xml"/>
-    <tool file="openms/DatabaseFilter.xml"/>
-    <tool file="openms/DecoyDatabase.xml"/>
-    <tool file="openms/DeMeanderize.xml"/>
-    <tool file="openms/Digestor.xml"/>
-    <tool file="openms/DigestorMotif.xml"/>
-    <tool file="openms/ERPairFinder.xml"/>
-    <tool file="openms/FFEval.xml"/>
-    <tool file="openms/FuzzyDiff.xml"/>
-    <tool file="openms/IDDecoyProbability.xml"/>
-    <tool file="openms/IDExtractor.xml"/>
-    <tool file="openms/IDMassAccuracy.xml"/>
-    <tool file="openms/IDScoreSwitcher.xml"/>
-    <tool file="openms/IDSplitter.xml"/>
-    <tool file="openms/LabeledEval.xml"/>
-    <tool file="openms/LowMemPeakPickerHiRes.xml"/>
-    <tool file="openms/LowMemPeakPickerHiRes_RandomAccess.xml"/>
-    <tool file="openms/MassCalculator.xml"/>
-    <tool file="openms/MetaboliteSpectralMatcher.xml"/>
-    <tool file="openms/MetaProSIP.xml"/>
-    <tool file="openms/MRMPairFinder.xml"/>
-    <tool file="openms/MRMTransitionGroupPicker.xml"/>
-    <tool file="openms/MSSimulator.xml"/>
-    <tool file="openms/MultiplexResolver.xml"/>
-    <tool file="openms/MzMLSplitter.xml"/>
-    <tool file="openms/OpenSwathWorkflow.xml"/>
-    <tool file="openms/QCCalculator.xml"/>
-    <tool file="openms/QCEmbedder.xml"/>
-    <tool file="openms/QCExporter.xml"/>
-    <tool file="openms/QCExtractor.xml"/>
-    <tool file="openms/QCImporter.xml"/>
-    <tool file="openms/QCMerger.xml"/>
-    <tool file="openms/QCShrinker.xml"/>
-    <tool file="openms/RNPxl.xml"/>
-    <tool file="openms/RNPxlSearch.xml"/>
-    <tool file="openms/RNPxlXICFilter.xml"/>
-    <tool file="openms/RTEvaluation.xml"/>
-    <tool file="openms/SemanticValidator.xml"/>
-    <tool file="openms/SequenceCoverageCalculator.xml"/>
-    <tool file="openms/SimpleSearchEngine.xml"/>
-    <tool file="openms/SpecLibCreator.xml"/>
-    <tool file="openms/SpectraSTSearchAdapter.xml"/>
-    <tool file="openms/SvmTheoreticalSpectrumGeneratorTrainer.xml"/>
-    <tool file="openms/TICCalculator.xml"/>
-    <tool file="openms/TopPerc.xml"/>
-    <tool file="openms/TransformationEvaluation.xml"/>
-    <tool file="openms/XMLValidator.xml"/>
-  </section>
-  <section id="section-id-MapAlignment" name="Map Alignment">
-    <tool file="openms/ConsensusMapNormalizer.xml"/>
-    <tool file="openms/FeatureLinkerLabeled.xml"/>
-    <tool file="openms/FeatureLinkerUnlabeled.xml"/>
-    <tool file="openms/FeatureLinkerUnlabeledKD.xml"/>
-    <tool file="openms/FeatureLinkerUnlabeledQT.xml"/>
-    <tool file="openms/MapRTTransformer.xml"/>
-  </section>
-  <section id="section-id-IDProcessing" name="ID Processing">
-    <tool file="openms/ConsensusID.xml"/>
-    <tool file="openms/FalseDiscoveryRate.xml"/>
-    <tool file="openms/FidoAdapter.xml"/>
-    <tool file="openms/IDConflictResolver.xml"/>
-    <tool file="openms/IDFileConverter.xml"/>
-    <tool file="openms/IDFilter.xml"/>
-    <tool file="openms/IDMapper.xml"/>
-    <tool file="openms/IDPosteriorErrorProbability.xml"/>
-    <tool file="openms/IDRTCalibration.xml"/>
-    <tool file="openms/LuciphorAdapter.xml"/>
-    <tool file="openms/PeptideIndexer.xml"/>
-    <tool file="openms/PhosphoScoring.xml"/>
-  </section>
-  <section id="section-id-Signalprocessingandpreprocessing" name="Signal processing and preprocessing">
-    <tool file="openms/BaselineFilter.xml"/>
-    <tool file="openms/ExternalCalibration.xml"/>
-    <tool file="openms/HighResPrecursorMassCorrector.xml"/>
-    <tool file="openms/InternalCalibration.xml"/>
-    <tool file="openms/MapNormalizer.xml"/>
-    <tool file="openms/MassTraceExtractor.xml"/>
-    <tool file="openms/NoiseFilterGaussian.xml"/>
-    <tool file="openms/NoiseFilterSGolay.xml"/>
-    <tool file="openms/PeakPickerHiRes.xml"/>
-    <tool file="openms/PeakPickerIterative.xml"/>
-    <tool file="openms/PeakPickerWavelet.xml"/>
-    <tool file="openms/PrecursorMassCorrector.xml"/>
-    <tool file="openms/SpectraMerger.xml"/>
-    <tool file="openms/TOFCalibration.xml"/>
-  </section>
-  <section id="section-id-Identification" name="Identification">
-    <tool file="openms/CompNovoCID.xml"/>
-    <tool file="openms/CompNovo.xml"/>
-    <tool file="openms/InspectAdapter.xml"/>
-    <tool file="openms/MascotAdapter.xml"/>
-    <tool file="openms/MascotAdapterOnline.xml"/>
-    <tool file="openms/MSGFPlusAdapter.xml"/>
-    <tool file="openms/ProteinInference.xml"/>
-    <tool file="openms/SpectraFilterBernNorm.xml"/>
-    <tool file="openms/SpectraFilterMarkerMower.xml"/>
-    <tool file="openms/SpectraFilterNLargest.xml"/>
-    <tool file="openms/SpectraFilterNormalizer.xml"/>
-    <tool file="openms/SpectraFilterParentPeakMower.xml"/>
-    <tool file="openms/SpectraFilterScaler.xml"/>
-    <tool file="openms/SpectraFilterSqrtMower.xml"/>
-    <tool file="openms/SpectraFilterThresholdMower.xml"/>
-    <tool file="openms/SpectraFilterWindowMower.xml"/>
-    <tool file="openms/XTandemAdapter.xml"/>
-  </section>
-  <section id="section-id-FileHandling" name="File Handling">
-    <tool file="openms/DTAExtractor.xml"/>
-    <tool file="openms/FileConverter.xml"/>
-    <tool file="openms/FileFilter.xml"/>
-    <tool file="openms/FileInfo.xml"/>
-    <tool file="openms/FileMerger.xml"/>
-    <tool file="openms/IDMerger.xml"/>
-    <tool file="openms/IDRipper.xml"/>
-    <tool file="openms/MapStatistics.xml"/>
-    <tool file="openms/MzTabExporter.xml"/>
-    <tool file="openms/TextExporter.xml"/>
-  </section>
-  <section id="section-id-Quantitation" name="Quantitation">
-    <tool file="openms/AdditiveSeries.xml"/>
-    <tool file="openms/Decharger.xml"/>
-    <tool file="openms/EICExtractor.xml"/>
-    <tool file="openms/FeatureFinderCentroided.xml"/>
-    <tool file="openms/FeatureFinderIdentification.xml"/>
-    <tool file="openms/FeatureFinderIsotopeWavelet.xml"/>
-    <tool file="openms/FeatureFinderMetabo.xml"/>
-    <tool file="openms/FeatureFinderMRM.xml"/>
-    <tool file="openms/FeatureFinderMultiplex.xml"/>
-    <tool file="openms/FeatureFinderSuperHirn.xml"/>
-    <tool file="openms/IsobaricAnalyzer.xml"/>
-    <tool file="openms/ProteinQuantifier.xml"/>
-    <tool file="openms/ProteinResolver.xml"/>
-  </section>
-</toolbox>
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c tool_data_table_conf.xml.sample
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample Wed Sep 09 19:55:24 2020 +0000
b
@@ -0,0 +1,7 @@
+<tables>
+    <!-- Locations of pepnovo models -->
+    <table name="pepnovo_models" comment_char="#" allow_duplicate_entries="False">
+        <columns>name,value,path</columns>
+        <file path="tool-data/pepnovo_models.loc" />
+    </table>
+</tables>
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c tool_data_table_conf.xml.test
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.test Wed Sep 09 19:55:24 2020 +0000
b
@@ -0,0 +1,7 @@
+<tables>
+    <!-- Locations of all eggnog_mapper data -->
+    <table name="pepnovo_models" comment_char="#">
+        <columns>name,value,path</columns>
+        <file path="${__HERE__}/test-data/pepnovo_models.loc" />
+    </table>
+</tables>
b
diff -r 8b814eafce2c -r 0ab17cc5cf0c tools_blacklist.txt
--- a/tools_blacklist.txt Fri May 17 09:58:11 2019 -0400
+++ b/tools_blacklist.txt Wed Sep 09 19:55:24 2020 +0000
b
@@ -1,9 +1,24 @@
-OMSSAAdapter
-MyriMatchAdapter
-PepNovoAdapter
-SeedListGenerator
-SpecLibSearcher
-MapAlignerIdentification
-MapAlignerPoseClustering
-MapAlignerSpectrum
-MapAlignerRTTransformer
+# seems not possible for 2.5 https://github.com/OpenMS/OpenMS/issues/4426
+DigestorMotif
+
+# deprecated https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Documentation/release/latest/html/UTILS_IDDecoyProbability.html
+# https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Documentation/release/latest/html/a16242.html
+IDDecoyProbability
+
+# personal communication with author V. Bafna: 
+# "InsPect is no longer maintained as there are many better tools including MS-GF+"
+InspectAdapter
+
+# licence? see https://github.com/bioconda/bioconda-recipes/issues/18953
+#MSFraggerAdapter
+
+# seems currently unusable https://github.com/OpenMS/OpenMS/pull/4443
+OpenSwathDIAPreScoring
+
+# https://github.com/OpenMS/OpenMS/issues/4550#issuecomment-594065727
+ProteomicsLFQ
+
+# https://github.com/OpenMS/OpenMS/issues/4401
+InclusionExclusionListCreator
+RTPredict
+PTPredict