|
Commit message:
planemo upload for repository https://github.com/galaxyproteomics/tools-galaxyp/tree/master/tools/openms commit 5c080b1e2b99f1c88f4557e9fec8c45c9d23b906 |
|
added:
JSONExporter.xml fill_ctd.py get_tests.py macros.xml prepare_test_data_manual.sh readme.md test-data.sh test-data/pepnovo_models.loc test-data/random.fa test-data/random_RNA.fa tool-data/pepnovo_models.loc.sample tool_data_table_conf.xml.sample tool_data_table_conf.xml.test |
| b |
| diff -r 000000000000 -r f483ffdc7014 JSONExporter.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/JSONExporter.xml Fri Jun 14 21:32:42 2024 +0000 |
| [ |
| @@ -0,0 +1,101 @@ +<!--This is a configuration file for the integration of a tools into Galaxy (https://galaxyproject.org/). This file was automatically generated using CTDConverter.--> +<!--Proposed Tool Section: [[for Developers]]--> +<tool id="JSONExporter" name="JSONExporter" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="21.05"> + <description>Exports .oms (SQLite) files in JSON format</description> + <macros> + <token name="@EXECUTABLE@">JSONExporter</token> + <import>macros.xml</import> + </macros> + <expand macro="requirements"/> + <expand macro="stdio"/> + <command detect_errors="exit_code"><![CDATA[@QUOTE_FOO@ +@EXT_FOO@ +#import re + +## Preprocessing +mkdir in && +cp '$in' 'in/${re.sub("[^\w\-_]", "_", $in.element_identifier)}.$gxy2omsext($in.ext)' && +mkdir out && + +## Main program call + +set -o pipefail && +@EXECUTABLE@ -write_ctd ./ && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hardcoded_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +-in +'in/${re.sub("[^\w\-_]", "_", $in.element_identifier)}.$gxy2omsext($in.ext)' +-out +'out/output.${gxy2omsext("json")}' + +## Postprocessing +&& mv 'out/output.${gxy2omsext("json")}' '$out' +#if "ctd_out_FLAG" in $OPTIONAL_OUTPUTS + && mv '@EXECUTABLE@.ctd' '$ctd_out' +#end if]]></command> + <configfiles> + <inputs name="args_json" data_style="paths"/> + <configfile name="hardcoded_json"><![CDATA[{"log": "log.txt", "threads": "\${GALAXY_SLOTS:-1}", "no_progress": true}]]></configfile> + </configfiles> + <inputs> + <param argument="-in" type="data" format="sqlite" label="Input file" help=" select sqlite data sets(s)"/> + <expand macro="adv_opts_macro"> + <param argument="-force" type="boolean" truevalue="true" falsevalue="false" checked="false" label="Overrides tool-specific checks" help=""/> + <param argument="-test" type="hidden" value="False" label="Enables the test mode (needed for internal use only)" help="" optional="true"> + <expand macro="list_string_san" name="test"/> + </param> + </expand> + <param name="OPTIONAL_OUTPUTS" type="select" optional="true" multiple="true" label="Optional outputs"> + <option value="ctd_out_FLAG">Output used ctd (ini) configuration file</option> + </param> + </inputs> + <outputs> + <data name="out" label="${tool.name} on ${on_string}: out" format="json"/> + <data name="ctd_out" format="xml" label="${tool.name} on ${on_string}: ctd"> + <filter>OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS</filter> + </data> + </outputs> + <tests> + <!-- TOPP_JSONExporter_1 --> + <test expect_num_outputs="2"> + <section name="adv_opts"> + <param name="force" value="false"/> + <param name="test" value="true"/> + </section> + <param name="in" value="JSONExporter_protein.oms"/> + <output name="out" value="JSONExporter_1_output.json" compare="sim_size" delta_frac="0.7" ftype="json"/> + <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/> + <output name="ctd_out" ftype="xml"> + <assert_contents> + <is_valid_xml/> + </assert_contents> + </output> + <assert_stdout> + <has_text_matching expression="@EXECUTABLE@ took .* \(wall\), .* \(CPU\), .* \(system\), .* \(user\)(; Peak Memory Usage: 32 MB)?."/> + </assert_stdout> + </test> + <!-- TOPP_JSONExporter_2 --> + <test expect_num_outputs="2"> + <section name="adv_opts"> + <param name="force" value="false"/> + <param name="test" value="true"/> + </section> + <param name="in" value="JSONExporter_RNA.oms"/> + <output name="out" value="JSONExporter_2_output.json" compare="sim_size" delta_frac="0.7" ftype="json"/> + <param name="OPTIONAL_OUTPUTS" value="ctd_out_FLAG"/> + <output name="ctd_out" ftype="xml"> + <assert_contents> + <is_valid_xml/> + </assert_contents> + </output> + <assert_stdout> + <has_text_matching expression="@EXECUTABLE@ took .* \(wall\), .* \(CPU\), .* \(system\), .* \(user\)(; Peak Memory Usage: 32 MB)?."/> + </assert_stdout> + </test> + </tests> + <help><![CDATA[Exports .oms (SQLite) files in JSON format + + +For more information, visit https://openms.de/doxygen/release/3.1.0/html/TOPP_JSONExporter.html]]></help> + <expand macro="references"/> +</tool> |
| b |
| diff -r 000000000000 -r f483ffdc7014 fill_ctd.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/fill_ctd.py Fri Jun 14 21:32:42 2024 +0000 |
| [ |
| @@ -0,0 +1,197 @@ +import collections +import json +import operator +import os +import re +import subprocess +import sys +from functools import reduce # forward compatibility for Python 3 + +from CTDopts.CTDopts import ( + _Choices, + _InFile, + _Null, + _NumericRange, + CTDModel +) + + +def getFromDict(dataDict, mapList): + return reduce(operator.getitem, mapList, dataDict) + + +def setInDict(dataDict, mapList, value): + getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value + + +def mergeDicts(d, e): + """ + insert values from the dict e into dict d + no values of d are overwritten + """ + for k, v in e.items(): + if (k in d and isinstance(d[k], dict) and isinstance(e[k], collections.abc.Mapping)): + mergeDicts(d[k], e[k]) + elif k not in d: + d[k] = e[k] + else: + sys.stderr.write("fill_ctd.py: could not merge key %s for %s in %s" % (k, d, e)) + sys.exit(1) + + +def _json_object_hook_noenvlookup(d): + return _json_object_hook(d, envlookup=False) + + +def _json_object_hook(d, envlookup=True): + """ + wee helper to transform the json written by galaxy + while loading + - True/False (bool objects) -> "true"/"false" (lowercase string) + - data inputs with multiple and optional true give [None] if no file is given -> [] + - None -> "" (empty string) + - replace bash expressions (if envlookup is True): + - environment variables (need to consist capital letters and _) by their value + - expressions + """ + for k in d.keys(): + # if type(d[k]) is bool: + # d[k] = str(d[k]).lower() + # else + if type(d[k]) is list and len(d[k]) == 1 and d[k][0] is None: + d[k] = [] + elif d[k] is None: + d[k] = "" + elif envlookup and type(d[k]) is str and d[k].startswith("$"): + m = re.fullmatch(r"\$([A-Z_]+)", d[k]) + if m: + d[k] = os.environ.get(m.group(1), "") + continue + m = re.fullmatch(r"\$(\{[A-Z_]+):-(.*)\}", d[k]) + if m: + d[k] = os.environ.get(m.group(1), m.group(2)) + continue + + try: + p = subprocess.run("echo %s" % d[k], shell=True, check=True, stdout=subprocess.PIPE, encoding="utf8") + d[k] = p.stdout.strip() + except subprocess.CalledProcessError: + sys.stderr.write("fill_ctd error: Could not evaluate %s" % d[k]) + continue + return d + + +def qstring2list(qs): + """ + transform a space separated string that is quoted by " into a list + """ + lst = list() + qs = qs.split(" ") + quoted = False + for p in qs: + if p == "": + continue + if p.startswith('"') and p.endswith('"'): + lst.append(p[1:-1]) + elif p.startswith('"'): + quoted = True + lst.append(p[1:] + " ") + elif p.endswith('"'): + quoted = False + lst[-1] += p[:-1] + else: + if quoted: + lst[-1] += p + " " + else: + lst.append(p) + return lst + + +def fix_underscores(args): + if type(args) is dict: + for k in list(args.keys()): + v = args[k] + if type(v) is dict: + fix_underscores(args[k]) + if k.startswith("_"): + args[k[1:]] = v + del args[k] + elif type(args) is list: + for i, v in enumerate(args): + if type(v) is dict: + fix_underscores(args[i]) + + +input_ctd = sys.argv[1] + +# load user specified parameters from json +with open(sys.argv[2]) as fh: + args = json.load(fh, object_hook=_json_object_hook_noenvlookup) + +# load hardcoded parameters from json +with open(sys.argv[3]) as fh: + hc_args = json.load(fh, object_hook=_json_object_hook) + +# insert the hc_args into the args +mergeDicts(args, hc_args) + +# put the contents of the advanced options section into the main dict +if "adv_opts" in args: + args.update(args["adv_opts"]) + del args["adv_opts"] + +# IDMapper has in and spectra:in params, in is used in out as format_source", +# which does not work in Galaxy: https://github.com/galaxyproject/galaxy/pull/9493" +# therefore hardcoded params change the name of spectra:in to spectra:_in +# which is corrected here again +# TODO remove once PR is in and adapt profile accordingly +fix_underscores(args) + +model = CTDModel(from_file=input_ctd) + +# transform values from json that correspond to +# - old style booleans (string + restrictions) -> transformed to a str +# - new style booleans that get a string (happens for hidden parameters [-test]) +# are transformed to a bool +# - unrestricted ITEMLIST which are represented as strings +# ("=quoted and space separated) in Galaxy -> transform to lists +# - optional data input parameters that have defaults and for which no +# value is given -> overwritte with the default +for p in model.get_parameters(): + + # check if the parameter is in the arguments from the galaxy tool + # (from the json file(s)), since advanced parameters are absent + # if the conditional is set to basic parameters + try: + getFromDict(args, p.get_lineage(name_only=True)) + except KeyError: + # few tools use dashes in parameters which are automatically replaced + # by underscores by Galaxy. in these cases the dictionary needs to be + # updated (better: then dash and the underscore variant are in the dict) + # TODO might be removed later https://github.com/OpenMS/OpenMS/pull/4529 + try: + lineage = [_.replace("-", "_") for _ in p.get_lineage(name_only=True)] + val = getFromDict(args, lineage) + except KeyError: + continue + else: + setInDict(args, p.get_lineage(name_only=True), val) + + if p.type is str and type(p.restrictions) is _Choices and set(p.restrictions.choices) == set(["true", "false"]): + v = getFromDict(args, p.get_lineage(name_only=True)) + setInDict(args, p.get_lineage(name_only=True), str(v).lower()) + elif p.type is bool: + v = getFromDict(args, p.get_lineage(name_only=True)) + if isinstance(v, str): + v = (v.lower() == "true") + setInDict(args, p.get_lineage(name_only=True), v) + elif p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange): + v = getFromDict(args, p.get_lineage(name_only=True)) + if type(v) is str: + setInDict(args, p.get_lineage(name_only=True), qstring2list(v)) + elif p.type is _InFile and not (p.default is None or type(p.default) is _Null): + v = getFromDict(args, p.get_lineage(name_only=True)) + if v in [[], ""]: + setInDict(args, p.get_lineage(name_only=True), p.default) + +model.write_ctd(input_ctd, arg_dict=args) |
| b |
| diff -r 000000000000 -r f483ffdc7014 get_tests.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/get_tests.py Fri Jun 14 21:32:42 2024 +0000 |
| [ |
| b'@@ -0,0 +1,344 @@\n+#!/usr/bin/env python\n+\n+import argparse\n+import os.path\n+import re\n+import shlex\n+import sys\n+import tempfile\n+from typing import (\n+ Dict,\n+ List,\n+ Optional,\n+ TextIO,\n+ Tuple,\n+)\n+\n+from ctdconverter.common.utils import (\n+ ParameterHardcoder,\n+ parse_hardcoded_parameters,\n+ parse_input_ctds,\n+)\n+from ctdconverter.galaxy.converter import convert_models\n+from CTDopts.CTDopts import (\n+ CTDModel,\n+ ModelTypeError,\n+ Parameters,\n+)\n+\n+SKIP_LIST = [\n+ r"_prepare\\"",\n+ r"_convert",\n+ r"WRITEINI",\n+ r"WRITECTD",\n+ r"INVALIDVALUE",\n+ r"\\.ini\\.json",\n+ r"OpenSwathMzMLFileCacher .*-convert_back", # - OpenSwathMzMLFileCacher with -convert_back argument https://github.com/OpenMS/OpenMS/issues/4399\n+ r"MaRaClusterAdapter.*-consensus_out", # - MaRaCluster with -consensus_out (parameter blacklister: https://github.com/OpenMS/OpenMS/issues/4456)\n+ r"FileMerger_1_input1.dta2d.*FileMerger_1_input2.dta ", # - FileMerger with mixed dta dta2d input (ftype can not be specified in the test, dta can not be sniffed)\n+ r\'^(TOPP_OpenSwathAnalyzer_test_3|TOPP_OpenSwathAnalyzer_test_4)$\', # no suppert for cached mzML\n+ r\'TOPP_SiriusAdapter_[0-9]+$\', # Do not test SiriusAdapter https://github.com/OpenMS/OpenMS/issues/7000 .. will be removed anyway\n+ r\'TOPP_AssayGeneratorMetabo_(7|8|9|10|11|12|13|14|15|16|17|18)$\' # Skip AssayGeneratorMetabo tests using Sirius https://github.com/OpenMS/OpenMS/issues/7150 (will be replaced by two tools)\n+]\n+\n+\n+def get_failing_tests(cmake: List[str]) -> List[str]:\n+ failing_tests = []\n+ re_fail = re.compile(r"set_tests_properties\\(\\"([^\\"]+)\\" PROPERTIES WILL_FAIL 1\\)")\n+\n+ for cmake in args.cmake:\n+ with open(cmake) as cmake_fh:\n+ for line in cmake_fh:\n+ match = re_fail.search(line)\n+ if match:\n+ failing_tests.append(match.group(1))\n+ return failing_tests\n+\n+\n+def fix_tmp_files(line: str, diff_pairs: Dict[str, str]) -> str:\n+ """\n+ OpenMS tests output to tmp files and compare with FuzzyDiff to the expected file.\n+ problem: the extension of the tmp files is unusable for test generation.\n+ unfortunately the extensions used in the DIFF lines are not always usable for the CLI\n+ (e.g. for prepare_test_data, e.g. CLI expects csv but test file is txt)\n+ this function replaces the tmp file by the expected file.\n+ """\n+ cmd = shlex.split(line)\n+ for i, e in enumerate(cmd):\n+ if e in diff_pairs:\n+ dst = os.path.join("test-data", diff_pairs[e])\n+ if os.path.exists(dst):\n+ os.unlink(dst)\n+ sys.stderr.write(f"symlink {e} {dst}\\n")\n+ os.symlink(e, dst)\n+ cmd[i] = diff_pairs[e]\n+ return shlex.join(cmd)\n+\n+\n+def get_ini(line: str, tool_id: str) -> Tuple[str, str]:\n+ """\n+ if there is an ini file then we use this to generate the test\n+ otherwise the ctd file is used\n+ other command line parameters are inserted later into this xml\n+ """\n+ cmd = shlex.split(line)\n+ ini = None\n+ for i, e in enumerate(cmd):\n+ if e == "-ini":\n+ ini = cmd[i + 1]\n+ cmd = cmd[:i] + cmd[i + 2:]\n+ if ini:\n+ return os.path.join("test-data", ini), shlex.join(cmd)\n+ else:\n+ return os.path.join("ctd", f"{tool_id}.ctd"), line\n+\n+\n+def unique_files(line: str):\n+ """\n+ some tests use the same file twice which does not work in planemo tests\n+ hence we create symlinks for each file used twice\n+ """\n+ cmd = shlex.split(line)\n+ # print(f"{cmd}")\n+ files = {}\n+ # determine the list of indexes where each file argument (anything appearing in test-data/) appears\n+ for idx, e in enumerate(cmd):\n+ p = os.path.join("test-data", e)\n+ if not os.path.exists(p) and not os.path.islink(p):\n+ continue\n+ try:\n+ files[e].append(idx)\n+ except KeyError:\n+ '..b'tr]] = field(default_factory=list)\n+ test_test: bool = False\n+ test_only: bool = False\n+ test_unsniffable: Optional[List[str]] = field(default_factory=list)\n+ test_condition: Optional[List[str]] = ("compare=sim_size", "delta_frac=0.05")\n+ tool_version: str = None\n+ tool_profile: str = None\n+ bump_file: str = None\n+\n+ # create an ini/ctd file where the values are equal to the arguments from the command line\n+ # and transform it to xml\n+ test = [f"<!-- {test_id} -->\\n"]\n+ with tempfile.NamedTemporaryFile(\n+ mode="w+", delete_on_close=False\n+ ) as ctd_tmp, tempfile.NamedTemporaryFile(\n+ mode="w+", delete_on_close=False\n+ ) as xml_tmp:\n+ fill_ctd_clargs(ini, line, ctd_tmp)\n+ ctd_tmp.close()\n+ xml_tmp.close()\n+ parsed_ctd = parse_input_ctds(None, [ctd_tmp.name], xml_tmp.name, "xml")\n+ ctd_args = CTDConverterArgs(\n+ input_files=[ctd_tmp.name],\n+ output_destination=xml_tmp.name,\n+ macros_files=["macros.xml"],\n+ skip_tools_file="aux/tools_blacklist.txt",\n+ formats_file="aux/filetypes.txt",\n+ # tool_conf_destination = "tool.conf",\n+ hardcoded_parameters="aux/hardcoded_params.json",\n+ tool_version="3.1",\n+ test_only=True,\n+ test_unsniffable=[\n+ "csv",\n+ "tsv",\n+ "txt",\n+ "dta",\n+ "dta2d",\n+ "edta",\n+ "mrm",\n+ "splib",\n+ ],\n+ test_condition=["compare=sim_size", "delta_frac=0.7"],\n+ )\n+ ctd_args.parameter_hardcoder = parse_hardcoded_parameters(\n+ ctd_args.hardcoded_parameters\n+ )\n+ convert_models(ctd_args, parsed_ctd)\n+ xml_tmp = open(xml_tmp.name, "r")\n+ for l in xml_tmp:\n+ test.append(l)\n+\n+ return "".join(test)\n+\n+\n+parser = argparse.ArgumentParser(description="Create Galaxy tests for a OpenMS tools")\n+parser.add_argument("--id", dest="id", help="tool id")\n+parser.add_argument("--cmake", nargs="+", help="OpenMS test CMake files")\n+args = parser.parse_args()\n+sys.stderr.write(f"generate tests for {args.id}\\n")\n+\n+re_comment = re.compile("#.*")\n+re_empty_prefix = re.compile(r"^\\s*")\n+re_empty_suffix = re.compile(r"\\s*$")\n+re_add_test = re.compile(r"add_test\\(\\"(TOPP|UTILS)_.*/" + args.id)\n+re_diff = re.compile(r"\\$\\{DIFF\\}.* -in1 ([^ ]+) -in2 ([^ ]+)")\n+failing_tests = get_failing_tests(args.cmake)\n+tests = []\n+\n+# process the given CMake files and compile lists of\n+# - test lines .. essentially add_test(...)\n+# - and pairs of files that are diffed\n+jline = ""\n+test_lines = []\n+diff_pairs = {}\n+for cmake in args.cmake:\n+ with open(cmake) as cmake_fh:\n+ for line in cmake_fh:\n+ # remove comments, empty prefixes and suffixes\n+ line = re_comment.sub("", line)\n+ line = re_empty_prefix.sub("", line)\n+ line = re_empty_suffix.sub("", line)\n+ # skip empty lines\n+ if line == "":\n+ continue\n+\n+ # join test statements that are split over multiple lines\n+ if line.endswith(")"):\n+ jline += " " + line[:-1]\n+ else:\n+ jline = line\n+ continue\n+ line, jline = jline.strip(), ""\n+ match = re_diff.search(line)\n+ if match:\n+ in1 = match.group(1).split("/")[-1]\n+ in2 = match.group(2).split("/")[-1]\n+ if in1 != in2:\n+ diff_pairs[in1] = in2\n+ elif re_add_test.match(line):\n+ test_lines.append(line)\n+\n+for line in test_lines:\n+ test = process_test_line(args.id, line, failing_tests, SKIP_LIST, diff_pairs)\n+ if test:\n+ tests.append(test)\n+\n+tests = "\\n".join(tests)\n+print(\n+ f"""\n+<xml name="autotest_{args.id}">\n+{tests}\n+</xml>\n+"""\n+)\n' |
| b |
| diff -r 000000000000 -r f483ffdc7014 macros.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros.xml Fri Jun 14 21:32:42 2024 +0000 |
| [ |
| @@ -0,0 +1,125 @@ +<?xml version='1.0' encoding='UTF-8'?> +<!-- CTD2Galaxy depends on this file and on the stdio, advanced_options macros! + You can edit this file to add your own macros, if you so desire, or you can + add additional macro files using the m/macros parameter --> +<macros> + <token name="@TOOL_VERSION@">3.1</token> + <token name="@VERSION_SUFFIX@">0</token> + <token name="@TEST_DATA_LOCATION@"/> + <xml name="requirements"> + <requirements> + <requirement type="package" version="@TOOL_VERSION@">openms</requirement> + <requirement type="package" version="@TOOL_VERSION@">openms-thirdparty</requirement> + <!-- omssa (which has been excluded from 3rdparty) and makeblastdb for OMSSAAdapter --> + <requirement type="package" version="2.14.1">blast</requirement> + <!--<requirement type="package" version="5.0.0">tpp</requirement>--> + <!-- for realpath (used e.g. in LuciphorAdapter) --> + <!--<requirement type="package" version="8.25">coreutils</requirement>--> + <requirement type="package" version="1.5">ctdopts</requirement> + <yield/> + </requirements> + </xml> + <xml name="stdio"> + <stdio> + <regex match="std::bad_alloc" level="fatal_oom" description="Could not allocate memory"/> + <regex match="OutOfMemoryError" level="fatal_oom" description="Could not allocate memory"/> + <regex match="Could not allocate metaspace" level="fatal_oom" description="Java memory Exception"/> + <regex match="Cannot create VM thread" level="fatal_oom" description="Java memory Exception"/> + <regex match="qUncompress: could not allocate enough memory to uncompress data" level="fatal_oom" description="Java memory Exception"/> + <regex match="OMSSA ran out of RAM" level="fatal_oom" description="Could not allocate memory"/> + <regex match="comet' crashed hard (segfault-like). Please check the log." level="fatal_oom" description="Could not allocate memory"/> + </stdio> + </xml> + <xml name="references"> + <citations> + <citation type="doi">doi:10.1186/1471-2105-9-163</citation> + </citations> + </xml> + <xml name="adv_opts_macro"> + <section name="adv_opts" title="Advanced Options" expanded="false"> + <yield/> + </section> + </xml> + + <!-- sanitizers and validators --> + <xml name="list_string_val" token_name=""> + <validator type="regex" message="parameter @NAME@: must not start with $">^[^$]</validator> + <validator type="regex" message="parameter @NAME@: a space separated list of string is needed (strings that contain spaces can be quoted with ")">^ *((?:\"[^\"]*\" +)|(?:[^ \"]+ +))*((?:\"[^\"]*\")|(?:[^ \"]+)) *$</validator> + </xml> + <xml name="list_string_san"> + <sanitizer> + <valid initial="string.printable"> +<!-- <remove value="'"/>--> +<!-- <remove value="\"/>--><!-- otherwise the user could quote the final quote --> +<!-- <remove value="`"/>--> +<!-- <remove value="$"/>--> +<!-- <remove value="<"/>--> + <!--<remove value="&"/> removed for MascotAdapterOnline -Mascot_server:export_params which is a URL POST string which can contain & .. could be mapped to & but there is still a & --> + <!--<remove value=";"/>--> +<!-- <remove value="#"/>--> + </valid> + </sanitizer> + </xml> + <xml name="list_float_valsan" token_name=""> + <validator type="regex" message="parameter @NAME@: a space separated list of float values is required">^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$</validator> + <yield/> + <sanitizer> + <valid initial="string.digits"> + <add value=" "/> + <add value="."/> + <add value="E"/> + <add value="e"/> + <add value="+"/> + <add value="-"/> + </valid> + </sanitizer> + </xml> + <xml name="list_integer_valsan" token_name=""> + <validator type="regex" message="parameter @NAME@: a space separated list of integer values is required">^ *[+-]?[0-9]+( *[+-]?[0-9]+)* *$</validator> + <yield/> + <sanitizer> + <valid initial="string.digits"> + <add value=" "/> + <add value="+"/> + <add value="-"/> + </valid> + </sanitizer> + </xml> + + <!-- helper function to quote space separated strings --> + <token name="@QUOTE_FOO@"> +#def quote(s): + #set $s = [ _ for _ in $s.split(" ") if _ != "" ] + #set $q = False + #for $i, $p in enumerate($s): + #if $p == "": + #continue + #end if + #if $p.startswith('"'): + #set $q = True + #end if +## #if p.startswith('-'): +## #set p = "\\" + p +## #elif p.startswith('"-'): +## #set p = "\\" + p[1:] +## #end if + #if not $q: + #set $s[i] = '"%s"' % p + #end if + #if $p.endswith('"'): + #set $q = False + #end if + #end for + #return " ".join($s) +#end def + </token> + +<token name="@EXT_FOO@"><![CDATA[#def oms2gxyext(o) + #set m={'txt': 'txt', 'tsv': 'tabular', 'bioml': 'xml', 'consensusXML': 'consensusxml', 'csv': 'csv', 'dta': 'dta', 'dta2d': 'dta2d', 'edta': 'edta', 'fa': 'fasta', 'fas': 'fasta', 'fasta': 'fasta', 'FASTA': 'fasta', 'featureXML': 'featurexml', 'featurexml': 'featurexml', 'html': 'html', 'HTML': 'html', 'idXML': 'idxml', 'json': 'json', 'kroenik': 'kroenik', 'mascotXML': 'mascotxml', 'mgf': 'mgf', 'mrm': 'mrm', 'ms': 'sirius.ms', 'ms2': 'ms2', 'msp': 'msp', 'mzData': 'mzdata', 'mzid': 'mzid', 'mzML': 'mzml', 'mzml': 'mzml', 'mzq': 'mzq', 'mzQC': 'mzqc', 'mzTab': 'mztab', 'mzXML': 'mzxml', 'novor': 'txt', 'obo': 'obo', 'oms': 'sqlite', 'omssaXML': 'idxml', 'osw': 'osw', 'OSW': 'osw', 'params': 'txt', 'paramXML': 'paramxml', 'peplist': 'peplist', 'pep.xml': 'pepxml', 'pepXML': 'pepxml', 'png': 'png', 'PNG': 'png', 'protXML': 'protxml', 'psms': 'psms', 'pqp': 'pqp', 'qcML': 'qcml', 'spec.xml': 'spec.xml', 'splib': 'splib', 'sqMass': 'sqmass', 'tandem.xml': 'tandem', 'trafoXML': 'trafoxml', 'traML': 'traml', 'TraML': 'traml', 'tab': 'tabular', 'raw': 'thermo.raw', 'xls': 'tsv', 'XML': 'xml', 'xml': 'xml', 'xquest.xml': 'xquest.xml', 'xsd': 'xsd', 'zip': 'zip'} + #return m[o] +#end def +#def gxy2omsext(g) + #set m={'txt': 'txt', 'tabular': 'tsv', 'xml': 'bioml', 'consensusxml': 'consensusXML', 'csv': 'csv', 'dta': 'dta', 'dta2d': 'dta2d', 'edta': 'edta', 'fasta': 'fa', 'featurexml': 'featureXML', 'html': 'html', 'idxml': 'idXML', 'json': 'json', 'kroenik': 'kroenik', 'mascotxml': 'mascotXML', 'mgf': 'mgf', 'mrm': 'mrm', 'sirius.ms': 'ms', 'ms2': 'ms2', 'msp': 'msp', 'mzdata': 'mzData', 'mzid': 'mzid', 'mzml': 'mzML', 'mzq': 'mzq', 'mzqc': 'mzQC', 'mztab': 'mzTab', 'mzxml': 'mzXML', 'obo': 'obo', 'sqlite': 'oms', 'osw': 'osw', 'paramxml': 'paramXML', 'peff': 'fasta', 'peplist': 'peplist', 'pepxml': 'pep.xml', 'png': 'png', 'protxml': 'protXML', 'psms': 'psms', 'pqp': 'pqp', 'qcml': 'qcML', 'spec.xml': 'spec.xml', 'splib': 'splib', 'sqmass': 'sqMass', 'tandem': 'tandem.xml', 'trafoxml': 'trafoXML', 'traml': 'traML', 'thermo.raw': 'raw', 'tsv': 'xls', 'xquest.xml': 'xquest.xml', 'xsd': 'xsd', 'zip': 'zip'} + #return m[g] +#end def +]]></token></macros> |
| b |
| diff -r 000000000000 -r f483ffdc7014 prepare_test_data_manual.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/prepare_test_data_manual.sh Fri Jun 14 21:32:42 2024 +0000 |
| [ |
| b'@@ -0,0 +1,127 @@\n+\n+ClusterMassTracesByPrecursor -test -in_ms1 ConsensusMapNormalizer_input.consensusXML -in_swath ConsensusMapNormalizer_input.consensusXML -out ClusterMassTracesByPrecursor.mzml > ClusterMassTracesByPrecursor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'ClusterMassTracesByPrecursor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+ClusterMassTraces -test -in ConsensusMapNormalizer_input.consensusXML -out ClusterMassTraces.mzml > ClusterMassTraces.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'ClusterMassTraces failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+CVInspector -test -cv_files CHEMISTRY/XLMOD.obo -cv_names XLMOD -mapping_file MAPPING/ms-mapping.xml -html CVInspector.html > CVInspector.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'CVInspector failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO DeMeanderize\n+\n+# TODO DigestorMotif\n+\n+Digestor -test -in random.fa -out Digestor.fasta -out_type fasta > Digestor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'Digestor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+EICExtractor -test -in spectra.mzML -pos FileConverter_10_input.edta -out EICExtractor.csv > EICExtractor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'EICExtractor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+#TODO ERPairFinder\n+\n+FeatureFinderIsotopeWavelet -test -in FeatureFinderCentroided_1_input.mzML -out FeatureFinderIsotopeWavelet.featureXML > FeatureFinderIsotopeWavelet.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'FeatureFinderIsotopeWavelet failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+\n+# TODO? deprecated IDDecoyProbability\n+\n+IDExtractor -test -in MSGFPlusAdapter_1_out.idXML -best_hits -number_of_peptides 1 -out IDExtractor.idXML > IDExtractor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'IDExtractor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+MapStatistics -test -in SiriusAdapter_3_input.featureXML -out MapStatistics.txt > MapStatistics_1.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MapStatistics_1 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+MapStatistics -test -in ConsensusXMLFile_1.consensusXML -out MapStatistics2.txt > MapStatistics_2.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MapStatistics_2 failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+MetaboliteSpectralMatcher -test -in spectra.mzML -database MetaboliteSpectralDB.mzML -out MetaboliteSpectralMatcher.mzTab > MetaboliteSpectralMatcher.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'MetaboliteSpectralMatcher failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO MRMPairFinder\n+\n+# generate two inputs for OpenSwathDIAPreScoring\n+OpenSwathDIAPreScoring -tr OpenSwathWorkflow_1_input.TraML -swath_files OpenSwathAnalyzer_2_swathfile.mzML -output_files OpenSwathDIAPreScoring.tsv > OpenSwathDIAPreScoring.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'OpenSwathDIAPreScoring failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# generate two inputs for OpenSwathDIAPreScoring by linking\n+ln -s OpenSwathAnalyzer_2_swathfile.mzML OpenSwathDIAPreScoring_in1.mzML\n+ln -s OpenSwathAnalyzer_2_swathfile.mzML OpenSwathDIAPreScoring_in2.mzML\n+OpenSwathDIAPreScoring -tr OpenSwathWorkflow_1_input.TraML -swath_files OpenSwathDIAPreScoring_in1.mzML OpenSwathDIAPreScoring_in2.mzML -output_files OpenSwathDIAPreScoring_2_1.tsv OpenSwathDIAPreScoring_2_2.tsv > OpenSwathDIAPreScoring.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'OpenSwathDIAPreScoring failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+OpenSwathRewriteToFeatureXML -featureXML OpenSwathFeatureXMLToTSV_input.featureXML -out OpenSwathRewriteToFeatureXML.featureXML > OpenSwathRewriteToFeatureXML.'..b'nker -test -in QCCalculator1.qcML -out QCShrinker.qcML > QCShrinker.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'QCShrinker failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+RNADigestor -test -in random_RNA.fa -out RNADigestor.fasta > RNADigestor.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'RNADigestor failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+RNPxlXICFilter -test -control FileFilter_1_input.mzML -treatment FileFilter_1_input.mzML -out RNPxlXICFilter.mzML > RNPxlXICFilter.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'RNPxlXICFilter failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SemanticValidator -test -in FileFilter_1_input.mzML -mapping_file MAPPING/ms-mapping.xml > SemanticValidator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SemanticValidator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+IDFilter -in PeptideIndexer_1.idXML -best:strict -out SequenceCoverageCalculator_1.idXML > IDFilter.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'IDFilter failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+SequenceCoverageCalculator -test -in_database PeptideIndexer_1.fasta -in_peptides SequenceCoverageCalculator_1.idXML -out SequenceCoverageCalculator.txt > SequenceCoverageCalculator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SequenceCoverageCalculator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+# TODO SpecLibCreator\n+\n+SpectraFilterBernNorm -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterBernNorm.mzML > SpectraFilterBernNorm.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterBernNorm failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterMarkerMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterMarkerMower.mzML > SpectraFilterMarkerMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterMarkerMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterNLargest -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterNLargest.mzML > SpectraFilterNLargest.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterNLargest failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterNormalizer -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterNormalizer.mzML > SpectraFilterNormalizer.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterNormalizer failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterParentPeakMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterParentPeakMower.mzML > SpectraFilterParentPeakMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterParentPeakMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterScaler -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterScaler.mzML > SpectraFilterScaler.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterScaler failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraFilterThresholdMower -test -in SpectraFilterSqrtMower_1_input.mzML -out SpectraFilterThresholdMower.mzML > SpectraFilterThresholdMower.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraFilterThresholdMower failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+SpectraMerger -test -in NovorAdapter_in.mzML -out SpectraMerger_1.mzML -algorithm:average_gaussian:ms_level 2 > SpectraMerger.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'SpectraMerger failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n+\n+XMLValidator -test -in FileFilter_1_input.mzML > XMLValidator.stdout 2> stderr\n+if [[ "$?" -ne "0" ]]; then >&2 echo \'XMLValidator failed\'; >&2 echo -e "stderr:\\n$(cat stderr | sed \'s/^/ /\')"; fi\n' |
| b |
| diff -r 000000000000 -r f483ffdc7014 readme.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/readme.md Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,152 @@ +Galaxy wrapper for OpenMS +========================= + +OpenMS is an open-source software C++ library for LC/MS data management and analyses. +It offers an infrastructure for the rapid development of mass spectrometry related software. +OpenMS is free software available under the three clause BSD license and runs under Windows, MacOSX and Linux. + +More informations are available at: + + * https://github.com/OpenMS/OpenMS + * https://www.openms.de/ + +The wrappers for these tools and most of their tests are automatically +generated using the `./aux/generate.sh` script. The generation of the tools is +based on the CTDConverter (https://github.com/WorkflowConversion/CTDConverter) +which can be fine tuned via the `hardcoded_params.json` file. This file allows +to blacklist and hardcode parameters and to modify or set arbitrary +CTD/XML attributes. + +Note that, due to its size, the test data is excluded from this repository. In +order to generate the test data on call `test-data.sh`. + +Manual updates should only be done to + +- and the manually contributed tests in `macros_test.xml` (The goal is that all + tools that do not have an automatically generated test are covered here) +- the `hardcoded_params.json` files + +Wrapper versions are managed in `bump.json`. For tools listed in the file +the wrapper version will be set accordingly and otherwise `0` is used. +For a major update of the tool version the bump file should be reset (to `{}`). + +In a few cases patches may be acceptable. + +Installation +============ + +The Galaxy OpenMS tools can be installed from the toolshed. While most tools +will work out of the box some need attention since requirements can not be +fulfilled via Conda: + +Not yet in Conda are: + +- SpectraST (http://tools.proteomecenter.org/wiki/index.php?title=SpectraST) +- MaRaCluster (https://github.com/statisticalbiotechnology/maracluster) + +Binaries for these tools can easily be obtained via: + +``` +VERSION=.... +git git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git OpenMS$VERSION.0-git +git submodule init OpenMS$VERSION.0-git +git submodule update OpenMS$VERSION.0-git +``` + +They are located in `OpenMS$VERSION-git/THIRDPARTY/`. + +Not in Conda due to licencing restrictions: + +- Mascot http://www.matrixscience.com/ +- MSFragger https://github.com/Nesvilab/MSFragger +- Novor http://www.rapidnovor.org/novor + +There are multiple ways to enable the Galaxy tools to use these binaries. + +- Just copy them to the `bin` path within Galaxy's conda environment +- Put them in any other path that that is included in PATH +- Edit the corresponding tools: In the command line part search for the parameters `-executable`, `-maracluster_executable`, or `-mascot_directory` and edit them appropriately. + +Working +======= + +The tools work by: + +Preprocessing: + +- For input data set parameters the links to the actual location of the data + sets are created, the link names are `element_identifier`.`EXT`, where `EXT` + is an extension that is known by OpenMS +- In order to avoid name collisions for the created links each is placed in a + unique directory: `PARAM_NAME/DATASET_ID`, where `PARAM_NAME` is the name + of the parameter and `DATASET_ID` is the id of the Galaxy dataset +- the same happens for output parameters that are in 1:1 correspondence with + an input parameter + + +Main: + +- The galaxy wrapper create two json config files: one containing the + parameters and the values chosen by the user and the other the values of + hardcoded parameters. +- With `OpenMSTool -write_ctd ./` a CTD (names OpenMSTool.ctd) file is + generated that contains the default values. +- A call to `fill_ctd.py` fills in the values from the json config files into + the CTD file +- The actual tool is called `OpenMSTool -ini OpenMSTool.ctd` and also all input + and output parameters are given on the command line. + +Postprocessing: + +- output data sets are moved to the final locations + +Note: The reason for handling data sets on the command line (and not specifying +them in the CTD file) is mainly that all files in Galaxy have the extension +`.dat` and OpenMS tools require an appropriate extension. But this may change +in the future. + +Generating OpenMS wrappers +========================== + +1. remove old test data: `rm -rf $(ls -d test-data/* | egrep -v "random|\.loc")` +2. `./generate.sh` + +Whats happening: + +1. The binaries of the OpenMS package can generate a CTD file that describes + the parameters. These CTD files are converted to xml Galaxy tool descriptions + using the `CTDConverter`. + +2. The CI testing framework of OpenMS contains command lines and test data + (https://github.com/OpenMS/OpenMS/tree/develop/src/tests/topp). These tests + are described in two CMake files. + + - From these CMake files Galaxy tests are auto generated and stored in `macros_autotest.xml` + - The command lines are stored in `prepare_test_data.sh` for regeneration of test data + +More details can be found in the comments of the shell script. + +Open problems +============= + +Licence (MIT) +============= + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + |
| b |
| diff -r 000000000000 -r f483ffdc7014 test-data.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data.sh Fri Jun 14 21:32:42 2024 +0000 |
| [ |
| b'@@ -0,0 +1,308 @@\n+#!/usr/bin/env bash\n+\n+# set -x\n+\n+VERSION=3.1\n+FILETYPES="aux/filetypes.txt"\n+CONDAPKG="https://anaconda.org/bioconda/openms/3.1.0/download/linux-64/openms-3.1.0-h8964181_1.tar.bz2"\n+\n+# install conda\n+if [ -z "$tmp" ]; then\n+ tmp=$(mktemp -d)\n+ created="yes"\n+fi\n+\n+export OPENMSGIT="$tmp/OpenMS$VERSION.0-git"\n+export OPENMSPKG="$tmp/OpenMS$VERSION-pkg/"\n+export OPENMSENV="OpenMS$VERSION-env"\n+\n+if [ -z "$CTDCONVERTER" ]; then\n+ export CTDCONVERTER="$tmp/CTDConverter"\n+fi\n+\n+if [[ -z "$1" ]]; then\n+\tautotests="/dev/null"\n+else\n+\tautotests="$1"\n+fi\n+\n+if type conda > /dev/null; then \n+ true\n+else\n+ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh\n+ bash Miniconda3-latest-Linux-x86_64.sh -b -p "$tmp/miniconda"\n+ source "$tmp/miniconda/bin/activate"\n+fi\n+eval "$(conda shell.bash hook)"\n+\n+\n+###############################################################################\n+## get \n+## - conda environment (for executing the binaries) and \n+## - the git clone of OpenMS (for generating the tests)\n+###############################################################################\n+\n+echo "Clone OpenMS $VERSION sources"\n+if [[ ! -d $OPENMSGIT ]]; then\n+ if [[ "$created" == "yes" ]]; then\n+ GIT_DIR=$(mktemp -d --dry-run)\n+ GIT_EXTRA_OPTS="--separate-git-dir=$GIT_DIR"\n+ fi\n+ git clone -b release/$VERSION.0 --depth 1 --recurse-submodules=THIRDPARTY --shallow-submodules $GIT_EXTRA_OPTS https://github.com/OpenMS/OpenMS.git $OPENMSGIT\n+ ## save some space by just keeping the needed binaries\n+ find $OPENMSGIT/THIRDPARTY/ -type f -not \\( -name maracluster -o -name spectrast \\) -delete\n+ find $OPENMSGIT/THIRDPARTY/ -empty -type d -delete\n+ if [[ "$created" == "yes" ]]; then\n+ rm -rf $GIT_DIR\n+ fi\n+else\n+ cd $OPENMSGIT\n+ git pull origin release/$VERSION.0\n+ cd -\n+fi\n+\n+echo "Create OpenMS $VERSION conda env"\n+# TODO currently add lxml (needed by CTDConverter)\n+# TODO for some reason a to recent openjdk is used\n+if conda env list | grep "$OPENMSENV"; then\n+ true\n+else\n+ conda create -y --quiet --solver libmamba --override-channels --strict-channel-priority --channel conda-forge --channel bioconda -n $OPENMSENV openms=$VERSION openms-thirdparty=$VERSION ctdopts=1.5 lxml\n+# chmod -R u-w $OPENMSENV \n+fi\n+###############################################################################\n+## get the \n+## - conda package (for easy access and listing of the OpenMS binaries), \n+###############################################################################\n+echo "Download OpenMS $VERSION package $CONDAPKG"\n+\n+if [[ ! -d $OPENMSPKG ]]; then\n+ mkdir $OPENMSPKG\n+ wget -q -P $OPENMSPKG/ "$CONDAPKG"\n+ tar -xf $OPENMSPKG/"$(basename $CONDAPKG)" -C $OPENMSPKG/\n+ rm $OPENMSPKG/"$(basename $CONDAPKG)"\n+fi\n+\n+###############################################################################\n+## Get python libaries for CTD -> Galaxy conversion\n+## TODO fix to main repo OR conda packkage if PRs are merged \n+###############################################################################\n+echo "Clone CTDConverter"\n+if [[ ! -d $CTDCONVERTER ]]; then\n+ #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter\n+ git clone -b topic/fix-selects2 https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER\n+else\n+ cd $CTDCONVERTER\n+ git pull origin topic/fix-selects2\n+ cd -\n+fi\n+conda activate $OPENMSENV\n+cd $CTDCONVERTER\n+python -m pip install . --no-deps\n+cd -\n+conda deactivate\n+\n+\n+# # ###############################################################################\n+# # ## copy all the test data files to test-data\n+# # ## most of it (outputs) will be overwritten later, but its needed for\n+# # ## prepare_test_data\n+# # ###############################################################################\n+echo "Get test data"\n+find test-data -type f,l,d ! -name "*fa" ! -name "*loc" ! -name "test-data" ! -name Met'..b'port CRUX_BINARY="crux"\' >> prepare_test_data.sh\n+echo \'export FIDOCHOOSEPARAMS_BINARY="FidoChooseParameters"\' >> prepare_test_data.sh\n+echo \'export FIDO_BINARY="Fido"\' >> prepare_test_data.sh\n+echo \'export LUCIPHOR_BINARY="$(dirname $(realpath $(which luciphor2)))/luciphor2.jar"\' >> prepare_test_data.sh\n+\n+echo \'export MARACLUSTER_BINARY="\'"$OPENMSGIT"\'/THIRDPARTY/Linux/64bit/MaRaCluster/maracluster"\'>> prepare_test_data.sh\n+echo \'export MSFRAGGER_BINARY="/home/berntm/Downloads/MSFragger-3.5/MSFragger-3.5.jar"\'>> prepare_test_data.sh\n+echo \'export MSGFPLUS_BINARY="$(msgf_plus -get_jar_path)"\' >> prepare_test_data.sh\n+echo \'export MYRIMATCH_BINARY="myrimatch"\'>> prepare_test_data.sh\n+echo \'export NOVOR_BINARY="/home/berntm/Downloads/novor/lib/novor.jar"\' >> prepare_test_data.sh\n+echo \'export PERCOLATOR_BINARY="percolator"\'>> prepare_test_data.sh\n+echo \'export SIRIUS_BINARY="$(which sirius)"\' >> prepare_test_data.sh\n+echo \'export SPECTRAST_BINARY="\'"$OPENMSGIT"\'/THIRDPARTY/Linux/64bit/SpectraST/spectrast"\' >> prepare_test_data.sh\n+echo \'export XTANDEM_BINARY="xtandem"\' >> prepare_test_data.sh\n+echo \'export THERMORAWFILEPARSER_BINARY="ThermoRawFileParser.exe"\' >> prepare_test_data.sh\n+echo \'export SAGE_BINARY=sage\' >> prepare_test_data.sh\n+\n+prepare_test_data >> prepare_test_data.sh #tmp_test_data.sh\n+\n+echo "Execute test shell script"\n+chmod u+x prepare_test_data.sh\n+cd ./test-data || exit\n+../prepare_test_data.sh\n+cd - || exit\n+\n+\n+# ###############################################################################\n+# ## create/update test data for the manually generated tests\n+# ## - run convert once with the manual tests only and \n+# ## - update test-data (needs to run 2x)\n+# ###############################################################################\n+echo "Execute test shell script for manually curated tests"\n+chmod u+x prepare_test_data_manual.sh\n+cd ./test-data || exit\n+../prepare_test_data_manual.sh\n+cd - || exit\n+\n+\n+###############################################################################\n+## auto generate tests\n+###############################################################################\n+\n+echo "Write test macros to $autotests"\n+echo "<macros>" > "$autotests"\n+\n+for i in $(ls ctd/*ctd)\n+do\n+ b=$(basename "$i" .ctd)\n+ ./get_tests.py --id "$b" --cmake "$OPENMSGIT"/src/tests/topp/CMakeLists.txt "$OPENMSGIT"/src/tests/topp/THIRDPARTY/third_party_tests.cmake >> "$autotests"\n+ wc -l "$autotests"\n+done\n+echo "</macros>" >> "$autotests"\n+\n+# tests for tools using output_prefix parameters can not be auto generated\n+# hence we output the tests for manual curation in macros_test.xml\n+# and remove them from the autotests\n+# -> OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator\n+# TODO reevaluate in >2.8 \n+# - https://github.com/OpenMS/OpenMS/pull/5873\n+# - https://github.com/OpenMS/OpenMS/pull/5912\n+#\n+# Furthermore we remove tests for tools without binaries in conda\n+# -> MSFragger MaRaClusterAdapter NovorAdapter \n+#\n+# not able to specify composite test data \n+# -> SpectraSTSearchAdapter \n+echo "Discard some tests"\n+if [[ ! -z "$1" ]]; then\n+ echo "" > macros_discarded_auto.xml\n+ for i in OpenSwathFileSplitter IDRipper MzMLSplitter SeedListGenerator MSFraggerAdapter MaRaClusterAdapter NovorAdapter SpectraSTSearchAdapter\n+ do\n+ echo "<xml name=\\"manutest_$i\\">" >> macros_discarded_auto.xml\n+ xmlstarlet sel -t -c "/macros/xml[@name=\'autotest_$i\']/test" macros_autotest.xml >> macros_discarded_auto.xml\n+ echo "</xml>" >> macros_discarded_auto.xml\n+ xmlstarlet ed -d "/macros/xml[@name=\'autotest_$i\']/test" macros_autotest.xml > tmp\n+ mv tmp macros_autotest.xml\n+ done\n+ >&2 echo "discarded autogenerated macros for curation in macros_discarded_auto.xml"\n+fi\n+conda deactivate\n+\n+## remove broken symlinks in test-data\n+find test-data/ -xtype l -delete\n+\n+if [[ "$created" == "yes" ]]; then\n+ echo "Removing temporary directory"\n+ rm -rf "$tmp"\n+fi\n' |
| b |
| diff -r 000000000000 -r f483ffdc7014 test-data/pepnovo_models.loc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/pepnovo_models.loc Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,13 @@ +#name value path +default_models CID_IT_TRYP ${__HERE__}/pepnovo_models/ +default_models LTQ_COMP ${__HERE__}/pepnovo_models/ +default_models DBC4_PEAK ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_TAG5 ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_TAG6 ${__HERE__}/pepnovo_models/ +default_models ITDNV_PEAK ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_SCORE ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_TAG3 ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_DNVPART ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_TAG4 ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_DB ${__HERE__}/pepnovo_models/ +default_models CID_IT_TRYP_CSP ${__HERE__}/pepnovo_models/ |
| b |
| diff -r 000000000000 -r f483ffdc7014 test-data/random.fa --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/random.fa Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,18 @@ +>RND24402 Randomly generated sequence, created by ExPASy tool RandSeq, using average amino acid composition +LALLTDKYSVTKSIKGYAGQQQKCTDDEGLAEDSAAMSLVPIRAAWTISVSVDLFYLGIV +TNVTKDSVEHLVGIPLVTHEFMASRCEMRGQVVSATFGSWQKAESKAYRIPLKATPLDEF +VESAVYLFGGSSNEYECVLIGNSHPVLIFLDIDAVPGARKPRTGFFMAEGFHSKGETRAL +VGKSPPLGEYRKGAFHFTFPIKEAIRLGPPKKRIMGYRDALEGGLNHYVQTQVLVLLPMI +QVARRWENGLGLLVGKFLKLPTHPLDLNQVTLCWSEAVTEDNKRFLLTIKTSAQGKSAPT +SHINYVPQHNSMELMAINGSPFAAQHKSNDEIESMRDLSKLYADAETLESHGERGVRHQA +TETKISKVTNLRRKLPQLLDLNVVDNACNWESVGAHVLEYVLVNLYLKELQEPKVELQPR +LNETTMKAGASSLGVESGASAHSFYKGGVSEAKLRFRHVATPAAARIWWCVVMFRINRRY +DGITYNSVGEQLSGVHEYVRAAQLFGLTTGKNLRSTGIVIIKLSTAIDLECLVQAKPKEA +YVLANDYIGAKPHPARLETGPALVLFIVETINNDTLNAAILITALGGKFLNVRPDLLFGV +QALFGCVRMFRHADCTIGREKFVQTEISHKAKFLYEINEFFLERILQFEEAKSPVGAPAY +DIPIGRGLVMDSSTDLWNIYVVELISGQEKRTGIDPDTPMGTSHNLYMTDARLDERDQRS +FLNSEFVKPSKLANGSEWADPYVEPDKTEVIAFFPATLIVIMADGSALNGQVCIQPAKDN +SKMADDLATVHIGQDRPCDWGISASHEYDEVNRPARINGVMMQQLMAEDNQGPGASPRDQ +MGDADDLKEIKWNKYVIDNEIIGRERGISAERVKIFLGDTLTARGLLDSPPGQTKVFDLR +PRQSDKNQSGMFKRDQNAMYFPLEYDRIGAQTDTGSLYSTLITKFASISIDLVKLSMPRE +KQIDEERLHSEFIENQKRSALPAVQKNLACISCVEACRGT |
| b |
| diff -r 000000000000 -r f483ffdc7014 test-data/random_RNA.fa --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/random_RNA.fa Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,2 @@ +> random RNA +GUGUUACUGCCACGAAACAAAAUGUUCAAGACACCGGGCGCCAUCUGUAUAUUACUCGCCAAUCAGACGGUCUGCAACGCUACAGACAUGGAGCUCAGCGCUGACGAUGUCGCCGGACCAAGUACGAUCACUUUGCUCGUGCAAAUGUUCGUCCGCAUUGGGCACUAUAACUCGAAUUGUCGAAUCCGGGUGGCGAGCCGCCACUUAUAGGAUAAAUAUUCAAACUAACAUUAUGGCGCCAAAUCUGCAAUCUCUACUUUAGACAUUAUAUACCCACAUUUACAAUUAGAGUUAUUAUUAGUUAACGUGUGCCAGAGCAGGGAUGGCUCUUGUCAGCCAUAGUUGUGUGAACGGGCUGUAUUUCCUUCCUAAUUAUAGAGCGGCACCGGAAAGCAAUGCACGAUCCACGAGGGCACUUCACAUGGUCACAAACAGUCAUUCUGGUACCCUGAUUCGUUCCCGAAAGGGAAGUAUAUACACGGCCCCCGUGUAUAUCGCCAGUCACACGGCAGGAGCGAGAGUUCGUUUGUAUACAUGCCCAGGAGCCUUCUCUAACUUUUGAAGCUGUGCAACUUUGUUGGCGCGUCACCACUAAGUCAGCUUAAUAGACAGCAGAUGGGAGAAUUUACCAUUUCAUUUUGUCCGAGCUGAUACCGGUAGGUCAUCUCUAAUCACCCGUUAUCCUCUCGUAAUAUAAUCGCUACUAAGGUAUGAAGGUGUCUGCGAAAGGUAACGUAAAUCAUUCUCGGCUCCUUGCAAAGUACGACUAGGAUCCAUCGUACACAUCCGGACGAAGAUGUAAAAUUGACGCCCCUGUAGGCCGUGAGACAGACGUGAGCCAAACCAUCUGCUCUACUUCUGGAGGCCUUGAAUAGUGGCGCGUUGUGUAAUCUUAAGAGAGAUUUUACUUGGAAUUACAGCCUACUUUGACCAGUAGCGCAUUGUGAACAAAUAUUCCCGUACGCGUCCAAUUGCAGCAAAACGUGGGCCUGUGUCCAGU |
| b |
| diff -r 000000000000 -r f483ffdc7014 tool-data/pepnovo_models.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/pepnovo_models.loc.sample Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,23 @@ +# This is a sample file distributed with Galaxy that enables tools +# to use pepnovo models +# The file has three tab separated columns: name, value, and path. +# The idea is that there are a number of models in a directory: +# - each model directory has a unique name (columns 2 and 0) +# - each model can contain a set of models (column 1) +# +# The following example works fo the default models from +# http://proteomics.ucsd.edu/Software/PepNovo.html (just remove the comment +# chars and replace DIR_TO_PEPNOVO_MODELS) + +#default_models CID_IT_TRYP DIR_TO_PEPNOVO_MODELS +#default_models LTQ_COMP DIR_TO_PEPNOVO_MODELS +#default_models DBC4_PEAK DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG5 DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG6 DIR_TO_PEPNOVO_MODELS +#default_models ITDNV_PEAK DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_SCORE DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG3 DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_DNVPART DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_TAG4 DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_DB DIR_TO_PEPNOVO_MODELS +#default_models CID_IT_TRYP_CSP DIR_TO_PEPNOVO_MODELS |
| b |
| diff -r 000000000000 -r f483ffdc7014 tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,7 @@ +<tables> + <!-- Locations of pepnovo models --> + <table name="pepnovo_models" comment_char="#" allow_duplicate_entries="False"> + <columns>name,value,path</columns> + <file path="tool-data/pepnovo_models.loc" /> + </table> +</tables> |
| b |
| diff -r 000000000000 -r f483ffdc7014 tool_data_table_conf.xml.test --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.test Fri Jun 14 21:32:42 2024 +0000 |
| b |
| @@ -0,0 +1,7 @@ +<tables> + <!-- Locations of all eggnog_mapper data --> + <table name="pepnovo_models" comment_char="#"> + <columns>name,value,path</columns> + <file path="${__HERE__}/test-data/pepnovo_models.loc" /> + </table> +</tables> |