Mercurial > repos > fubar > tool_factory_2
changeset 38:a30536c100bf draft
Updated history outputs
| author | fubar |
|---|---|
| date | Wed, 12 Aug 2020 01:43:46 -0400 |
| parents | 099047ee7094 |
| children | 2cd6555baf44 |
| files | toolfactory/galaxyxml/__init__.py toolfactory/galaxyxml/__pycache__/__init__.cpython-36.pyc toolfactory/galaxyxml/tool/__init__.py toolfactory/galaxyxml/tool/__pycache__/__init__.cpython-36.pyc toolfactory/galaxyxml/tool/import_xml.py toolfactory/galaxyxml/tool/parameters/__init__.py toolfactory/galaxyxml/tool/parameters/__pycache__/__init__.cpython-36.pyc toolfactory/rgToolFactory2.py toolfactory/rgToolFactory2.xml toolfactory/testtf.sh |
| diffstat | 10 files changed, 231 insertions(+), 262 deletions(-) [+] |
line wrap: on
line diff
--- a/toolfactory/galaxyxml/__init__.py Mon Aug 10 23:25:51 2020 -0400 +++ b/toolfactory/galaxyxml/__init__.py Wed Aug 12 01:43:46 2020 -0400 @@ -1,5 +1,6 @@ +from builtins import object from builtins import str -from builtins import object + from lxml import etree @@ -17,11 +18,7 @@ """Recursive data sanitisation """ if isinstance(data, dict): - return { - k: cls.coerce(v, kill_lists=kill_lists) - for k, v in list(data.items()) - if v is not None - } + return {k: cls.coerce(v, kill_lists=kill_lists) for k, v in list(data.items()) if v is not None} elif isinstance(data, list): if kill_lists: return cls.coerce(data[0])
--- a/toolfactory/galaxyxml/tool/__init__.py Mon Aug 10 23:25:51 2020 -0400 +++ b/toolfactory/galaxyxml/tool/__init__.py Wed Aug 12 01:43:46 2020 -0400 @@ -1,8 +1,10 @@ import copy import logging + +from galaxyxml import GalaxyXML, Util +from galaxyxml.tool.parameters import XMLParam + from lxml import etree -from galaxyxml import Util, GalaxyXML -from galaxyxml.tool.parameters import XMLParam VALID_TOOL_TYPES = ("data_source", "data_source_async") VALID_URL_METHODS = ("get", "post") @@ -12,6 +14,7 @@ class Tool(GalaxyXML): + def __init__( self, name, @@ -51,9 +54,7 @@ if tool_type is not None: if tool_type not in VALID_TOOL_TYPES: - raise Exception( - "Tool type must be one of %s" % ",".join(VALID_TOOL_TYPES) - ) + raise Exception("Tool type must be one of %s" % ",".join(VALID_TOOL_TYPES)) else: kwargs["tool_type"] = tool_type @@ -61,9 +62,7 @@ if URL_method in VALID_URL_METHODS: kwargs["URL_method"] = URL_method else: - raise Exception( - "URL_method must be one of %s" % ",".join(VALID_URL_METHODS) - ) + raise Exception("URL_method must be one of %s" % ",".join(VALID_URL_METHODS)) description_node = etree.SubElement(self.root, "description") description_node.text = description @@ -150,16 +149,10 @@ if getattr(self, "command", None): command_node.text = etree.CDATA(export_xml.command) else: - logger.warning( - "The tool does not have any old command stored. " - + "Only the command line is written." - ) + logger.warning("The tool does not have any old command stored. " + "Only the command line is written.") command_node.text = export_xml.executable else: - actual_cli = "%s %s" % ( - export_xml.executable, - export_xml.clean_command_string(command_line), - ) + actual_cli = "%s %s" % (export_xml.executable, export_xml.clean_command_string(command_line)) command_node.text = etree.CDATA(actual_cli.strip()) try:
--- a/toolfactory/galaxyxml/tool/import_xml.py Mon Aug 10 23:25:51 2020 -0400 +++ b/toolfactory/galaxyxml/tool/import_xml.py Wed Aug 12 01:43:46 2020 -0400 @@ -1,5 +1,6 @@ import logging import xml.etree.ElementTree as ET + import galaxyxml.tool as gxt import galaxyxml.tool.parameters as gxtp @@ -115,9 +116,7 @@ value = req.text if req.tag == "requirement": version = req.attrib.get("version", None) - tool.requirements.append( - gxtp.Requirement(req_type, value, version=version) - ) + tool.requirements.append(gxtp.Requirement(req_type, value, version=version)) elif req.tag == "container": tool.requirements.append(gxtp.Container(req_type, value)) else: @@ -350,9 +349,7 @@ """ root.append( gxtp.SelectOption( - option.attrib.get("value", None), - option.text, - selected=option.attrib.get("selected", False), + option.attrib.get("value", None), option.text, selected=option.attrib.get("selected", False) ) ) @@ -432,13 +429,9 @@ # Deal with child nodes (usually option and options) for sel_child in sel_param: try: - getattr(self, "_load_{}_select".format(sel_child.tag))( - select_param, sel_child - ) + getattr(self, "_load_{}_select".format(sel_child.tag))(select_param, sel_child) except AttributeError: - logger.warning( - sel_child.tag + " tag is not processed for <param type='select'>." - ) + logger.warning(sel_child.tag + " tag is not processed for <param type='select'>.") root.append(select_param) def _load_param(self, root, param_root): @@ -539,12 +532,7 @@ try: getattr(self, "_load_{}".format(inp_child.tag))(root, inp_child) except AttributeError: - logger.warning( - inp_child.tag - + " tag is not processed for <" - + inputs_root.tag - + "> tag." - ) + logger.warning(inp_child.tag + " tag is not processed for <" + inputs_root.tag + "> tag.") class OutputsParser(object): @@ -589,9 +577,7 @@ for chfmt_child in chfmt_root: change_format.append( gxtp.ChangeFormatWhen( - chfmt_child.attrib["input"], - chfmt_child.attrib["format"], - chfmt_child.attrib["value"], + chfmt_child.attrib["input"], chfmt_child.attrib["format"], chfmt_child.attrib["value"] ) ) root.append(change_format) @@ -618,9 +604,7 @@ try: getattr(self, "_load_{}".format(coll_child.tag))(collection, coll_child) except AttributeError: - logger.warning( - coll_child.tag + " tag is not processed for <collection>." - ) + logger.warning(coll_child.tag + " tag is not processed for <collection>.") outputs_root.append(collection) def _load_discover_datasets(self, root, disc_root): @@ -725,7 +709,5 @@ try: getattr(self, "_load_{}".format(test_child.tag))(test, test_child) except AttributeError: - logger.warning( - test_child.tag + " tag is not processed within <test>." - ) + logger.warning(test_child.tag + " tag is not processed within <test>.") root.append(test)
--- a/toolfactory/galaxyxml/tool/parameters/__init__.py Mon Aug 10 23:25:51 2020 -0400 +++ b/toolfactory/galaxyxml/tool/parameters/__init__.py Wed Aug 12 01:43:46 2020 -0400 @@ -1,7 +1,10 @@ +from builtins import object from builtins import str -from builtins import object + +from galaxyxml import Util + from lxml import etree -from galaxyxml import Util + class XMLParam(object): @@ -23,13 +26,11 @@ self.children.append(sub_node) else: raise Exception( - "Child was unacceptable to parent (%s is not appropriate for %s)" - % (type(self), type(sub_node)) + "Child was unacceptable to parent (%s is not appropriate for %s)" % (type(self), type(sub_node)) ) else: raise Exception( - "Child was unacceptable to parent (%s is not appropriate for %s)" - % (type(self), type(sub_node)) + "Child was unacceptable to parent (%s is not appropriate for %s)" % (type(self), type(sub_node)) ) def validate(self): @@ -155,9 +156,7 @@ # This bodes to be an issue -__- def acceptable_child(self, child): - return issubclass(type(child), Requirement) or issubclass( - type(child), Container - ) + return issubclass(type(child), Requirement) or issubclass(type(child), Container) class Requirement(XMLParam): @@ -187,9 +186,7 @@ name = "configfiles" def acceptable_child(self, child): - return issubclass(type(child), Configfile) or issubclass( - type(child), ConfigfileDefaultInputs - ) + return issubclass(type(child), Configfile) or issubclass(type(child), ConfigfileDefaultInputs) class Configfile(XMLParam): @@ -217,15 +214,7 @@ name = "inputs" # This bodes to be an issue -__- - def __init__( - self, - action=None, - check_value=None, - method=None, - target=None, - nginx_upload=None, - **kwargs, - ): + def __init__(self, action=None, check_value=None, method=None, target=None, nginx_upload=None, **kwargs): params = Util.clean_kwargs(locals().copy()) super(Inputs, self).__init__(**params) @@ -262,9 +251,7 @@ # TODO: replace with positional attribute if len(self.flag()) > 0: if kwargs["label"] is None: - kwargs[ - "label" - ] = "Author did not provide help for this parameter... " + kwargs["label"] = "Author did not provide help for this parameter... " if not self.positional: kwargs["argument"] = self.flag() @@ -297,11 +284,7 @@ if self.positional: return self.mako_name() else: - return "%s%s%s" % ( - self.flag(), - self.space_between_arg, - self.mako_name(), - ) + return "%s%s%s" % (self.flag(), self.space_between_arg, self.mako_name()) def mako_name(self): # TODO: enhance logic to check up parents for things like @@ -353,9 +336,7 @@ super(Conditional, self).__init__(**params) def acceptable_child(self, child): - return issubclass(type(child), InputParameter) and not isinstance( - child, Conditional - ) + return issubclass(type(child), InputParameter) and not isinstance(child, Conditional) def validate(self): # Find a way to check if one of the kids is a WHEN @@ -383,22 +364,16 @@ super(Param, self).__init__(**params) if type(self) == Param: - raise Exception( - "Param class is not an actual parameter type, use a subclass of Param" - ) + raise Exception("Param class is not an actual parameter type, use a subclass of Param") def acceptable_child(self, child): - return issubclass( - type(child, InputParameter) or isinstance(child), ValidatorParam - ) + return issubclass(type(child, InputParameter) or isinstance(child), ValidatorParam) class TextParam(Param): type = "text" - def __init__( - self, name, optional=None, label=None, help=None, value=None, **kwargs - ): + def __init__(self, name, optional=None, label=None, help=None, value=None, **kwargs): params = Util.clean_kwargs(locals().copy()) super(TextParam, self).__init__(**params) @@ -413,17 +388,7 @@ class _NumericParam(Param): - def __init__( - self, - name, - value, - optional=None, - label=None, - help=None, - min=None, - max=None, - **kwargs, - ): + def __init__(self, name, value, optional=None, label=None, help=None, min=None, max=None, **kwargs): params = Util.clean_kwargs(locals().copy()) super(_NumericParam, self).__init__(**params) @@ -440,15 +405,7 @@ type = "boolean" def __init__( - self, - name, - optional=None, - label=None, - help=None, - checked=False, - truevalue=None, - falsevalue=None, - **kwargs, + self, name, optional=None, label=None, help=None, checked=False, truevalue=None, falsevalue=None, **kwargs ): params = Util.clean_kwargs(locals().copy()) @@ -477,16 +434,7 @@ class DataParam(Param): type = "data" - def __init__( - self, - name, - optional=None, - label=None, - help=None, - format=None, - multiple=None, - **kwargs, - ): + def __init__(self, name, optional=None, label=None, help=None, format=None, multiple=None, **kwargs): params = Util.clean_kwargs(locals().copy()) super(DataParam, self).__init__(**params) @@ -505,7 +453,7 @@ multiple=None, options=None, default=None, - **kwargs, + **kwargs ): params = Util.clean_kwargs(locals().copy()) del params["options"] @@ -544,14 +492,7 @@ class Options(InputParameter): name = "options" - def __init__( - self, - from_dataset=None, - from_file=None, - from_data_table=None, - from_parameter=None, - **kwargs, - ): + def __init__(self, from_dataset=None, from_file=None, from_data_table=None, from_parameter=None, **kwargs): params = Util.clean_kwargs(locals().copy()) super(Options, self).__init__(None, **params) @@ -583,7 +524,7 @@ value=None, ref_attribute=None, index=None, - **kwargs, + **kwargs ): params = Util.clean_kwargs(locals().copy()) super(Filter, self).__init__(**params) @@ -602,7 +543,7 @@ line_startswith=None, min=None, max=None, - **kwargs, + **kwargs ): params = Util.clean_kwargs(locals().copy()) super(ValidatorParam, self).__init__(**params) @@ -630,7 +571,7 @@ label=None, from_work_dir=None, hidden=False, - **kwargs, + **kwargs ): # TODO: validate format_source&metadata_source against something in the # XMLParam children tree. @@ -659,11 +600,7 @@ return flag + self.mako_identifier def acceptable_child(self, child): - return ( - isinstance(child, OutputFilter) - or isinstance(child, ChangeFormat) - or isinstance(child, DiscoverDatasets) - ) + return isinstance(child, OutputFilter) or isinstance(child, ChangeFormat) or isinstance(child, DiscoverDatasets) class OutputFilter(XMLParam): @@ -713,25 +650,19 @@ type_source=None, structured_like=None, inherit_format=None, - **kwargs, + **kwargs ): params = Util.clean_kwargs(locals().copy()) super(OutputCollection, self).__init__(**params) def acceptable_child(self, child): - return ( - isinstance(child, OutputData) - or isinstance(child, OutputFilter) - or isinstance(child, DiscoverDatasets) - ) + return isinstance(child, OutputData) or isinstance(child, OutputFilter) or isinstance(child, DiscoverDatasets) class DiscoverDatasets(XMLParam): name = "discover_datasets" - def __init__( - self, pattern, directory=None, format=None, ext=None, visible=None, **kwargs - ): + def __init__(self, pattern, directory=None, format=None, ext=None, visible=None, **kwargs): params = Util.clean_kwargs(locals().copy()) super(DiscoverDatasets, self).__init__(**params) @@ -773,7 +704,7 @@ compare=None, lines_diff=None, delta=None, - **kwargs, + **kwargs ): params = Util.clean_kwargs(locals().copy()) super(TestOutput, self).__init__(**params)
--- a/toolfactory/rgToolFactory2.py Mon Aug 10 23:25:51 2020 -0400 +++ b/toolfactory/rgToolFactory2.py Wed Aug 12 01:43:46 2020 -0400 @@ -1,3 +1,4 @@ +#!/usr/bin/env python # rgToolFactory.py # see https://github.com/fubar2/toolfactory # @@ -33,40 +34,45 @@ import galaxyxml.tool as gxt import galaxyxml.tool.parameters as gxtp + import lxml -foo = lxml.__name__ # fug you, flake8. Say my name! Please accept the PR, Helena! - -progname = os.path.split(sys.argv[0])[1] myversion = "V2.1 July 2020" verbose = True debug = True toolFactoryURL = "https://github.com/fubar2/toolfactory" ourdelim = "~~~" -# --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label~~~$input_help" +# --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label +# ~~~$input_help" IPATHPOS = 0 ICLPOS = 1 IFMTPOS = 2 ILABPOS = 3 IHELPOS = 4 IOCLPOS = 5 + # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL ONAMEPOS = 0 OFMTPOS = 1 OCLPOS = 2 OOCLPOS = 3 -# --additional_parameters="$i.param_name~~~$i.param_value~~~$i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL" +# --additional_parameters="$i.param_name~~~$i.param_value~~~ +# $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride" ANAMEPOS = 0 AVALPOS = 1 ALABPOS = 2 AHELPPOS = 3 ATYPEPOS = 4 ACLPOS = 5 -AOCLPOS = 6 +AOVERPOS = 6 +AOCLPOS = 7 +foo = len(lxml.__version__) +# fug you, flake8. Say my name! + def timenow(): """return current time as a string """ @@ -108,9 +114,11 @@ citation_tuples = [] for citation in citations: if citation.startswith("doi"): - citation_tuples.append(("doi", citation[len("doi"):].strip())) + citation_tuples.append(("doi", citation[len("doi") :].strip())) else: - citation_tuples.append(("bibtex", citation[len("bibtex"):].strip())) + citation_tuples.append( + ("bibtex", citation[len("bibtex") :].strip()) + ) return citation_tuples @@ -144,7 +152,6 @@ ], 'Parameter passing in args.parampass must be "0","positional" or "argparse"' self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) self.tool_id = self.tool_name - self.xmlfile = "%s.xml" % self.tool_name if self.args.interpreter_name: exe = "$runMe" else: @@ -177,25 +184,41 @@ clsuffix = [] xclsuffix = [] for i, p in enumerate(self.infiles): - appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS]] + if p[IOCLPOS] == "STDIN": + appendme = [ + p[IOCLPOS], + p[ICLPOS], + p[IPATHPOS], + "< %s" % p[IPATHPOS], + ] + xappendme = [ + p[IOCLPOS], + p[ICLPOS], + p[IPATHPOS], + "< $%s" % p[ICLPOS], + ] + else: + appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""] + xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] clsuffix.append(appendme) - xclsuffix.append([p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS]]) + xclsuffix.append(xappendme) # print('##infile i=%d, appendme=%s' % (i,appendme)) for i, p in enumerate(self.outfiles): if p[OOCLPOS] == "STDOUT": self.lastclredirect = [">", p[ONAMEPOS]] self.lastxclredirect = [">", "$%s" % p[OCLPOS]] - # print('##outfiles i=%d lastclredirect = %s' % (i,self.lastclredirect)) else: - appendme = [p[OOCLPOS], p[OCLPOS], p[ONAMEPOS]] - clsuffix.append(appendme) - xclsuffix.append([p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS]]) - # print('##outfiles i=%d' % i,'appendme',appendme) + clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""]) + xclsuffix.append( + [p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""] + ) for p in self.addpar: - appendme = [p[AOCLPOS], p[ACLPOS], p[AVALPOS]] - clsuffix.append(appendme) - xclsuffix.append([p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS]]) - # print('##adpar %d' % i,'appendme=',appendme) + clsuffix.append( + [p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]] + ) + xclsuffix.append( + [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] + ) clsuffix.sort() xclsuffix.sort() self.xclsuffix = xclsuffix @@ -213,13 +236,17 @@ assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" self.script = "\n".join(rx) fhandle, self.sfile = tempfile.mkstemp( - prefix=self.tool_name, suffix=".%s" % (self.args.interpreter_name) + prefix=self.tool_name, suffix="_%s" % (self.args.interpreter_name) ) tscript = open(self.sfile, "w") tscript.write(self.script) tscript.close() - self.indentedScript = " %s" % "\n".join([" %s" % html_escape(x) for x in rx]) - self.escapedScript = "%s" % "\n".join([" %s" % html_escape(x) for x in rx]) + self.indentedScript = " %s" % "\n".join( + [" %s" % html_escape(x) for x in rx] + ) + self.escapedScript = "%s" % "\n".join( + [" %s" % html_escape(x) for x in rx] + ) art = "%s.%s" % (self.tool_name, self.args.interpreter_name) artifact = open(art, "wb") artifact.write(bytes(self.script, "utf8")) @@ -282,13 +309,13 @@ def clpositional(self): # inputs in order then params aCL = self.cl.append - for (o_v, k, v) in self.clsuffix: + for (o_v, k, v, koverride) in self.clsuffix: if " " in v: aCL("%s" % v) else: aCL(v) aXCL = self.xmlcl.append - for (o_v, k, v) in self.xclsuffix: + for (o_v, k, v, koverride) in self.xclsuffix: aXCL(v) if self.lastxclredirect: aXCL(self.lastxclredirect[0]) @@ -300,15 +327,19 @@ aCL = self.cl.append aXCL = self.xmlcl.append # inputs then params in argparse named form - for (o_v, k, v) in self.xclsuffix: - if len(k.strip()) == 1: + for (o_v, k, v, koverride) in self.xclsuffix: + if koverride > "": + k = koverride + elif len(k.strip()) == 1: k = "-%s" % k else: k = "--%s" % k aXCL(k) aXCL(v) - for (o_v, k, v) in self.clsuffix: - if len(k.strip()) == 1: + for (o_v, k, v, koverride) in self.clsuffix: + if koverride > "": + k = koverride + elif len(k.strip()) == 1: k = "-%s" % k else: k = "--%s" % k @@ -339,7 +370,9 @@ aparm.positional = int(oldcl) aparm.command_line_override = "$%s" % newcl self.toutputs.append(aparm) - tp = gxtp.TestOutput(name=newcl, value="%s_sample" % newcl, format=newfmt) + tp = gxtp.TestOutput( + name=newcl, value="%s_sample" % newcl, format=newfmt + ) self.testparam.append(tp) for p in self.infiles: newname = p[ICLPOS] @@ -363,7 +396,7 @@ tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname) self.testparam.append(tparm) for p in self.addpar: - newname, newval, newlabel, newhelp, newtype, newcl, oldcl = p + newname, newval, newlabel, newhelp, newtype, newcl, override, oldcl = p if not len(newlabel) > 0: newlabel = newname ndash = self.getNdash(newname) @@ -434,7 +467,9 @@ anout.command_line_override = "> $%s" % newname anout.positional = self.is_positional self.toutputs.append(anout) - tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname, format=newfmt) + tp = gxtp.TestOutput( + name=newname, value="%s_sample" % newname, format=newfmt + ) self.testparam.append(tp) def makeXML(self): @@ -445,7 +480,7 @@ """ self.tool.command_line_override = self.xmlcl if self.args.interpreter_name: - self.tool.interpreter = self.interp + self.tool.interpreter = self.args.interpreter_name if self.args.help_text: helptext = open(self.args.help_text, "r").readlines() helptext = [html_escape(x) for x in helptext] @@ -462,7 +497,9 @@ if self.args.interpreter_name: if self.args.interpreter_name == "python": requirements.append( - gxtp.Requirement("package", "python", self.args.interpreter_version) + gxtp.Requirement( + "package", "python", self.args.interpreter_version + ) ) elif self.args.interpreter_name not in ["bash", "sh"]: requirements.append( @@ -476,14 +513,16 @@ if self.args.exe_package and self.args.parampass != "system": requirements.append( gxtp.Requirement( - "package", self.args.exe_package, self.args.exe_package_version + "package", + self.args.exe_package, + self.args.exe_package_version, ) ) self.tool.requirements = requirements if self.args.parampass == "0": - self.doXMLNoparam() + self.doNoXMLparam() else: - self.doXMLParam() + self.doXMLparam() self.tool.outputs = self.toutputs self.tool.inputs = self.tinputs if self.args.runmode not in ["Executable", "system"]: @@ -505,7 +544,7 @@ "Cite: Creating re-usable tools from scripts doi: 10.1093/bioinformatics/bts573" ) exml = self.tool.export() - xf = open(self.xmlfile, "w") + xf = open('%s.xml' % self.tool_name, "w") xf.write(exml) xf.write("\n") xf.close() @@ -520,7 +559,9 @@ """ retval = self.run() if retval: - sys.stderr.write("## Run failed. Cannot build yet. Please fix and retry") + sys.stderr.write( + "## Run failed. Cannot build yet. Please fix and retry" + ) sys.exit(1) tdir = "tfout" if not os.path.exists(tdir): @@ -549,19 +590,32 @@ shutil.copyfile(pth, dest) if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0: - shutil.copyfile(self.tlog, os.path.join(testdir, "test1_log.txt")) + shutil.copyfile(self.tlog, os.path.join(testdir, "test1_log_outfiletxt")) if self.args.runmode not in ["Executable", "system"]: stname = os.path.join(tdir, "%s" % (self.sfile)) if not os.path.exists(stname): shutil.copyfile(self.sfile, stname) - xtname = os.path.join(tdir, self.xmlfile) - if not os.path.exists(xtname): - shutil.copyfile(self.xmlfile, xtname) + xreal = '%s.xml' % self.tool_name + xout = os.path.join(tdir,xreal) + shutil.copyfile(xreal, xout) tarpath = "toolfactory_%s.tgz" % self.tool_name tf = tarfile.open(tarpath, "w:gz") tf.add(name=tdir, arcname=self.tool_name) tf.close() shutil.copyfile(tarpath, self.args.new_tool) + shutil.copyfile(xreal,"tool_xml.txt") + repdir = "TF_run_report_tempdir" + if not os.path.exists(repdir): + os.mkdir(repdir) + repoutnames = [x[OCLPOS] for x in self.outfiles] + with os.scandir('.') as outs: + for entry in outs: + if entry.name.endswith('.tgz') or not entry.is_file(): + continue + if entry.name in repoutnames: + shutil.copyfile(entry.name,os.path.join(repdir,entry.name)) + elif entry.name == "%s.xml" % self.tool_name: + shutil.copyfile(entry.name,os.path.join(repdir,"new_tool_xml")) return retval def run(self): @@ -577,12 +631,15 @@ if self.args.parampass != "0": ste = open(self.elog, "wb") if self.lastclredirect: - sto = open(self.lastclredirect[1], "wb") # is name of an output file + sto = open( + self.lastclredirect[1], "wb" + ) # is name of an output file else: sto = open(self.tlog, "wb") sto.write( bytes( - "## Executing Toolfactory generated command line = %s\n" % scl, + "## Executing Toolfactory generated command line = %s\n" + % scl, "utf8", ) ) @@ -654,17 +711,23 @@ 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' % (args.bad_user, args.bad_user) ) - assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq" + assert ( + args.tool_name + ), "## Tool Factory expects a tool name - eg --tool_name=DESeq" assert ( args.interpreter_name or args.exe_package ), "## Tool Factory wrapper expects an interpreter or an executable package" assert args.exe_package or ( len(args.script_path) > 0 and os.path.isfile(args.script_path) ), "## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable" - args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files] + args.input_files = [ + x.replace('"', "").replace("'", "") for x in args.input_files + ] # remove quotes we need to deal with spaces in CL params for i, x in enumerate(args.additional_parameters): - args.additional_parameters[i] = args.additional_parameters[i].replace('"', "") + args.additional_parameters[i] = args.additional_parameters[i].replace( + '"', "" + ) r = ScriptRunner(args) if args.make_Tool: retcode = r.makeTooltar()
--- a/toolfactory/rgToolFactory2.xml Mon Aug 10 23:25:51 2020 -0400 +++ b/toolfactory/rgToolFactory2.xml Wed Aug 12 01:43:46 2020 -0400 @@ -1,10 +1,10 @@ -<tool id="rgTF2" name="toolfactory" version="2.00"> +<tool id="rgTF2" name="toolfactory" version="2.00" profile="16.04" > <description>Scripts into tools</description> <macros> <xml name="io"> <repeat name="history_inputs" title="Add a data file from your history to pass in to the script. Use the '+' button as needed" help="USE SMALL SAMPLES because these will be used for the new tool's test. The names will become a history item selector as input for users of the new tool you are making"> - <param name="input_files" type="data" format="data" label="Select an input file from your history" optional="true" size="120" multiple="false" + <param name="input_files" type="data" format="data" label="Select an input file from your history" optional="true" multiple="false" help=""/> <param name="input_formats" type="select" multiple="true" label="Select the datatype(s) that your tool/script accepts as input" help="If your datatype is not listed here, it has to be added in galaxy's datatypes_conf.xml" value="tabular"> @@ -12,7 +12,7 @@ <column name="value" index="0"/> </options> </param> - <param name="input_label" type="text" value="" label="This will become the user prompt for the form so please make it informative" size="60" + <param name="input_label" type="text" value="" label="This will become the user prompt for the form so please make it informative" help="Note that '~~~' is an internal delimiter so must not appear in this text field - please work around this technical limitation" > <sanitizer invalid_char=""> <valid initial="string.printable"> <remove value='~~~'/> </valid> @@ -20,33 +20,31 @@ </sanitizer> </param> <param name="input_help" type="text" value="parameter_help" label="This will become help text on the form." - help="Note that three consecutive ~ cannot be used in this text field - please work around this technical limitation" size="60"> + help="Note that three consecutive ~ cannot be used in this text field - please work around this technical limitation"> <sanitizer invalid_char=""> <valid initial="string.printable"> <remove value='~~~'/> </valid> <mapping initial="none"/> </sanitizer> </param> - <param name="input_CL" type="text" size="60" label="Positional: ordinal integer. Argparse: argument name." - help="If you will pass positional parameters, enter the integer ordinal for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""> + <param name="input_CL" type="text" label="Positional: ordinal integer. Argparse: argument name. STDIN if the executable/script expects it" + help="If you will pass positional parameters, enter the integer ordinal for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""> </param> </repeat> <repeat name="history_outputs" title="Add a tool run output file to the user's history from your tool - Use the '+' button to add as many as needed" help="The name will become a history item for users of the new tool you are making containing one of it's outputs"> - <param name="history_name" type="text" label="Name for this output to appear in new history" optional="false" size="120" - help=""/> + <param name="history_name" type="text" label="Name for this output to appear in new history" optional="false" help=""/> <param name="history_format" type="select" multiple="false" label="Select the datatype for this output" help="If your datatype is not listed here, it has to be added in galaxy's datatypes_conf.xml" value="tabular"> <options from_parameter="tool.app.datatypes_registry.upload_file_formats"> <column name="value" index="0"/> </options> </param> - <param name="history_CL" type="text" size="60" label="Positional: ordinal integer. Argparse: argument name expected for this output parameter" - help="If positional parameters, enter the integer ordinal expected for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""> - </param> + <param name="history_CL" type="text" label="Positional: ordinal integer. Argparse: argument name expected for this output parameter. Use STDOUT if '>' required." + help="If positional parameters, enter the integer ordinal expected for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""/> </repeat> </xml> <xml name="additparam"> - <param name="edit_params" type="select" display="radio" label="Add any additional parameters to the generated tool form so they are user editable?" + <param name="edit_params" type="select" display="radio" label="Add any additional parameters to the generated tool form so they are user editable?" help="If no (default), users will NOT be able to alter any additional parameters. If yes, these will appear on the tool form as text fields with no validation or sanitizing"> <option value="yes">Yes, allow user to edit all additional parameters on the generated tool form</option> <option value="no" selected="true">No - use the fixed values for all additional parameters - no user editing</option> @@ -54,10 +52,10 @@ <repeat name="additional_parameters" title="Pass additional parameters to the script" help="See examples below to see how these can be parsed by scripts in the various languages"> - <param name="param_name" type="text" value="parameter_name" label="Choose the name for this parameter" size="60"> + <param name="param_name" type="text" value="parameter_name" label="Choose the name for this parameter"> <sanitizer invalid_char=""> <valid initial="string.letters,string.digits"/> - <mapping initial="none"/> + <mapping initial="none"/> </sanitizer> </param> <param name="param_type" type="select" label="Select the type for this parameter"> @@ -65,14 +63,14 @@ <option value="integer">integer</option> <option value="float">float</option> </param> - <param name="param_value" type="text" value="" label="Enter this parameter's default value" size="60" + <param name="param_value" type="text" value="" label="Enter this parameter's default value" help="Note that '~~~' is an internal delimiter must not appear in this text field - please work around this technical limitation" > <sanitizer invalid_char=""> <valid initial="string.printable"> <remove value='~~~'/> </valid> <mapping initial="none"/> </sanitizer> </param> - <param name="param_label" type="text" value="parameter_label" label="Enter this parameter's label for the form" size="60" + <param name="param_label" type="text" value="parameter_label" label="Enter this parameter's label for the form" help="Note that '~~~' is an internal delimiter so must not appear in this text field - please work around this technical limitation" > <sanitizer invalid_char=""> <valid initial="string.printable"> <remove value='~~~'/> </valid> @@ -80,23 +78,25 @@ </sanitizer> </param> <param name="param_help" type="text" value="parameter_help" label="Help for this parameter" - help="Note that three consecutive ~ cannot be used in this text field - please work around this technical limitation" size="60"> + help="Note that three consecutive ~ cannot be used in this text field - please work around this technical limitation" > <sanitizer invalid_char=""> <valid initial="string.printable"> <remove value='~~~'/> </valid> <mapping initial="none"/> </sanitizer> </param> - <param name="param_CL" type="text" size="60" label="Positional order (must be integer > 1) or Argument name to be prefixed with '--' or '-' if single letter" - help="Using positional parameters, enter the integer ordinal for this parameter on the command line. Using Argparse style, '--' will be prepended on the CL" value=""> - </param> + <param name="param_CL" type="text" label="Positional ordinal or argparse argument name" + help="Using positional parameters, enter the integer ordinal for this parameter on the command line. Using Argparse style, '--' will be prepended on the CL" value="" /> + <param name="param_CLprefixed" type="text" label="Override the generated default argparse name prefix if not empty - eg ----foo if needed" + help="Some targets like Planemo expect an unadorned action like 'test' before --galaxy_root and so on. " value="" /> + </repeat> </xml> <xml name="builtin_dynpar"> - <param name="interpreter_version" type="text" value="" + <param name="interpreter_version" type="text" value="" label="Specific interpreter version to match dependency (Conda) repositories - e.g. for python '3.8.5' Latest if empty"/> <param name="exe_package_version" type="text" value="" label="Executable package version to match dependency (Conda) repositories - e.g. for bwa='0.7.17'. Latest if empty"/> - <param name="dynScript" label="Cut and paste the script to be executed here" type="text" value="Script goes here" area="True" + <param name="dynScript" label="Cut and paste the script to be executed here" type="text" value="Script goes here" area="True" help="If there are parameters, either positional or argparse style, the script must parse and use them appropriately"> <sanitizer> <valid initial="string.printable"> @@ -108,9 +108,9 @@ <xml name="dynpar"> <param name="interpreter_version" type="text" value="" label="Specific interpreter version to match dependency (Conda) repositories - e.g. for python '3.8.5' Latest if empty"/> - <param name="exe_package_version" type="text" value="" + <param name="exe_package_version" type="text" value="" label="Executable package version to match dependency (Conda) repositories - e.g. for bwa='0.7.17'. Latest if empty"/> - <param name="dynScript" label="Cut and paste the script to be executed here" type="text" value="Script goes here" area="True" + <param name="dynScript" label="Cut and paste the script to be executed here" type="text" value="Script goes here" area="True" help="If there are parameters, either positional or argparse style, the script must parse and use them appropriately"> <sanitizer> <valid initial="string.printable"> @@ -130,15 +130,15 @@ #if not $dev_env and ( $__user_email__ not in $__admin_users__ ): $__tool_directory__/rgToolFactory2.py --bad_user $__user_email__ #else: -$__tool_directory__/rgToolFactory2.py +$__tool_directory__/rgToolFactory2.py --runmode "$interexe.interpreter" #if str($interexe.interpreter)=="Executable" or str($interexe.interpreter)=="system": ---exe_package="$interexe.exe_package" - #if str($interexe.exe_package_version) != 'None' : ---exe_package_version="$interexe.exe_package_version" +--exe_package="$interexe.exe_package" + #if str($interexe.exe_package_version) != 'None' : +--exe_package_version="$interexe.exe_package_version" #end if #else: ---interpreter_name="$interexe.interpreter" --script_path "$runme" +--interpreter_name="$interexe.interpreter" --script_path "$runme" #if str($interexe.interpreter_version) != 'None': --interpreter_version="$interexe.interpreter_version" #end if @@ -153,10 +153,10 @@ #end if #if $ppass.parampass != '0': #if str($ppass.edit_params) == "yes": ---edit_additional_parameters +--edit_additional_parameters #end if #for apar in $ppass.additional_parameters: ---additional_parameters="$apar.param_name~~~$apar.param_value~~~$apar.param_label~~~$apar.param_help~~~$apar.param_type~~~$apar.param_CL" +--additional_parameters="$apar.param_name~~~$apar.param_value~~~$apar.param_label~~~$apar.param_help~~~$apar.param_type~~~$apar.param_CL~~~$apar.param_CLprefixed" #end for #end if @@ -168,7 +168,7 @@ #for $otab in $ppass.history_outputs: --output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL" #end for - #end if + #end if ]]></command> <configfiles> <configfile name="runme"> @@ -202,14 +202,14 @@ </configfiles> <inputs> - <param name="tool_name" type="text" value="tool1" label="New tool ID and title for outputs" size="60" + <param name="tool_name" type="text" value="tool1" label="New tool ID and title for outputs" help="This will become the toolshed repository name so choose thoughtfully to avoid namespace clashes with other tool writers. lower case, digits and underscores only"> <sanitizer invalid_char=""> <valid initial="string.letters,string.digits"> <add value="_"/> </valid> </sanitizer> - </param> + </param> <conditional name="interexe"> <param name="interpreter" type="select" label="For binaries, choose 'Executable'. Otherwise choose the interpreter for your code" help = "If executable, the supplied package will become a requirement so must match the tool dependency resolver package name - conda is the default."> @@ -223,11 +223,11 @@ <option value="specialtestcaseinterpreterpython" >for testing only - do not use me please</option> </param> <when value="Executable"> - <param name="exe_package" type="text" value="" label="Executable package name in dependency (Conda) repositories - e.g. 'bwa'" size="60" optional="false"/> - <param name="exe_package_version" type="text" value="" label="Executable package version to match dependency (Conda) repositories - e.g. for bwa='0.7.17'. Latest if empty" size="60"/> + <param name="exe_package" type="text" value="" label="Executable package name in dependency (Conda) repositories - e.g. 'bwa'" optional="false"/> + <param name="exe_package_version" type="text" value="" label="Executable package version to match dependency (Conda) repositories - e.g. for bwa='0.7.17'. Latest if empty" /> </when> <when value="system"> - <param name="exe_package" type="text" value="" label="System executable to run" size="60" optional="false"/> + <param name="exe_package" type="text" value="" label="System executable to run" optional="false"/> <param name="exe_package_version" type="text" value="" label="Executable package version to match dependency (Conda) repositories - e.g. for bwa='0.7.17'. Latest if empty"/> </when> @@ -253,9 +253,9 @@ <conditional name = "ppass"> <param name="parampass" type="select" display="radio" label="Command line parameter passing method to use"> - <option value="argparse" selected="true">Argparse style: on CL in the form of --inputbam="foo.bam" if label below is "inputbam"</option> - <option value="positional">Positional: on CL in the order defined on this screen -> "... foo.bam bar.idx zot.xls"</option> - <option value="0">No parameters needed because tool reads selected history file from STDIN and writes to STDOUT for history"</option> + <option value="argparse" selected="true">Argparse style: passed in the form of '--[clname] [value]'</option> + <option value="positional">Positional: Passed in the order of positional ordinals '...foo.bam bar.idx zot.xls'</option> + <option value="0">No parameters needed because tool reads selected input file from STDIN and writes STDOUT with new history output"</option> </param> <when value = "argparse"> <expand macro="io" /> @@ -268,22 +268,21 @@ <when value = "0"> <expand macro="io"/> </when> - </conditional> + </conditional> <conditional name="makeMode"> - <param name="make_Tool" type="select" display="radio" label="Generate new tool as a tar.gz file ready to upload to a toolshed repository" - help="Generate a toolshed archive - upload to a toolshed from where it can be auto-installed via the Galaxy admin functions" - size="60"> + <param name="make_Tool" type="select" display="radio" label="Generate new tool as a tar.gz file ready to upload to a toolshed repository" + help="Generate a toolshed archive - upload to a toolshed from where it can be auto-installed via the Galaxy admin functions" > <option value="yes">Generate a Galaxy ToolShed compatible toolshed.gz</option> <option value="" selected="true">No. Just run the script please</option> </param> <when value = "yes"> <param name="tool_version" label="Tool Version - bump this to warn users trying to redo old analyses" type="text" value="0.01" help="If you change your script and regenerate the 'same' tool, you should inform Galaxy (and users) by changing (bumping is traditional) this number"/> - <param name="tool_desc" label="Tool Description" type="text" value="" size="40" + <param name="tool_desc" label="Tool Description" type="text" value="" help="Supply a brief tool description for the Galaxy tool menu entry (optional - appears after the tool name)" /> - <param name="help_text" label="Tool form documentation and help text for users" type="text" area="true" - size="8x120" value="**What it Does**" - help="Supply the brief user documentation to appear on the new tool form as reStructured text - http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html" > + <param name="help_text" label="Tool form documentation and help text for users" type="text" area="true" + value="**What it Does**" + help="Supply the brief user documentation to appear on the new tool form as reStructured text - http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html" > <sanitizer> <valid initial="string.printable"> </valid> @@ -297,11 +296,11 @@ <option value="bibtex">BibTeX</option> </param> <when value="doi"> - <param name="doi" label="DOI" type="text" value="" + <param name="doi" label="DOI" type="text" value="" help="Supply a DOI (e.g. doi: 10.1111/j.1740-9713.2007.00258.x) that should be cited when this tool is used in published research." /> </when> <when value="bibtex"> - <param name="bibtex" label="BibTex" type="text" area="true" size="8x120" + <param name="bibtex" label="BibTex" type="text" area="true" help="Supply a BibTex entry that should be cited when this tool is used in published research." value="" > <sanitizer> <valid initial="string.printable"> @@ -315,15 +314,19 @@ </when> <when value = ""> </when> - </conditional> + </conditional> </inputs> <outputs> - <collection name="ToolFactory_Outputs" type="list" label="Toolfactory ${tool_name} test run outputs" > - <discover_datasets pattern="__name__" directory="tfout/test-data" format="txt"/> - </collection> - <data format="tgz" name="new_tool" label="${tool_name}_toolshed.tgz"> + + <data format="tgz" name="new_tool" label="${tool_name}_toolshed.tgz" > <filter>makeMode['make_Tool'] == "yes"</filter> </data> + + <collection name="TF_run_report" type="list" label="${tool_name} outputs"> + <discover_datasets pattern="__name__" directory="TF_run_report_tempdir" format="txt"/> + </collection> + + </outputs> <tests> <test> @@ -342,27 +345,25 @@ <param name="history_name" value="output2_sample" /> <param name="history_format" value="txt" /> <param name="history_CL" value="2" /> - <param name="dynScript" value="import sys; inp = sys.argv[1]; outp = sys.argv[2]; inlist = open(inp,'r').readlines(); o = open(outp,'w'); rs = [''.join(list(reversed(x.rstrip()))) for x in inlist]; o.write('\n'.join(rs)); o.close()"/> - <output_collection name="ToolFactory_Outputs" type="list"> - <element name="output2_sample_sample" file="output2_sample" ftype="txt" compare="diff" lines_diff = "10" /> - </output_collection> + <param name="dynScript" value="import sys; inp = sys.argv[1]; outp = sys.argv[2]; inlist = open(inp,'r').readlines(); o = open(outp,'w'); rs = [''.join(list(reversed(x.rstrip()))) for x in inlist]; o.write('\n'.join(rs)); o.close()"/> <output name="new_tool" file="toolfactory_pyrevpos_tgz_sample" compare="sim_size" delta="6000" /> + <!-- <output name="output" file="output2_sample" ftype="txt" compare="diff" lines_diff = "10" /> --> </test> </tests> <help> .. class:: warningmark -**Details and attribution** +**Details and attribution** (see GTF_) -**Local Admins ONLY** +**Local Admins ONLY** Only users whose IDs found in the local admin_user configuration setting in universe_wsgi.ini can run this tool. -**If you find a bug** +**If you find a bug** Please raise an issue, or even better, submit a pull request fixing it, on the github repository GTF_ -**What it does** +**What it does** This tool optionally generates normal workflow compatible first class Galaxy tools Generated tools can run existing binary packages that become requirements, existing scripts, or new scripts pasted into this tool form. @@ -382,7 +383,7 @@ .. class:: warningmark -**Note to system administrators** +**Note to system administrators** This tool offers *NO* built in protection against malicious scripts. It should only be installed on private/personnal Galaxy instances. Admin_users will have the power to do anything they want as the Galaxy user if you install this tool. @@ -412,7 +413,7 @@ o.close() With argparse style parameters: - + :: # reverse order of text by row @@ -433,7 +434,7 @@ o.write(''.join(rs)) o.write('\n') o.close() - + Paper_ : @@ -441,7 +442,7 @@ Ross Lazarus; Antony Kaspi; Mark Ziemann; The Galaxy Team Bioinformatics 2012; doi: 10.1093/bioinformatics/bts573 -**Licensing** +**Licensing** Copyright Ross Lazarus (ross period lazarus at gmail period com) May 2012 All rights reserved.