Mercurial > repos > fubar > tool_factory_2
diff toolfactory/rgToolFactory2.py @ 126:def0f754ee1b draft
Uploaded
| author | fubar |
|---|---|
| date | Fri, 26 Mar 2021 09:29:58 +0000 |
| parents | f267ed4a3026 |
| children | 98c8a76b3638 |
line wrap: on
line diff
--- a/toolfactory/rgToolFactory2.py Fri Mar 26 09:27:55 2021 +0000 +++ b/toolfactory/rgToolFactory2.py Fri Mar 26 09:29:58 2021 +0000 @@ -16,6 +16,7 @@ import argparse import copy +import json import logging import os import re @@ -36,44 +37,11 @@ import yaml -myversion = "V2.1 July 2020" + +myversion = "V2.2 February 2021" verbose = True debug = True toolFactoryURL = "https://github.com/fubar2/toolfactory" -ourdelim = "~~~" - -# --input_files="$intab.input_files~~~$intab.input_CL~~~ -# $intab.input_formats# ~~~$intab.input_label -# ~~~$intab.input_help" -IPATHPOS = 0 -ICLPOS = 1 -IFMTPOS = 2 -ILABPOS = 3 -IHELPOS = 4 -IOCLPOS = 5 - -# --output_files "$otab.history_name~~~$otab.history_format~~~ -# $otab.history_CL~~~$otab.history_test" -ONAMEPOS = 0 -OFMTPOS = 1 -OCLPOS = 2 -OTESTPOS = 3 -OOCLPOS = 4 - - -# --additional_parameters="$i.param_name~~~$i.param_value~~~ -# $i.param_label~~~$i.param_help~~~$i.param_type -# ~~~$i.CL~~~i$.param_CLoverride" -ANAMEPOS = 0 -AVALPOS = 1 -ALABPOS = 2 -AHELPPOS = 3 -ATYPEPOS = 4 -ACLPOS = 5 -AOVERPOS = 6 -AOCLPOS = 7 - - foo = len(lxml.__version__) # fug you, flake8. Say my name! FAKEEXE = "~~~REMOVE~~~ME~~~" @@ -86,32 +54,9 @@ return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) -def quote_non_numeric(s): - """return a prequoted string for non-numerics - useful for perl and Rscript parameter passing? - """ - try: - _ = float(s) - return s - except ValueError: - return '"%s"' % s - - -html_escape_table = { - "&": "&", - ">": ">", - "<": "<", - "#": "#", - "$": "$", -} cheetah_escape_table = {"$": "\\$", "#": "\\#"} -def html_escape(text): - """Produce entities within text.""" - return "".join([html_escape_table.get(c, c) for c in text]) - - def cheetah_escape(text): """Produce entities within text.""" return "".join([cheetah_escape_table.get(c, c) for c in text]) @@ -141,10 +86,48 @@ and prepare elements needed for galaxyxml tool generation """ self.ourcwd = os.getcwd() - self.ourenv = copy.deepcopy(os.environ) - self.infiles = [x.split(ourdelim) for x in args.input_files] - self.outfiles = [x.split(ourdelim) for x in args.output_files] - self.addpar = [x.split(ourdelim) for x in args.additional_parameters] + self.collections = [] + if len(args.collection) > 0: + try: + self.collections = [ + json.loads(x) for x in args.collection if len(x.strip()) > 1 + ] + except Exception: + print( + f"--collections parameter {str(args.collection)} is malformed - should be a dictionary" + ) + try: + self.infiles = [ + json.loads(x) for x in args.input_files if len(x.strip()) > 1 + ] + except Exception: + print( + f"--input_files parameter {str(args.input_files)} is malformed - should be a dictionary" + ) + try: + self.outfiles = [ + json.loads(x) for x in args.output_files if len(x.strip()) > 1 + ] + except Exception: + print( + f"--output_files parameter {args.output_files} is malformed - should be a dictionary" + ) + try: + self.addpar = [ + json.loads(x) for x in args.additional_parameters if len(x.strip()) > 1 + ] + except Exception: + print( + f"--additional_parameters {args.additional_parameters} is malformed - should be a dictionary" + ) + try: + self.selpar = [ + json.loads(x) for x in args.selecttext_parameters if len(x.strip()) > 1 + ] + except Exception: + print( + f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary" + ) self.args = args self.cleanuppar() self.lastclredirect = None @@ -153,10 +136,13 @@ self.xmlcl = [] self.is_positional = self.args.parampass == "positional" if self.args.sysexe: - self.executeme = self.args.sysexe + if ' ' in self.args.sysexe: + self.executeme = self.args.sysexe.split(' ') + else: + self.executeme = [self.args.sysexe, ] else: if self.args.packages: - self.executeme = self.args.packages.split(",")[0].split(":")[0].strip() + self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ] else: self.executeme = None aCL = self.cl.append @@ -175,7 +161,7 @@ self.args.tool_desc, FAKEEXE, ) - self.newtarpath = "toolfactory_%s.tgz" % self.tool_name + self.newtarpath = "%s_toolshed.gz" % self.tool_name self.tooloutdir = "./tfout" self.repdir = "./TF_run_report_tempdir" self.testdir = os.path.join(self.tooloutdir, "test-data") @@ -200,67 +186,122 @@ self.test_override = [x.rstrip() for x in stos] else: self.test_override = None - if self.args.cl_prefix: # DIY CL start + if self.args.script_path: + for ex in self.executeme: + aCL(ex) + aXCL(ex) + aCL(self.sfile) + aXCL("$runme") + else: + aCL(self.executeme[0]) + aXCL(self.executeme[0]) + self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name) + self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name) + if self.args.parampass == "0": + self.clsimple() + else: + if self.args.parampass == "positional": + self.prepclpos() + self.clpositional() + else: + self.prepargp() + self.clargparse() + if self.args.cl_prefix: # DIY CL end - misnamed! clp = self.args.cl_prefix.split(" ") for c in clp: aCL(c) aXCL(c) - else: - if self.args.script_path: - aCL(self.executeme) - aCL(self.sfile) - aXCL(self.executeme) - aXCL("$runme") - else: - aCL(self.executeme) - aXCL(self.executeme) - self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name) - self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name) + + def clsimple(self): + """no parameters - uses < and > for i/o""" + aCL = self.cl.append + aXCL = self.xmlcl.append + if len(self.infiles) > 0: + aCL("<") + aCL(self.infiles[0]["infilename"]) + aXCL("<") + aXCL("$%s" % self.infiles[0]["infilename"]) + if len(self.outfiles) > 0: + aCL(">") + aCL(self.outfiles[0]["name"]) + aXCL(">") + aXCL("$%s" % self.outfiles[0]["name"]) - if self.args.parampass == "0": - self.clsimple() - else: - clsuffix = [] - xclsuffix = [] - for i, p in enumerate(self.infiles): - if p[IOCLPOS].upper() == "STDIN": - appendme = [ - p[ICLPOS], - p[ICLPOS], - p[IPATHPOS], - "< %s" % p[IPATHPOS], - ] - xappendme = [ - p[ICLPOS], - p[ICLPOS], - p[IPATHPOS], - "< $%s" % p[ICLPOS], - ] - else: - appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""] - xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] - clsuffix.append(appendme) - xclsuffix.append(xappendme) - for i, p in enumerate(self.outfiles): - if p[OOCLPOS] == "STDOUT": - self.lastclredirect = [">", p[ONAMEPOS]] - self.lastxclredirect = [">", "$%s" % p[OCLPOS]] - else: - clsuffix.append([p[OCLPOS], p[ONAMEPOS], p[ONAMEPOS], ""]) - xclsuffix.append([p[OCLPOS], p[ONAMEPOS], "$%s" % p[ONAMEPOS], ""]) - for p in self.addpar: - clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]]) - xclsuffix.append( - [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] - ) - clsuffix.sort() - xclsuffix.sort() - self.xclsuffix = xclsuffix - self.clsuffix = clsuffix - if self.args.parampass == "positional": - self.clpositional() + def prepargp(self): + clsuffix = [] + xclsuffix = [] + for i, p in enumerate(self.infiles): + if p["origCL"].strip().upper() == "STDIN": + appendme = [ + p["infilename"], + p["infilename"], + "< %s" % p["infilename"], + ] + xappendme = [ + p["infilename"], + p["infilename"], + "< $%s" % p["infilename"], + ] + else: + appendme = [p["CL"], p["CL"], ""] + xappendme = [p["CL"], "$%s" % p["CL"], ""] + clsuffix.append(appendme) + xclsuffix.append(xappendme) + for i, p in enumerate(self.outfiles): + if p["origCL"].strip().upper() == "STDOUT": + self.lastclredirect = [">", p["name"]] + self.lastxclredirect = [">", "$%s" % p["name"]] else: - self.clargparse() + clsuffix.append([p["name"], p["name"], ""]) + xclsuffix.append([p["name"], "$%s" % p["name"], ""]) + for p in self.addpar: + clsuffix.append([p["CL"], p["name"], p["override"]]) + xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) + for p in self.selpar: + clsuffix.append([p["CL"], p["name"], p["override"]]) + xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) + clsuffix.sort() + xclsuffix.sort() + self.xclsuffix = xclsuffix + self.clsuffix = clsuffix + + def prepclpos(self): + clsuffix = [] + xclsuffix = [] + for i, p in enumerate(self.infiles): + if p["origCL"].strip().upper() == "STDIN": + appendme = [ + "999", + p["infilename"], + "< $%s" % p["infilename"], + ] + xappendme = [ + "999", + p["infilename"], + "< $%s" % p["infilename"], + ] + else: + appendme = [p["CL"], p["infilename"], ""] + xappendme = [p["CL"], "$%s" % p["infilename"], ""] + clsuffix.append(appendme) + xclsuffix.append(xappendme) + for i, p in enumerate(self.outfiles): + if p["origCL"].strip().upper() == "STDOUT": + self.lastclredirect = [">", p["name"]] + self.lastxclredirect = [">", "$%s" % p["name"]] + else: + clsuffix.append([p["CL"], p["name"], ""]) + xclsuffix.append([p["CL"], "$%s" % p["name"], ""]) + for p in self.addpar: + clsuffix.append([p["CL"], p["name"], p["override"]]) + xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) + for p in self.selpar: + clsuffix.append([p["CL"], p["name"], p["override"]]) + xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) + clsuffix.sort() + xclsuffix.sort() + self.xclsuffix = xclsuffix + self.clsuffix = clsuffix def prepScript(self): rx = open(self.args.script_path, "r").readlines() @@ -269,14 +310,14 @@ assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" self.script = "\n".join(rx) fhandle, self.sfile = tempfile.mkstemp( - prefix=self.tool_name, suffix="_%s" % (self.executeme) + prefix=self.tool_name, suffix="_%s" % (self.executeme[0]) ) tscript = open(self.sfile, "w") tscript.write(self.script) tscript.close() self.escapedScript = [cheetah_escape(x) for x in rx] self.spacedScript = [f" {x}" for x in rx if x.strip() > ""] - art = "%s.%s" % (self.tool_name, self.executeme) + art = "%s.%s" % (self.tool_name, self.executeme[0]) artifact = open(art, "wb") artifact.write(bytes("\n".join(self.escapedScript), "utf8")) artifact.close() @@ -286,78 +327,50 @@ if self.args.parampass == "positional": for i, p in enumerate(self.infiles): assert ( - p[ICLPOS].isdigit() or p[ICLPOS].strip().upper() == "STDIN" + p["CL"].isdigit() or p["CL"].strip().upper() == "STDIN" ), "Positional parameters must be ordinal integers - got %s for %s" % ( - p[ICLPOS], - p[ILABPOS], + p["CL"], + p["label"], ) for i, p in enumerate(self.outfiles): assert ( - p[OCLPOS].isdigit() or p[OCLPOS].strip().upper() == "STDOUT" + p["CL"].isdigit() or p["CL"].strip().upper() == "STDOUT" ), "Positional parameters must be ordinal integers - got %s for %s" % ( - p[OCLPOS], - p[ONAMEPOS], + p["CL"], + p["name"], ) for i, p in enumerate(self.addpar): assert p[ - ACLPOS + "CL" ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % ( - p[ACLPOS], - p[ANAMEPOS], + p["CL"], + p["name"], ) for i, p in enumerate(self.infiles): infp = copy.copy(p) - icl = infp[ICLPOS] - infp.append(icl) - if ( - infp[ICLPOS].isdigit() - or self.args.parampass == "0" - or infp[ICLPOS].strip().upper() == "STDOUT" - ): - scl = "input%d" % (i + 1) - infp[ICLPOS] = scl + infp["origCL"] = infp["CL"] + if self.args.parampass in ["positional", "0"]: + infp["infilename"] = infp["label"].replace(" ", "_") + else: + infp["infilename"] = infp["CL"] self.infiles[i] = infp for i, p in enumerate(self.outfiles): - p.append(p[OCLPOS]) # keep copy - if (p[OOCLPOS].isdigit() and self.args.parampass != "positional") or p[ - OOCLPOS - ].strip().upper() == "STDOUT": - scl = p[ONAMEPOS] - p[OCLPOS] = scl + p["origCL"] = p["CL"] # keep copy self.outfiles[i] = p for i, p in enumerate(self.addpar): - p.append(p[ACLPOS]) - if p[ACLPOS].isdigit(): - scl = "param%s" % p[ACLPOS] - p[ACLPOS] = scl + p["origCL"] = p["CL"] self.addpar[i] = p - def clsimple(self): - """no parameters - uses < and > for i/o""" - aCL = self.cl.append - aXCL = self.xmlcl.append - - if len(self.infiles) > 0: - aCL("<") - aCL(self.infiles[0][IPATHPOS]) - aXCL("<") - aXCL("$%s" % self.infiles[0][ICLPOS]) - if len(self.outfiles) > 0: - aCL(">") - aCL(self.outfiles[0][OCLPOS]) - aXCL(">") - aXCL("$%s" % self.outfiles[0][ONAMEPOS]) - def clpositional(self): # inputs in order then params aCL = self.cl.append - for (o_v, k, v, koverride) in self.clsuffix: + for (k, v, koverride) in self.clsuffix: if " " in v: aCL("%s" % v) else: aCL(v) aXCL = self.xmlcl.append - for (o_v, k, v, koverride) in self.xclsuffix: + for (k, v, koverride) in self.xclsuffix: aXCL(v) if self.lastxclredirect: aXCL(self.lastxclredirect[0]) @@ -369,7 +382,7 @@ aXCL = self.xmlcl.append # inputs then params in argparse named form - for (o_v, k, v, koverride) in self.xclsuffix: + for (k, v, koverride) in self.xclsuffix: if koverride > "": k = koverride elif len(k.strip()) == 1: @@ -378,7 +391,7 @@ k = "--%s" % k aXCL(k) aXCL(v) - for (o_v, k, v, koverride) in self.clsuffix: + for (k, v, koverride) in self.clsuffix: if koverride > "": k = koverride elif len(k.strip()) == 1: @@ -400,8 +413,11 @@ def doXMLparam(self): """flake8 made me do this...""" for p in self.outfiles: - # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL~~~$otab.history_test" - newname, newfmt, newcl, test, oldcl = p + newname = p["name"] + newfmt = p["format"] + newcl = p["CL"] + test = p["test"] + oldcl = p["origCL"] test = test.strip() ndash = self.getNdash(newcl) aparm = gxtp.OutputData( @@ -446,39 +462,50 @@ delta=delta, delta_frac=delta_frac, ) + else: + c = test + tp = gxtp.TestOutput( + name=newname, + value="%s_sample" % newname, + compare=c, + ) self.testparam.append(tp) for p in self.infiles: - newname = p[ICLPOS] - newfmt = p[IFMTPOS] + newname = p["infilename"] + newfmt = p["format"] ndash = self.getNdash(newname) - if not len(p[ILABPOS]) > 0: - alab = p[ICLPOS] + if not len(p["label"]) > 0: + alab = p["CL"] else: - alab = p[ILABPOS] + alab = p["label"] aninput = gxtp.DataParam( newname, optional=False, label=alab, - help=p[IHELPOS], + help=p["help"], format=newfmt, multiple=False, num_dashes=ndash, ) aninput.positional = self.is_positional + if self.is_positional: + if p["origCL"].upper() == "STDIN": + aparm.positional = 9999998 + aparm.command_line_override = "> $%s" % newname + else: + aparm.positional = int(p["origCL"]) + aparm.command_line_override = "$%s" % newname self.tinputs.append(aninput) tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname) self.testparam.append(tparm) for p in self.addpar: - ( - newname, - newval, - newlabel, - newhelp, - newtype, - newcl, - override, - oldcl, - ) = p + newname = p["name"] + newval = p["value"] + newlabel = p["label"] + newhelp = p["help"] + newtype = p["type"] + newcl = p["CL"] + oldcl = p["origCL"] if not len(newlabel) > 0: newlabel = newname ndash = self.getNdash(newname) @@ -506,6 +533,14 @@ value=newval, num_dashes=ndash, ) + elif newtype == "boolean": + aparm = gxtp.BooleanParam( + newname, + label=newname, + help=newhelp, + value=newval, + num_dashes=ndash, + ) else: raise ValueError( 'Unrecognised parameter type "%s" for\ @@ -518,43 +553,90 @@ self.tinputs.append(aparm) tparm = gxtp.TestParam(newname, value=newval) self.testparam.append(tparm) + for p in self.selpar: + newname = p["name"] + newval = p["value"] + newlabel = p["label"] + newhelp = p["help"] + newtype = p["type"] + newcl = p["CL"] + if not len(newlabel) > 0: + newlabel = newname + ndash = self.getNdash(newname) + if newtype == "selecttext": + newtext = p["texts"] + aparm = gxtp.SelectParam( + newname, + label=newlabel, + help=newhelp, + num_dashes=ndash, + ) + for i in range(len(newval)): + anopt = gxtp.SelectOption( + value=newval[i], + text=newtext[i], + ) + aparm.append(anopt) + aparm.positional = self.is_positional + if self.is_positional: + aparm.positional = int(newcl) + self.tinputs.append(aparm) + tparm = gxtp.TestParam(newname, value=newval) + self.testparam.append(tparm) + else: + raise ValueError( + 'Unrecognised parameter type "%s" for\ + selecttext parameter %s in makeXML' + % (newtype, newname) + ) + for p in self.collections: + newkind = p["kind"] + newname = p["name"] + newlabel = p["label"] + newdisc = p["discover"] + collect = gxtp.OutputCollection(newname, label=newlabel, type=newkind) + disc = gxtp.DiscoverDatasets( + pattern=newdisc, directory=f"{newname}", visible="false" + ) + collect.append(disc) + self.toutputs.append(collect) + tparm = gxtp.TestOutput(newname, ftype="pdf") + self.testparam.append(tparm) def doNoXMLparam(self): """filter style package - stdin to stdout""" if len(self.infiles) > 0: - alab = self.infiles[0][ILABPOS] + alab = self.infiles[0]["label"] if len(alab) == 0: - alab = self.infiles[0][ICLPOS] + alab = self.infiles[0]["infilename"] max1s = ( "Maximum one input if parampass is 0 but multiple input files supplied - %s" % str(self.infiles) ) assert len(self.infiles) == 1, max1s - newname = self.infiles[0][ICLPOS] + newname = self.infiles[0]["infilename"] aninput = gxtp.DataParam( newname, optional=False, label=alab, - help=self.infiles[0][IHELPOS], - format=self.infiles[0][IFMTPOS], + help=self.infiles[0]["help"], + format=self.infiles[0]["format"], multiple=False, num_dashes=0, ) aninput.command_line_override = "< $%s" % newname - aninput.positional = self.is_positional + aninput.positional = True self.tinputs.append(aninput) tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) self.testparam.append(tp) if len(self.outfiles) > 0: - newname = self.outfiles[0][OCLPOS] - newfmt = self.outfiles[0][OFMTPOS] + newname = self.outfiles[0]["name"] + newfmt = self.outfiles[0]["format"] anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) anout.command_line_override = "> $%s" % newname anout.positional = self.is_positional self.toutputs.append(anout) - tp = gxtp.TestOutput( - name=newname, value="%s_sample" % newname - ) + tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname) self.testparam.append(tp) def makeXML(self): @@ -639,7 +721,7 @@ ): # cannot do this inside galaxyxml as it expects lxml objects for tests part1 = exml.split("<tests>")[0] part2 = exml.split("</tests>")[1] - fixed = "%s\n%s\n%s" % (part1, self.test_override, part2) + fixed = "%s\n%s\n%s" % (part1, "\n".join(self.test_override), part2) exml = fixed # exml = exml.replace('range="1:"', 'range="1000:"') xf = open("%s.xml" % self.tool_name, "w") @@ -674,22 +756,22 @@ ) sto.flush() subp = subprocess.run( - self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste + self.cl, shell=False, stdout=sto, stderr=ste ) sto.close() ste.close() retval = subp.returncode else: # work around special case - stdin and write to stdout if len(self.infiles) > 0: - sti = open(self.infiles[0][IPATHPOS], "rb") + sti = open(self.infiles[0]["name"], "rb") else: sti = sys.stdin if len(self.outfiles) > 0: - sto = open(self.outfiles[0][ONAMEPOS], "wb") + sto = open(self.outfiles[0]["name"], "wb") else: sto = sys.stdout subp = subprocess.run( - self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti + self.cl, shell=False, stdout=sto, stdin=sti ) sto.write("## Executing Toolfactory generated command line = %s\n" % scl) retval = subp.returncode @@ -785,7 +867,6 @@ tout.write("running\n%s\n" % " ".join(cll)) subp = subprocess.run( cll, - env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, @@ -815,20 +896,25 @@ def makeTool(self): """write xmls and input samples into place""" - self.makeXML() + if self.args.parampass == 0: + self.doNoXMLparam() + else: + self.makeXML() if self.args.script_path: - stname = os.path.join(self.tooloutdir, "%s" % (self.sfile)) + stname = os.path.join(self.tooloutdir, self.sfile) if not os.path.exists(stname): shutil.copyfile(self.sfile, stname) xreal = "%s.xml" % self.tool_name xout = os.path.join(self.tooloutdir, xreal) shutil.copyfile(xreal, xout) for p in self.infiles: - pth = p[IPATHPOS] - dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS]) + pth = p["name"] + dest = os.path.join(self.testdir, "%s_sample" % p["infilename"]) + shutil.copyfile(pth, dest) + dest = os.path.join(self.repdir, "%s_sample" % p["infilename"]) shutil.copyfile(pth, dest) - def makeToolTar(self): + def makeToolTar(self, report_fail=False): """move outputs into test-data and prepare the tarball""" excludeme = "_planemo_test_report.html" @@ -841,19 +927,20 @@ else: tout = open(self.tlog, "w") for p in self.outfiles: - oname = p[ONAMEPOS] + oname = p["name"] tdest = os.path.join(self.testdir, "%s_sample" % oname) + src = os.path.join(self.testdir, oname) if not os.path.isfile(tdest): - src = os.path.join(self.testdir, oname) if os.path.isfile(src): shutil.copyfile(src, tdest) dest = os.path.join(self.repdir, "%s.sample" % (oname)) shutil.copyfile(src, dest) else: - tout.write( - "###Output file %s not found in testdir %s. This is normal during the first Planemo run that generates test outputs" - % (tdest, self.testdir) - ) + if report_fail: + tout.write( + "###Tool may have failed - output file %s not found in testdir after planemo run %s." + % (tdest, self.testdir) + ) tf = tarfile.open(self.newtarpath, "w:gz") tf.add( name=self.tooloutdir, @@ -870,33 +957,41 @@ if not entry.is_file(): continue if "." in entry.name: - nayme, ext = os.path.splitext(entry.name) - if ext in [".yml", ".xml", ".json", ".yaml"]: - ext = f"{ext}.txt" + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name else: - ext = ".txt" - ofn = "%s%s" % (entry.name.replace(".", "_"), ext) - dest = os.path.join(self.repdir, ofn) + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) src = os.path.join(self.tooloutdir, entry.name) shutil.copyfile(src, dest) - with os.scandir(self.testdir) as outs: - for entry in outs: - if ( - (not entry.is_file()) - or entry.name.endswith("_sample") - or entry.name.endswith("_planemo_test_report.html") - ): - continue - if "." in entry.name: - nayme, ext = os.path.splitext(entry.name) - else: - ext = ".txt" - newname = f"{entry.name}{ext}" - dest = os.path.join(self.repdir, newname) - src = os.path.join(self.testdir, entry.name) - shutil.copyfile(src, dest) + if self.args.include_tests: + with os.scandir(self.testdir) as outs: + for entry in outs: + if (not entry.is_file()) or entry.name.endswith( + "_planemo_test_report.html" + ): + continue + if "." in entry.name: + _, ext = os.path.splitext(entry.name) + if ext in [".tgz", ".json"]: + continue + if ext in [".yml", ".xml", ".yaml"]: + newname = f"{entry.name.replace('.','_')}.txt" + else: + newname = entry.name + else: + newname = f"{entry.name}.txt" + dest = os.path.join(self.repdir, newname) + src = os.path.join(self.testdir, entry.name) + shutil.copyfile(src, dest) - def planemo_test(self, genoutputs=True): + + def planemo_test_once(self): """planemo is a requirement so is available for testing but needs a different call if in the biocontainer - see above and for generating test outputs if command or test overrides are @@ -910,51 +1005,26 @@ tout = open(self.tlog, "a") else: tout = open(self.tlog, "w") - if genoutputs: - dummy, tfile = tempfile.mkstemp() - cll = [ - "planemo", - "test", - "--test_data", - os.path.abspath(self.testdir), - "--test_output", - os.path.abspath(tool_test_path), - "--skip_venv", - "--galaxy_root", - self.args.galaxy_root, - "--update_test_data", - os.path.abspath(xreal), - ] - p = subprocess.run( - cll, - env=self.ourenv, - shell=False, - cwd=self.tooloutdir, - stderr=dummy, - stdout=dummy, - ) - - else: - cll = [ - "planemo", - "test", - "--test_data", - os.path.abspath(self.testdir), - "--test_output", - os.path.abspath(tool_test_path), - "--skip_venv", - "--galaxy_root", - self.args.galaxy_root, - os.path.abspath(xreal), - ] - p = subprocess.run( - cll, - shell=False, - env=self.ourenv, - cwd=self.tooloutdir, - stderr=tout, - stdout=tout, - ) + cll = [ + "planemo", + "test", + "--conda_auto_init", + "--test_data", + os.path.abspath(self.testdir), + "--test_output", + os.path.abspath(tool_test_path), + "--galaxy_root", + self.args.galaxy_root, + "--update_test_data", + os.path.abspath(xreal), + ] + p = subprocess.run( + cll, + shell=False, + cwd=self.tooloutdir, + stderr=tout, + stdout=tout, + ) tout.close() return p.returncode @@ -986,6 +1056,7 @@ a("--command_override", default=None) a("--test_override", default=None) a("--additional_parameters", action="append", default=[]) + a("--selecttext_parameters", action="append", default=[]) a("--edit_additional_parameters", action="store_true", default=False) a("--parampass", default="positional") a("--tfout", default="./tfout") @@ -998,6 +1069,8 @@ a("--galaxy_api_key", default="fakekey") a("--galaxy_root", default="/galaxy-central") a("--galaxy_venv", default="/galaxy_venv") + a("--collection", action="append", default=[]) + a("--include_tests", default=False, action="store_true") args = parser.parse_args() assert not args.bad_user, ( 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \ @@ -1009,25 +1082,20 @@ args.sysexe or args.packages ), "## Tool Factory wrapper expects an interpreter \ or an executable package in --sysexe or --packages" - args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files] - # remove quotes we need to deal with spaces in CL params - for i, x in enumerate(args.additional_parameters): - args.additional_parameters[i] = args.additional_parameters[i].replace('"', "") r = ScriptRunner(args) r.writeShedyml() r.makeTool() if args.make_Tool == "generate": - retcode = r.run() + r.run() r.moveRunOutputs() r.makeToolTar() else: - retcode = r.planemo_test(genoutputs=True) # this fails :( - see PR + # r.planemo_test(genoutputs=True) # this fails :( - see PR + # r.moveRunOutputs() + # r.makeToolTar(report_fail=False) + r.planemo_test_once() r.moveRunOutputs() - r.makeToolTar() - retcode = r.planemo_test(genoutputs=False) - r.moveRunOutputs() - r.makeToolTar() - print(f"second planemo_test returned {retcode}") + r.makeToolTar(report_fail=True) if args.make_Tool == "gentestinstall": r.shedLoad() r.eph_galaxy_load()