Mercurial > repos > fubar > tacrev
comparison pyrevpos/test-data/input1_sample @ 0:bf92b6bd8b20 draft
Uploaded
| author | fubar |
|---|---|
| date | Tue, 24 Nov 2020 21:59:01 +0000 |
| parents | |
| children |
comparison
equal
deleted
inserted
replaced
| -1:000000000000 | 0:bf92b6bd8b20 |
|---|---|
| 1 #!/usr/bin/env python | |
| 2 # rgToolFactory.py | |
| 3 # see https://github.com/fubar2/toolfactory | |
| 4 # | |
| 5 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012 | |
| 6 # | |
| 7 # all rights reserved | |
| 8 # Licensed under the LGPL | |
| 9 # suggestions for improvement and bug fixes welcome at https://github.com/fubar2/toolfactory | |
| 10 # | |
| 11 # July 2020: BCC was fun and I feel like rip van winkle after 5 years. | |
| 12 # Decided to | |
| 13 # 1. Fix the toolfactory so it works - done for simplest case | |
| 14 # 2. Fix planemo so the toolfactory function works | |
| 15 # 3. Rewrite bits using galaxyxml functions where that makes sense - done | |
| 16 # | |
| 17 # removed all the old complications including making the new tool use this same script | |
| 18 # galaxyxml now generates the tool xml https://github.com/hexylena/galaxyxml | |
| 19 # No support for automatic HTML file creation from arbitrary outputs | |
| 20 # TODO: add option to run that code as a post execution hook | |
| 21 # TODO: add additional history input parameters - currently only one | |
| 22 | |
| 23 | |
| 24 import argparse | |
| 25 import logging | |
| 26 import os | |
| 27 import re | |
| 28 import shutil | |
| 29 import subprocess | |
| 30 import sys | |
| 31 import tarfile | |
| 32 import tempfile | |
| 33 import time | |
| 34 | |
| 35 import galaxyxml.tool as gxt | |
| 36 import galaxyxml.tool.parameters as gxtp | |
| 37 | |
| 38 import lxml | |
| 39 | |
| 40 myversion = "V2.1 July 2020" | |
| 41 verbose = True | |
| 42 debug = True | |
| 43 toolFactoryURL = "https://github.com/fubar2/toolfactory" | |
| 44 ourdelim = "~~~" | |
| 45 | |
| 46 # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label | |
| 47 # ~~~$input_help" | |
| 48 IPATHPOS = 0 | |
| 49 ICLPOS = 1 | |
| 50 IFMTPOS = 2 | |
| 51 ILABPOS = 3 | |
| 52 IHELPOS = 4 | |
| 53 IOCLPOS = 5 | |
| 54 | |
| 55 # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL | |
| 56 ONAMEPOS = 0 | |
| 57 OFMTPOS = 1 | |
| 58 OCLPOS = 2 | |
| 59 OOCLPOS = 3 | |
| 60 | |
| 61 # --additional_parameters="$i.param_name~~~$i.param_value~~~ | |
| 62 # $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride" | |
| 63 ANAMEPOS = 0 | |
| 64 AVALPOS = 1 | |
| 65 ALABPOS = 2 | |
| 66 AHELPPOS = 3 | |
| 67 ATYPEPOS = 4 | |
| 68 ACLPOS = 5 | |
| 69 AOVERPOS = 6 | |
| 70 AOCLPOS = 7 | |
| 71 | |
| 72 | |
| 73 foo = len(lxml.__version__) | |
| 74 # fug you, flake8. Say my name! | |
| 75 | |
| 76 def timenow(): | |
| 77 """return current time as a string | |
| 78 """ | |
| 79 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) | |
| 80 | |
| 81 | |
| 82 def quote_non_numeric(s): | |
| 83 """return a prequoted string for non-numerics | |
| 84 useful for perl and Rscript parameter passing? | |
| 85 """ | |
| 86 try: | |
| 87 _ = float(s) | |
| 88 return s | |
| 89 except ValueError: | |
| 90 return '"%s"' % s | |
| 91 | |
| 92 | |
| 93 html_escape_table = {"&": "&", ">": ">", "<": "<", "$": r"\$"} | |
| 94 | |
| 95 | |
| 96 def html_escape(text): | |
| 97 """Produce entities within text.""" | |
| 98 return "".join(html_escape_table.get(c, c) for c in text) | |
| 99 | |
| 100 | |
| 101 def html_unescape(text): | |
| 102 """Revert entities within text. Multiple character targets so use replace""" | |
| 103 t = text.replace("&", "&") | |
| 104 t = t.replace(">", ">") | |
| 105 t = t.replace("<", "<") | |
| 106 t = t.replace("\\$", "$") | |
| 107 return t | |
| 108 | |
| 109 | |
| 110 def parse_citations(citations_text): | |
| 111 """ | |
| 112 """ | |
| 113 citations = [c for c in citations_text.split("**ENTRY**") if c.strip()] | |
| 114 citation_tuples = [] | |
| 115 for citation in citations: | |
| 116 if citation.startswith("doi"): | |
| 117 citation_tuples.append(("doi", citation[len("doi") :].strip())) | |
| 118 else: | |
| 119 citation_tuples.append( | |
| 120 ("bibtex", citation[len("bibtex") :].strip()) | |
| 121 ) | |
| 122 return citation_tuples | |
| 123 | |
| 124 | |
| 125 class ScriptRunner: | |
| 126 """Wrapper for an arbitrary script | |
| 127 uses galaxyxml | |
| 128 | |
| 129 """ | |
| 130 | |
| 131 def __init__(self, args=None): | |
| 132 """ | |
| 133 prepare command line cl for running the tool here | |
| 134 and prepare elements needed for galaxyxml tool generation | |
| 135 """ | |
| 136 | |
| 137 self.infiles = [x.split(ourdelim) for x in args.input_files] | |
| 138 self.outfiles = [x.split(ourdelim) for x in args.output_files] | |
| 139 self.addpar = [x.split(ourdelim) for x in args.additional_parameters] | |
| 140 self.args = args | |
| 141 self.cleanuppar() | |
| 142 self.lastclredirect = None | |
| 143 self.lastxclredirect = None | |
| 144 self.cl = [] | |
| 145 self.xmlcl = [] | |
| 146 self.is_positional = self.args.parampass == "positional" | |
| 147 aCL = self.cl.append | |
| 148 assert args.parampass in [ | |
| 149 "0", | |
| 150 "argparse", | |
| 151 "positional", | |
| 152 ], 'Parameter passing in args.parampass must be "0","positional" or "argparse"' | |
| 153 self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) | |
| 154 self.tool_id = self.tool_name | |
| 155 if self.args.interpreter_name: | |
| 156 exe = "$runMe" | |
| 157 else: | |
| 158 exe = self.args.exe_package | |
| 159 assert ( | |
| 160 exe is not None | |
| 161 ), "No interpeter or executable passed in - nothing to run so cannot build" | |
| 162 self.tool = gxt.Tool( | |
| 163 self.args.tool_name, | |
| 164 self.tool_id, | |
| 165 self.args.tool_version, | |
| 166 self.args.tool_desc, | |
| 167 exe, | |
| 168 ) | |
| 169 self.tinputs = gxtp.Inputs() | |
| 170 self.toutputs = gxtp.Outputs() | |
| 171 self.testparam = [] | |
| 172 if ( | |
| 173 self.args.runmode == "Executable" or self.args.runmode == "system" | |
| 174 ): # binary - no need | |
| 175 aCL(self.args.exe_package) # this little CL will just run | |
| 176 else: | |
| 177 self.prepScript() | |
| 178 self.elog = "%s_error_log.txt" % self.tool_name | |
| 179 self.tlog = "%s_runner_log.txt" % self.tool_name | |
| 180 | |
| 181 if self.args.parampass == "0": | |
| 182 self.clsimple() | |
| 183 else: | |
| 184 clsuffix = [] | |
| 185 xclsuffix = [] | |
| 186 for i, p in enumerate(self.infiles): | |
| 187 if p[IOCLPOS] == "STDIN": | |
| 188 appendme = [ | |
| 189 p[IOCLPOS], | |
| 190 p[ICLPOS], | |
| 191 p[IPATHPOS], | |
| 192 "< %s" % p[IPATHPOS], | |
| 193 ] | |
| 194 xappendme = [ | |
| 195 p[IOCLPOS], | |
| 196 p[ICLPOS], | |
| 197 p[IPATHPOS], | |
| 198 "< $%s" % p[ICLPOS], | |
| 199 ] | |
| 200 else: | |
| 201 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""] | |
| 202 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] | |
| 203 clsuffix.append(appendme) | |
| 204 xclsuffix.append(xappendme) | |
| 205 # print('##infile i=%d, appendme=%s' % (i,appendme)) | |
| 206 for i, p in enumerate(self.outfiles): | |
| 207 if p[OOCLPOS] == "STDOUT": | |
| 208 self.lastclredirect = [">", p[ONAMEPOS]] | |
| 209 self.lastxclredirect = [">", "$%s" % p[OCLPOS]] | |
| 210 else: | |
| 211 clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""]) | |
| 212 xclsuffix.append( | |
| 213 [p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""] | |
| 214 ) | |
| 215 for p in self.addpar: | |
| 216 clsuffix.append( | |
| 217 [p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]] | |
| 218 ) | |
| 219 xclsuffix.append( | |
| 220 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] | |
| 221 ) | |
| 222 clsuffix.sort() | |
| 223 xclsuffix.sort() | |
| 224 self.xclsuffix = xclsuffix | |
| 225 self.clsuffix = clsuffix | |
| 226 if self.args.parampass == "positional": | |
| 227 self.clpositional() | |
| 228 else: | |
| 229 self.clargparse() | |
| 230 | |
| 231 def prepScript(self): | |
| 232 aCL = self.cl.append | |
| 233 rx = open(self.args.script_path, "r").readlines() | |
| 234 rx = [x.rstrip() for x in rx] | |
| 235 rxcheck = [x.strip() for x in rx if x.strip() > ""] | |
| 236 assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" | |
| 237 self.script = "\n".join(rx) | |
| 238 fhandle, self.sfile = tempfile.mkstemp( | |
| 239 prefix=self.tool_name, suffix="_%s" % (self.args.interpreter_name) | |
| 240 ) | |
| 241 tscript = open(self.sfile, "w") | |
| 242 tscript.write(self.script) | |
| 243 tscript.close() | |
| 244 self.indentedScript = " %s" % "\n".join( | |
| 245 [" %s" % html_escape(x) for x in rx] | |
| 246 ) | |
| 247 self.escapedScript = "%s" % "\n".join( | |
| 248 [" %s" % html_escape(x) for x in rx] | |
| 249 ) | |
| 250 art = "%s.%s" % (self.tool_name, self.args.interpreter_name) | |
| 251 artifact = open(art, "wb") | |
| 252 artifact.write(bytes(self.script, "utf8")) | |
| 253 artifact.close() | |
| 254 aCL(self.args.interpreter_name) | |
| 255 aCL(self.sfile) | |
| 256 | |
| 257 def cleanuppar(self): | |
| 258 """ positional parameters are complicated by their numeric ordinal""" | |
| 259 for i, p in enumerate(self.infiles): | |
| 260 if self.args.parampass == "positional": | |
| 261 assert p[ICLPOS].isdigit(), ( | |
| 262 "Positional parameters must be ordinal integers - got %s for %s" | |
| 263 % (p[ICLPOS], p[ILABPOS]) | |
| 264 ) | |
| 265 p.append(p[ICLPOS]) | |
| 266 if p[ICLPOS].isdigit() or self.args.parampass == "0": | |
| 267 scl = "input%d" % (i + 1) | |
| 268 p[ICLPOS] = scl | |
| 269 self.infiles[i] = p | |
| 270 for i, p in enumerate( | |
| 271 self.outfiles | |
| 272 ): # trying to automagically gather using extensions | |
| 273 if self.args.parampass == "positional" and p[OCLPOS] != "STDOUT": | |
| 274 assert p[OCLPOS].isdigit(), ( | |
| 275 "Positional parameters must be ordinal integers - got %s for %s" | |
| 276 % (p[OCLPOS], p[ONAMEPOS]) | |
| 277 ) | |
| 278 p.append(p[OCLPOS]) | |
| 279 if p[OCLPOS].isdigit() or p[OCLPOS] == "STDOUT": | |
| 280 scl = p[ONAMEPOS] | |
| 281 p[OCLPOS] = scl | |
| 282 self.outfiles[i] = p | |
| 283 for i, p in enumerate(self.addpar): | |
| 284 if self.args.parampass == "positional": | |
| 285 assert p[ACLPOS].isdigit(), ( | |
| 286 "Positional parameters must be ordinal integers - got %s for %s" | |
| 287 % (p[ACLPOS], p[ANAMEPOS]) | |
| 288 ) | |
| 289 p.append(p[ACLPOS]) | |
| 290 if p[ACLPOS].isdigit(): | |
| 291 scl = "input%s" % p[ACLPOS] | |
| 292 p[ACLPOS] = scl | |
| 293 self.addpar[i] = p | |
| 294 | |
| 295 def clsimple(self): | |
| 296 """ no parameters - uses < and > for i/o | |
| 297 """ | |
| 298 aCL = self.cl.append | |
| 299 aCL("<") | |
| 300 aCL(self.infiles[0][IPATHPOS]) | |
| 301 aCL(">") | |
| 302 aCL(self.outfiles[0][OCLPOS]) | |
| 303 aXCL = self.xmlcl.append | |
| 304 aXCL("<") | |
| 305 aXCL("$%s" % self.infiles[0][ICLPOS]) | |
| 306 aXCL(">") | |
| 307 aXCL("$%s" % self.outfiles[0][ONAMEPOS]) | |
| 308 | |
| 309 def clpositional(self): | |
| 310 # inputs in order then params | |
| 311 aCL = self.cl.append | |
| 312 for (o_v, k, v, koverride) in self.clsuffix: | |
| 313 if " " in v: | |
| 314 aCL("%s" % v) | |
| 315 else: | |
| 316 aCL(v) | |
| 317 aXCL = self.xmlcl.append | |
| 318 for (o_v, k, v, koverride) in self.xclsuffix: | |
| 319 aXCL(v) | |
| 320 if self.lastxclredirect: | |
| 321 aXCL(self.lastxclredirect[0]) | |
| 322 aXCL(self.lastxclredirect[1]) | |
| 323 | |
| 324 def clargparse(self): | |
| 325 """ argparse style | |
| 326 """ | |
| 327 aCL = self.cl.append | |
| 328 aXCL = self.xmlcl.append | |
| 329 # inputs then params in argparse named form | |
| 330 for (o_v, k, v, koverride) in self.xclsuffix: | |
| 331 if koverride > "": | |
| 332 k = koverride | |
| 333 elif len(k.strip()) == 1: | |
| 334 k = "-%s" % k | |
| 335 else: | |
| 336 k = "--%s" % k | |
| 337 aXCL(k) | |
| 338 aXCL(v) | |
| 339 for (o_v, k, v, koverride) in self.clsuffix: | |
| 340 if koverride > "": | |
| 341 k = koverride | |
| 342 elif len(k.strip()) == 1: | |
| 343 k = "-%s" % k | |
| 344 else: | |
| 345 k = "--%s" % k | |
| 346 aCL(k) | |
| 347 aCL(v) | |
| 348 | |
| 349 def getNdash(self, newname): | |
| 350 if self.is_positional: | |
| 351 ndash = 0 | |
| 352 else: | |
| 353 ndash = 2 | |
| 354 if len(newname) < 2: | |
| 355 ndash = 1 | |
| 356 return ndash | |
| 357 | |
| 358 def doXMLparam(self): | |
| 359 """flake8 made me do this...""" | |
| 360 for p in self.outfiles: | |
| 361 newname, newfmt, newcl, oldcl = p | |
| 362 ndash = self.getNdash(newcl) | |
| 363 aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash) | |
| 364 aparm.positional = self.is_positional | |
| 365 if self.is_positional: | |
| 366 if oldcl == "STDOUT": | |
| 367 aparm.positional = 9999999 | |
| 368 aparm.command_line_override = "> $%s" % newcl | |
| 369 else: | |
| 370 aparm.positional = int(oldcl) | |
| 371 aparm.command_line_override = "$%s" % newcl | |
| 372 self.toutputs.append(aparm) | |
| 373 tp = gxtp.TestOutput( | |
| 374 name=newcl, value="%s_sample" % newcl, format=newfmt | |
| 375 ) | |
| 376 self.testparam.append(tp) | |
| 377 for p in self.infiles: | |
| 378 newname = p[ICLPOS] | |
| 379 newfmt = p[IFMTPOS] | |
| 380 ndash = self.getNdash(newname) | |
| 381 if not len(p[ILABPOS]) > 0: | |
| 382 alab = p[ICLPOS] | |
| 383 else: | |
| 384 alab = p[ILABPOS] | |
| 385 aninput = gxtp.DataParam( | |
| 386 newname, | |
| 387 optional=False, | |
| 388 label=alab, | |
| 389 help=p[IHELPOS], | |
| 390 format=newfmt, | |
| 391 multiple=False, | |
| 392 num_dashes=ndash, | |
| 393 ) | |
| 394 aninput.positional = self.is_positional | |
| 395 self.tinputs.append(aninput) | |
| 396 tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname) | |
| 397 self.testparam.append(tparm) | |
| 398 for p in self.addpar: | |
| 399 newname, newval, newlabel, newhelp, newtype, newcl, override, oldcl = p | |
| 400 if not len(newlabel) > 0: | |
| 401 newlabel = newname | |
| 402 ndash = self.getNdash(newname) | |
| 403 if newtype == "text": | |
| 404 aparm = gxtp.TextParam( | |
| 405 newname, | |
| 406 label=newlabel, | |
| 407 help=newhelp, | |
| 408 value=newval, | |
| 409 num_dashes=ndash, | |
| 410 ) | |
| 411 elif newtype == "integer": | |
| 412 aparm = gxtp.IntegerParam( | |
| 413 newname, | |
| 414 label=newname, | |
| 415 help=newhelp, | |
| 416 value=newval, | |
| 417 num_dashes=ndash, | |
| 418 ) | |
| 419 elif newtype == "float": | |
| 420 aparm = gxtp.FloatParam( | |
| 421 newname, | |
| 422 label=newname, | |
| 423 help=newhelp, | |
| 424 value=newval, | |
| 425 num_dashes=ndash, | |
| 426 ) | |
| 427 else: | |
| 428 raise ValueError( | |
| 429 'Unrecognised parameter type "%s" for\ | |
| 430 additional parameter %s in makeXML' | |
| 431 % (newtype, newname) | |
| 432 ) | |
| 433 aparm.positional = self.is_positional | |
| 434 if self.is_positional: | |
| 435 aninput.positional = int(oldcl) | |
| 436 self.tinputs.append(aparm) | |
| 437 self.tparm = gxtp.TestParam(newname, value=newval) | |
| 438 self.testparam.append(tparm) | |
| 439 | |
| 440 def doNoXMLparam(self): | |
| 441 alab = self.infiles[0][ILABPOS] | |
| 442 if len(alab) == 0: | |
| 443 alab = self.infiles[0][ICLPOS] | |
| 444 max1s = ( | |
| 445 "Maximum one input if parampass is 0 - more than one input files supplied - %s" | |
| 446 % str(self.infiles) | |
| 447 ) | |
| 448 assert len(self.infiles) == 1, max1s | |
| 449 newname = self.infiles[0][ICLPOS] | |
| 450 aninput = gxtp.DataParam( | |
| 451 newname, | |
| 452 optional=False, | |
| 453 label=alab, | |
| 454 help=self.infiles[0][IHELPOS], | |
| 455 format=self.infiles[0][IFMTPOS], | |
| 456 multiple=False, | |
| 457 num_dashes=0, | |
| 458 ) | |
| 459 aninput.command_line_override = "< $%s" % newname | |
| 460 aninput.positional = self.is_positional | |
| 461 self.tinputs.append(aninput) | |
| 462 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) | |
| 463 self.testparam.append(tp) | |
| 464 newname = self.outfiles[0][OCLPOS] | |
| 465 newfmt = self.outfiles[0][OFMTPOS] | |
| 466 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) | |
| 467 anout.command_line_override = "> $%s" % newname | |
| 468 anout.positional = self.is_positional | |
| 469 self.toutputs.append(anout) | |
| 470 tp = gxtp.TestOutput( | |
| 471 name=newname, value="%s_sample" % newname, format=newfmt | |
| 472 ) | |
| 473 self.testparam.append(tp) | |
| 474 | |
| 475 def makeXML(self): | |
| 476 """ | |
| 477 Create a Galaxy xml tool wrapper for the new script | |
| 478 Uses galaxyhtml | |
| 479 Hmmm. How to get the command line into correct order... | |
| 480 """ | |
| 481 self.tool.command_line_override = self.xmlcl | |
| 482 if self.args.interpreter_name: | |
| 483 self.tool.interpreter = self.args.interpreter_name | |
| 484 if self.args.help_text: | |
| 485 helptext = open(self.args.help_text, "r").readlines() | |
| 486 helptext = [html_escape(x) for x in helptext] | |
| 487 self.tool.help = "".join([x for x in helptext]) | |
| 488 else: | |
| 489 self.tool.help = ( | |
| 490 "Please ask the tool author (%s) for help \ | |
| 491 as none was supplied at tool generation\n" | |
| 492 % (self.args.user_email) | |
| 493 ) | |
| 494 self.tool.version_command = None # do not want | |
| 495 requirements = gxtp.Requirements() | |
| 496 | |
| 497 if self.args.interpreter_name: | |
| 498 if self.args.interpreter_name == "python": | |
| 499 requirements.append( | |
| 500 gxtp.Requirement( | |
| 501 "package", "python", self.args.interpreter_version | |
| 502 ) | |
| 503 ) | |
| 504 elif self.args.interpreter_name not in ["bash", "sh"]: | |
| 505 requirements.append( | |
| 506 gxtp.Requirement( | |
| 507 "package", | |
| 508 self.args.interpreter_name, | |
| 509 self.args.interpreter_version, | |
| 510 ) | |
| 511 ) | |
| 512 else: | |
| 513 if self.args.exe_package and self.args.parampass != "system": | |
| 514 requirements.append( | |
| 515 gxtp.Requirement( | |
| 516 "package", | |
| 517 self.args.exe_package, | |
| 518 self.args.exe_package_version, | |
| 519 ) | |
| 520 ) | |
| 521 self.tool.requirements = requirements | |
| 522 if self.args.parampass == "0": | |
| 523 self.doNoXMLparam() | |
| 524 else: | |
| 525 self.doXMLparam() | |
| 526 self.tool.outputs = self.toutputs | |
| 527 self.tool.inputs = self.tinputs | |
| 528 if self.args.runmode not in ["Executable", "system"]: | |
| 529 configfiles = gxtp.Configfiles() | |
| 530 configfiles.append(gxtp.Configfile(name="runMe", text=self.script)) | |
| 531 self.tool.configfiles = configfiles | |
| 532 tests = gxtp.Tests() | |
| 533 test_a = gxtp.Test() | |
| 534 for tp in self.testparam: | |
| 535 test_a.append(tp) | |
| 536 tests.append(test_a) | |
| 537 self.tool.tests = tests | |
| 538 self.tool.add_comment( | |
| 539 "Created by %s at %s using the Galaxy Tool Factory." | |
| 540 % (self.args.user_email, timenow()) | |
| 541 ) | |
| 542 self.tool.add_comment("Source in git at: %s" % (toolFactoryURL)) | |
| 543 self.tool.add_comment( | |
| 544 "Cite: Creating re-usable tools from scripts doi: 10.1093/bioinformatics/bts573" | |
| 545 ) | |
| 546 exml = self.tool.export() | |
| 547 xf = open('%s.xml' % self.tool_name, "w") | |
| 548 xf.write(exml) | |
| 549 xf.write("\n") | |
| 550 xf.close() | |
| 551 # ready for the tarball | |
| 552 | |
| 553 def makeTooltar(self): | |
| 554 """ | |
| 555 a tool is a gz tarball with eg | |
| 556 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ... | |
| 557 NOTE names for test inputs and outputs are munged here so must | |
| 558 correspond to actual input and output names used on the generated cl | |
| 559 """ | |
| 560 retval = self.run() | |
| 561 if retval: | |
| 562 sys.stderr.write( | |
| 563 "## Run failed. Cannot build yet. Please fix and retry" | |
| 564 ) | |
| 565 sys.exit(1) | |
| 566 tdir = "tfout" | |
| 567 if not os.path.exists(tdir): | |
| 568 os.mkdir(tdir) | |
| 569 self.makeXML() | |
| 570 testdir = os.path.join(tdir, "test-data") | |
| 571 if not os.path.exists(testdir): | |
| 572 os.mkdir(testdir) # make tests directory | |
| 573 for p in self.infiles: | |
| 574 pth = p[IPATHPOS] | |
| 575 dest = os.path.join(testdir, "%s_sample" % p[ICLPOS]) | |
| 576 shutil.copyfile(pth, dest) | |
| 577 for p in self.outfiles: | |
| 578 pth = p[OCLPOS] | |
| 579 if p[OOCLPOS] == "STDOUT" or self.args.parampass == "0": | |
| 580 pth = p[ONAMEPOS] | |
| 581 dest = os.path.join(testdir, "%s_sample" % p[ONAMEPOS]) | |
| 582 shutil.copyfile(pth, dest) | |
| 583 dest = os.path.join(tdir, p[ONAMEPOS]) | |
| 584 shutil.copyfile(pth, dest) | |
| 585 else: | |
| 586 pth = p[OCLPOS] | |
| 587 dest = os.path.join(testdir, "%s_sample" % p[OCLPOS]) | |
| 588 shutil.copyfile(pth, dest) | |
| 589 dest = os.path.join(tdir, p[OCLPOS]) | |
| 590 shutil.copyfile(pth, dest) | |
| 591 | |
| 592 if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0: | |
| 593 shutil.copyfile(self.tlog, os.path.join(testdir, "test1_log_outfiletxt")) | |
| 594 if self.args.runmode not in ["Executable", "system"]: | |
| 595 stname = os.path.join(tdir, "%s" % (self.sfile)) | |
| 596 if not os.path.exists(stname): | |
| 597 shutil.copyfile(self.sfile, stname) | |
| 598 xreal = '%s.xml' % self.tool_name | |
| 599 xout = os.path.join(tdir,xreal) | |
| 600 shutil.copyfile(xreal, xout) | |
| 601 tarpath = "toolfactory_%s.tgz" % self.tool_name | |
| 602 tf = tarfile.open(tarpath, "w:gz") | |
| 603 tf.add(name=tdir, arcname=self.tool_name) | |
| 604 tf.close() | |
| 605 shutil.copyfile(tarpath, self.args.new_tool) | |
| 606 shutil.copyfile(xreal,"tool_xml.txt") | |
| 607 repdir = "TF_run_report_tempdir" | |
| 608 if not os.path.exists(repdir): | |
| 609 os.mkdir(repdir) | |
| 610 repoutnames = [x[OCLPOS] for x in self.outfiles] | |
| 611 with os.scandir('.') as outs: | |
| 612 for entry in outs: | |
| 613 if entry.name.endswith('.tgz') or not entry.is_file(): | |
| 614 continue | |
| 615 if entry.name in repoutnames: | |
| 616 shutil.copyfile(entry.name,os.path.join(repdir,entry.name)) | |
| 617 elif entry.name == "%s.xml" % self.tool_name: | |
| 618 shutil.copyfile(entry.name,os.path.join(repdir,"new_tool_xml")) | |
| 619 return retval | |
| 620 | |
| 621 def run(self): | |
| 622 """ | |
| 623 Some devteam tools have this defensive stderr read so I'm keeping with the faith | |
| 624 Feel free to update. | |
| 625 """ | |
| 626 s = "run cl=%s" % str(self.cl) | |
| 627 | |
| 628 logging.debug(s) | |
| 629 scl = " ".join(self.cl) | |
| 630 err = None | |
| 631 if self.args.parampass != "0": | |
| 632 ste = open(self.elog, "wb") | |
| 633 if self.lastclredirect: | |
| 634 sto = open( | |
| 635 self.lastclredirect[1], "wb" | |
| 636 ) # is name of an output file | |
| 637 else: | |
| 638 sto = open(self.tlog, "wb") | |
| 639 sto.write( | |
| 640 bytes( | |
| 641 "## Executing Toolfactory generated command line = %s\n" | |
| 642 % scl, | |
| 643 "utf8", | |
| 644 ) | |
| 645 ) | |
| 646 sto.flush() | |
| 647 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) | |
| 648 sto.close() | |
| 649 ste.close() | |
| 650 tmp_stderr = open(self.elog, "rb") | |
| 651 err = "" | |
| 652 buffsize = 1048576 | |
| 653 try: | |
| 654 while True: | |
| 655 err += str(tmp_stderr.read(buffsize)) | |
| 656 if not err or len(err) % buffsize != 0: | |
| 657 break | |
| 658 except OverflowError: | |
| 659 pass | |
| 660 tmp_stderr.close() | |
| 661 retval = p.returncode | |
| 662 else: # work around special case of simple scripts that take stdin and write to stdout | |
| 663 sti = open(self.infiles[0][IPATHPOS], "rb") | |
| 664 sto = open(self.outfiles[0][ONAMEPOS], "wb") | |
| 665 # must use shell to redirect | |
| 666 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) | |
| 667 retval = p.returncode | |
| 668 sto.close() | |
| 669 sti.close() | |
| 670 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0: | |
| 671 os.unlink(self.tlog) | |
| 672 if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0: | |
| 673 os.unlink(self.elog) | |
| 674 if p.returncode != 0 and err: # problem | |
| 675 sys.stderr.write(err) | |
| 676 logging.debug("run done") | |
| 677 return retval | |
| 678 | |
| 679 | |
| 680 def main(): | |
| 681 """ | |
| 682 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: | |
| 683 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" | |
| 684 </command> | |
| 685 """ | |
| 686 parser = argparse.ArgumentParser() | |
| 687 a = parser.add_argument | |
| 688 a("--script_path", default="") | |
| 689 a("--tool_name", default=None) | |
| 690 a("--interpreter_name", default=None) | |
| 691 a("--interpreter_version", default=None) | |
| 692 a("--exe_package", default=None) | |
| 693 a("--exe_package_version", default=None) | |
| 694 a("--input_files", default=[], action="append") | |
| 695 a("--output_files", default=[], action="append") | |
| 696 a("--user_email", default="Unknown") | |
| 697 a("--bad_user", default=None) | |
| 698 a("--make_Tool", default=None) | |
| 699 a("--help_text", default=None) | |
| 700 a("--tool_desc", default=None) | |
| 701 a("--tool_version", default=None) | |
| 702 a("--citations", default=None) | |
| 703 a("--additional_parameters", action="append", default=[]) | |
| 704 a("--edit_additional_parameters", action="store_true", default=False) | |
| 705 a("--parampass", default="positional") | |
| 706 a("--tfout", default="./tfout") | |
| 707 a("--new_tool", default="new_tool") | |
| 708 a("--runmode", default=None) | |
| 709 args = parser.parse_args() | |
| 710 assert not args.bad_user, ( | |
| 711 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' | |
| 712 % (args.bad_user, args.bad_user) | |
| 713 ) | |
| 714 assert ( | |
| 715 args.tool_name | |
| 716 ), "## Tool Factory expects a tool name - eg --tool_name=DESeq" | |
| 717 assert ( | |
| 718 args.interpreter_name or args.exe_package | |
| 719 ), "## Tool Factory wrapper expects an interpreter or an executable package" | |
| 720 assert args.exe_package or ( | |
| 721 len(args.script_path) > 0 and os.path.isfile(args.script_path) | |
| 722 ), "## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable" | |
| 723 args.input_files = [ | |
| 724 x.replace('"', "").replace("'", "") for x in args.input_files | |
| 725 ] | |
| 726 # remove quotes we need to deal with spaces in CL params | |
| 727 for i, x in enumerate(args.additional_parameters): | |
| 728 args.additional_parameters[i] = args.additional_parameters[i].replace( | |
| 729 '"', "" | |
| 730 ) | |
| 731 r = ScriptRunner(args) | |
| 732 if args.make_Tool: | |
| 733 retcode = r.makeTooltar() | |
| 734 else: | |
| 735 retcode = r.run() | |
| 736 if retcode: | |
| 737 sys.exit(retcode) # indicate failure to job runner | |
| 738 | |
| 739 | |
| 740 if __name__ == "__main__": | |
| 741 main() |
