Mercurial > repos > fubar > toolfactory_gtn

comparison toolfactory/ToolFactory.py @ 6:efefe43f23c8 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Fri, 30 Apr 2021 02:10:32 +0000
parents e2c8c2fa192d
children
comparison
equal deleted inserted replaced
5:e2c8c2fa192d 6:efefe43f23c8
26 import subprocess 26 import subprocess
27 import sys 27 import sys
28 import tarfile 28 import tarfile
29 import tempfile 29 import tempfile
30 import time 30 import time
31 import urllib
31 32
32 from bioblend import ConnectionError 33 from bioblend import ConnectionError
33 from bioblend import galaxy 34 from bioblend import galaxy
34 from bioblend import toolshed 35 from bioblend import toolshed
35 36
68 citation_tuples.append(("doi", citation[len("doi") :].strip())) 69 citation_tuples.append(("doi", citation[len("doi") :].strip()))
69 else: 70 else:
70 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip())) 71 citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
71 return citation_tuples 72 return citation_tuples
72 73
73 class ToolConfUpdater(): 74
75 class Tool_Conf_Updater():
74 # update config/tool_conf.xml with a new tool unpacked in /tools 76 # update config/tool_conf.xml with a new tool unpacked in /tools
75 # requires highly insecure docker settings - like write to tool_conf.xml and to tools ! 77 # requires highly insecure docker settings - like write to tool_conf.xml and to tools !
76 # if in a container possibly not so courageous. 78 # if in a container possibly not so courageous.
77 # Fine on your own laptop but security red flag for most production instances 79 # Fine on your own laptop but security red flag for most production instances
78 80
79 def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir): 81 def __init__(self, args, tool_conf_path, new_tool_archive_path, new_tool_name, tool_dir):
80 self.args = args 82 self.args = args
81 self.tool_conf_path = os.path.join(args.galaxy_root,tool_conf_path) 83 self.tool_conf_path = os.path.join(args.galaxy_root,tool_conf_path)
82 self.tool_dir = os.path.join(args.galaxy_root, tool_dir) 84 self.tool_dir = os.path.join(args.galaxy_root, tool_dir)
83 self.our_name = 'ToolFactory' 85 self.our_name = 'ToolFactory'
84 tff = tarfile.open(new_tool_archive_path, "r:*") 86 tff = tarfile.open(new_tool_archive_path, "r:*")
136 tree.write(newconf, pretty_print=True) 138 tree.write(newconf, pretty_print=True)
137 self.run_rsync(newconf,self.tool_conf_path) 139 self.run_rsync(newconf,self.tool_conf_path)
138 if False and self.args.packages and self.args.packages > '': 140 if False and self.args.packages and self.args.packages > '':
139 self.install_deps() 141 self.install_deps()
140 142
141 class ScriptRunner: 143 class Tool_Factory:
142 """Wrapper for an arbitrary script 144 """Wrapper for an arbitrary script
143 uses galaxyxml 145 uses galaxyxml
144 146
145 """ 147 """
146 148
192 print( 194 print(
193 f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary" 195 f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary"
194 ) 196 )
195 self.args = args 197 self.args = args
196 self.cleanuppar() 198 self.cleanuppar()
197 self.lastclredirect = None
198 self.lastxclredirect = None 199 self.lastxclredirect = None
199 self.cl = []
200 self.xmlcl = [] 200 self.xmlcl = []
201 self.is_positional = self.args.parampass == "positional" 201 self.is_positional = self.args.parampass == "positional"
202 if self.args.sysexe: 202 if self.args.sysexe:
203 if ' ' in self.args.sysexe: 203 if ' ' in self.args.sysexe:
204 self.executeme = self.args.sysexe.split(' ') 204 self.executeme = self.args.sysexe.split(' ')
207 else: 207 else:
208 if self.args.packages: 208 if self.args.packages:
209 self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ] 209 self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ]
210 else: 210 else:
211 self.executeme = None 211 self.executeme = None
212 aCL = self.cl.append
213 aXCL = self.xmlcl.append 212 aXCL = self.xmlcl.append
214 assert args.parampass in [ 213 assert args.parampass in [
215 "0", 214 "0",
216 "argparse", 215 "argparse",
217 "positional", 216 "positional",
250 self.test_override = [x.rstrip() for x in stos] 249 self.test_override = [x.rstrip() for x in stos]
251 else: 250 else:
252 self.test_override = None 251 self.test_override = None
253 if self.args.script_path: 252 if self.args.script_path:
254 for ex in self.executeme: 253 for ex in self.executeme:
255 aCL(ex)
256 aXCL(ex) 254 aXCL(ex)
257 aCL(self.sfile)
258 aXCL("$runme") 255 aXCL("$runme")
259 else: 256 else:
260 for ex in self.executeme: 257 for ex in self.executeme:
261 aCL(ex)
262 aXCL(ex) 258 aXCL(ex)
263 259
264 if self.args.parampass == "0": 260 if self.args.parampass == "0":
265 self.clsimple() 261 self.clsimple()
266 else: 262 else:
271 self.prepargp() 267 self.prepargp()
272 self.clargparse() 268 self.clargparse()
273 269
274 def clsimple(self): 270 def clsimple(self):
275 """no parameters or repeats - uses < and > for i/o""" 271 """no parameters or repeats - uses < and > for i/o"""
276 aCL = self.cl.append
277 aXCL = self.xmlcl.append 272 aXCL = self.xmlcl.append
278 if len(self.infiles) > 0: 273 if len(self.infiles) > 0:
279 aCL("<")
280 aCL(self.infiles[0]["infilename"])
281 aXCL("<") 274 aXCL("<")
282 aXCL("$%s" % self.infiles[0]["infilename"]) 275 aXCL("$%s" % self.infiles[0]["infilename"])
283 if len(self.outfiles) > 0: 276 if len(self.outfiles) > 0:
284 aCL(">")
285 aCL(self.outfiles[0]["name"])
286 aXCL(">") 277 aXCL(">")
287 aXCL("$%s" % self.outfiles[0]["name"]) 278 aXCL("$%s" % self.outfiles[0]["name"])
288 if self.args.cl_user_suffix: # DIY CL end 279 if self.args.cl_user_suffix: # DIY CL end
289 clp = shlex.split(self.args.cl_user_suffix) 280 clp = shlex.split(self.args.cl_user_suffix)
290 for c in clp: 281 for c in clp:
291 aCL(c)
292 aXCL(c) 282 aXCL(c)
293 283
294 def prepargp(self): 284 def prepargp(self):
295 clsuffix = []
296 xclsuffix = [] 285 xclsuffix = []
297 for i, p in enumerate(self.infiles): 286 for i, p in enumerate(self.infiles):
298 nam = p["infilename"] 287 nam = p["infilename"]
299 if p["origCL"].strip().upper() == "STDIN": 288 if p["origCL"].strip().upper() == "STDIN":
300 appendme = [
301 nam,
302 nam,
303 "< %s" % nam,
304 ]
305 xappendme = [ 289 xappendme = [
306 nam, 290 nam,
307 nam, 291 nam,
308 "< $%s" % nam, 292 "< $%s" % nam,
309 ] 293 ]
310 else: 294 else:
311 rep = p["repeat"] == "1" 295 rep = p["repeat"] == "1"
312 over = "" 296 over = ""
313 if rep: 297 if rep:
314 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' 298 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for'
315 appendme = [p["CL"], p["CL"], ""]
316 xappendme = [p["CL"], "$%s" % p["CL"], over] 299 xappendme = [p["CL"], "$%s" % p["CL"], over]
317 clsuffix.append(appendme)
318 xclsuffix.append(xappendme) 300 xclsuffix.append(xappendme)
319 for i, p in enumerate(self.outfiles): 301 for i, p in enumerate(self.outfiles):
320 if p["origCL"].strip().upper() == "STDOUT": 302 if p["origCL"].strip().upper() == "STDOUT":
321 self.lastclredirect = [">", p["name"]]
322 self.lastxclredirect = [">", "$%s" % p["name"]] 303 self.lastxclredirect = [">", "$%s" % p["name"]]
323 else: 304 else:
324 clsuffix.append([p["name"], p["name"], ""])
325 xclsuffix.append([p["name"], "$%s" % p["name"], ""]) 305 xclsuffix.append([p["name"], "$%s" % p["name"], ""])
326 for p in self.addpar: 306 for p in self.addpar:
327 nam = p["name"] 307 nam = p["name"]
328 rep = p["repeat"] == "1" 308 rep = p["repeat"] == "1"
329 if rep: 309 if rep:
330 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for' 310 over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for'
331 else: 311 else:
332 over = p["override"] 312 over = p["override"]
333 clsuffix.append([p["CL"], nam, over])
334 xclsuffix.append([p["CL"], '"$%s"' % nam, over]) 313 xclsuffix.append([p["CL"], '"$%s"' % nam, over])
335 for p in self.selpar: 314 for p in self.selpar:
336 clsuffix.append([p["CL"], p["name"], p["override"]])
337 xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) 315 xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]])
338 self.xclsuffix = xclsuffix 316 self.xclsuffix = xclsuffix
339 self.clsuffix = clsuffix
340 317
341 def prepclpos(self): 318 def prepclpos(self):
342 clsuffix = []
343 xclsuffix = [] 319 xclsuffix = []
344 for i, p in enumerate(self.infiles): 320 for i, p in enumerate(self.infiles):
345 if p["origCL"].strip().upper() == "STDIN": 321 if p["origCL"].strip().upper() == "STDIN":
346 appendme = [
347 "999",
348 p["infilename"],
349 "< $%s" % p["infilename"],
350 ]
351 xappendme = [ 322 xappendme = [
352 "999", 323 "999",
353 p["infilename"], 324 p["infilename"],
354 "< $%s" % p["infilename"], 325 "< $%s" % p["infilename"],
355 ] 326 ]
356 else: 327 else:
357 appendme = [p["CL"], p["infilename"], ""]
358 xappendme = [p["CL"], "$%s" % p["infilename"], ""] 328 xappendme = [p["CL"], "$%s" % p["infilename"], ""]
359 clsuffix.append(appendme)
360 xclsuffix.append(xappendme) 329 xclsuffix.append(xappendme)
361 for i, p in enumerate(self.outfiles): 330 for i, p in enumerate(self.outfiles):
362 if p["origCL"].strip().upper() == "STDOUT": 331 if p["origCL"].strip().upper() == "STDOUT":
363 self.lastclredirect = [">", p["name"]]
364 self.lastxclredirect = [">", "$%s" % p["name"]] 332 self.lastxclredirect = [">", "$%s" % p["name"]]
365 else: 333 else:
366 clsuffix.append([p["CL"], p["name"], ""])
367 xclsuffix.append([p["CL"], "$%s" % p["name"], ""]) 334 xclsuffix.append([p["CL"], "$%s" % p["name"], ""])
368 for p in self.addpar: 335 for p in self.addpar:
369 nam = p["name"] 336 nam = p["name"]
370 rep = p["repeat"] == "1" # repeats make NO sense 337 rep = p["repeat"] == "1" # repeats make NO sense
371 if rep: 338 if rep:
372 print(f'### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!') 339 print(f'### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!')
373 over = p["override"] 340 over = p["override"]
374 clsuffix.append([p["CL"], nam, over])
375 xclsuffix.append([p["CL"], '"$%s"' % nam, over]) 341 xclsuffix.append([p["CL"], '"$%s"' % nam, over])
376 for p in self.selpar: 342 for p in self.selpar:
377 clsuffix.append([p["CL"], p["name"], p["override"]])
378 xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]]) 343 xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]])
379 clsuffix.sort()
380 xclsuffix.sort() 344 xclsuffix.sort()
381 self.xclsuffix = xclsuffix 345 self.xclsuffix = xclsuffix
382 self.clsuffix = clsuffix
383 346
384 def prepScript(self): 347 def prepScript(self):
385 rx = open(self.args.script_path, "r").readlines() 348 rx = open(self.args.script_path, "r").readlines()
386 rx = [x.rstrip() for x in rx] 349 rx = [x.rstrip() for x in rx]
387 rxcheck = [x.strip() for x in rx if x.strip() > ""] 350 rxcheck = [x.strip() for x in rx if x.strip() > ""]
441 p["origCL"] = p["CL"] 404 p["origCL"] = p["CL"]
442 self.addpar[i] = p 405 self.addpar[i] = p
443 406
444 def clpositional(self): 407 def clpositional(self):
445 # inputs in order then params 408 # inputs in order then params
446 aCL = self.cl.append
447 for (k, v, koverride) in self.clsuffix:
448 if " " in v:
449 aCL("%s" % v)
450 else:
451 aCL(v)
452 aXCL = self.xmlcl.append 409 aXCL = self.xmlcl.append
453 for (k, v, koverride) in self.xclsuffix: 410 for (k, v, koverride) in self.xclsuffix:
454 aXCL(v) 411 aXCL(v)
455 if self.lastxclredirect: 412 if self.lastxclredirect:
456 aXCL(self.lastxclredirect[0]) 413 aXCL(self.lastxclredirect[0])
457 aXCL(self.lastxclredirect[1]) 414 aXCL(self.lastxclredirect[1])
458 if self.args.cl_user_suffix: # DIY CL end 415 if self.args.cl_user_suffix: # DIY CL end
459 clp = shlex.split(self.args.cl_user_suffix) 416 clp = shlex.split(self.args.cl_user_suffix)
460 for c in clp: 417 for c in clp:
461 aCL(c)
462 aXCL(c) 418 aXCL(c)
463 419
464 420
465 def clargparse(self): 421 def clargparse(self):
466 """argparse style""" 422 """argparse style"""
467 aCL = self.cl.append
468 aXCL = self.xmlcl.append 423 aXCL = self.xmlcl.append
469 # inputs then params in argparse named form 424 # inputs then params in argparse named form
470 425
471 for (k, v, koverride) in self.xclsuffix: 426 for (k, v, koverride) in self.xclsuffix:
472 if koverride > "": 427 if koverride > "":
477 k = "-%s" % k 432 k = "-%s" % k
478 else: 433 else:
479 k = "--%s" % k 434 k = "--%s" % k
480 aXCL(k) 435 aXCL(k)
481 aXCL(v) 436 aXCL(v)
482 for (k, v, koverride) in self.clsuffix:
483 if koverride > "":
484 k = koverride
485 elif len(k.strip()) == 1:
486 k = "-%s" % k
487 else:
488 k = "--%s" % k
489 aCL(k)
490 aCL(v)
491 if self.lastxclredirect: 437 if self.lastxclredirect:
492 aXCL(self.lastxclredirect[0]) 438 aXCL(self.lastxclredirect[0])
493 aXCL(self.lastxclredirect[1]) 439 aXCL(self.lastxclredirect[1])
494 if self.args.cl_user_suffix: # DIY CL end 440 if self.args.cl_user_suffix: # DIY CL end
495 clp = shlex.split(self.args.cl_user_suffix) 441 clp = shlex.split(self.args.cl_user_suffix)
496 for c in clp: 442 for c in clp:
497 aCL(c)
498 aXCL(c) 443 aXCL(c)
499 444
500 def getNdash(self, newname): 445 def getNdash(self, newname):
501 if self.is_positional: 446 if self.is_positional:
502 ndash = 0 447 ndash = 0
855 xf.write(exml) 800 xf.write(exml)
856 xf.write("\n") 801 xf.write("\n")
857 xf.close() 802 xf.close()
858 # ready for the tarball 803 # ready for the tarball
859 804
860 def run(self): #noqa
861 """
862 generate test outputs by running a command line
863 won't work if command or test override in play - planemo is the
864 easiest way to generate test outputs for that case so is
865 automagically selected
866 """
867 scl = " ".join(self.cl)
868 err = None
869 logname = f"{self.tool_name}_runner_log"
870 if self.args.parampass != "0":
871 if self.lastclredirect:
872 logf = open(self.lastclredirect[1], "wb") # is name of an output file
873 else:
874 logf = open(logname,'w')
875 logf.write("No dependencies so sending CL = '%s' to the fast direct runner instead of planemo to generate tests" % scl)
876 subp = subprocess.run(
877 self.cl, shell=False, stdout=logf, stderr=logf
878 )
879 logf.close()
880 retval = subp.returncode
881 else: # work around special case - stdin and write to stdout
882 if len(self.infiles) > 0:
883 sti = open(self.infiles[0]["name"], "rb")
884 else:
885 sti = sys.stdin
886 if len(self.outfiles) > 0:
887 sto = open(self.outfiles[0]["name"], "wb")
888 else:
889 sto = sys.stdout
890 subp = subprocess.run(
891 self.cl, shell=False, stdout=sto, stdin=sti
892 )
893 retval = subp.returncode
894 sto.close()
895 sti.close()
896 if retval != 0 and err: # problem
897 sys.stderr.write(err)
898 for p in self.outfiles:
899 oname = p["name"]
900 tdest = os.path.join(self.testdir, "%s_sample" % oname)
901 if not os.path.isfile(tdest):
902 if os.path.isfile(oname):
903 shutil.copyfile(oname, tdest)
904 dest = os.path.join(self.repdir, "%s.sample.%s" % (oname,p['format']))
905 shutil.copyfile(oname, dest)
906 else:
907 if report_fail:
908 tout.write(
909 "###Tool may have failed - output file %s not found in testdir after planemo run %s."
910 % (oname, self.testdir)
911 )
912 for p in self.infiles:
913 pth = p["name"]
914 dest = os.path.join(self.testdir, "%s_sample" % p["infilename"])
915 shutil.copyfile(pth, dest)
916 dest = os.path.join(self.repdir, "%s_sample.%s" % (p["infilename"],p["format"]))
917 shutil.copyfile(pth, dest)
918 with os.scandir('.') as outs:
919 for entry in outs:
920 newname = entry.name
921 if not entry.is_file() or entry.name.endswith('_sample'):
922 continue
923 if not (entry.name.endswith('.html') or entry.name.endswith('.gz') or entry.name.endswith(".tgz")):
924 fname, ext = os.path.splitext(entry.name)
925 if len(ext) > 1:
926 newname = f"{fname}_{ext[1:]}.txt"
927 else:
928 newname = f"{fname}.txt"
929 dest = os.path.join(self.repdir, newname)
930 src = entry.name
931 shutil.copyfile(src, dest)
932 return retval
933
934 def writeShedyml(self): 805 def writeShedyml(self):
935 """for planemo""" 806 """for planemo"""
936 yuser = self.args.user_email.split("@")[0] 807 yuser = self.args.user_email.split("@")[0]
937 yfname = os.path.join(self.tooloutdir, ".shed.yml") 808 yfname = os.path.join(self.tooloutdir, ".shed.yml")
938 yamlf = open(yfname, "w") 809 yamlf = open(yfname, "w")
1066 a("--new_tool", default="new_tool") 937 a("--new_tool", default="new_tool")
1067 a("--galaxy_root", default="/galaxy-central") 938 a("--galaxy_root", default="/galaxy-central")
1068 a("--galaxy_venv", default="/galaxy_venv") 939 a("--galaxy_venv", default="/galaxy_venv")
1069 a("--collection", action="append", default=[]) 940 a("--collection", action="append", default=[])
1070 a("--include_tests", default=False, action="store_true") 941 a("--include_tests", default=False, action="store_true")
942 a("--admin_only", default=False, action="store_true")
1071 a("--install", default=False, action="store_true") 943 a("--install", default=False, action="store_true")
1072 a("--run_test", default=False, action="store_true") 944 a("--run_test", default=False, action="store_true")
1073 a("--local_tools", default='tools') # relative to $__root_dir__ 945 a("--local_tools", default='tools') # relative to $__root_dir__
1074 a("--tool_conf_path", default='config/tool_conf.xml') # relative to $__root_dir__ 946 a("--tool_conf_path", default='config/tool_conf.xml') # relative to $__root_dir__
1075 a("--galaxy_url", default="http://localhost:8080") 947 a("--galaxy_url", default="http://localhost:8080")
1077 # make sure this is identical to tool_sheds_conf.xml 949 # make sure this is identical to tool_sheds_conf.xml
1078 # localhost != 127.0.0.1 so validation fails 950 # localhost != 127.0.0.1 so validation fails
1079 a("--toolshed_api_key", default="fakekey") 951 a("--toolshed_api_key", default="fakekey")
1080 a("--galaxy_api_key", default="8993d65865e6d6d1773c2c34a1cc207d") 952 a("--galaxy_api_key", default="8993d65865e6d6d1773c2c34a1cc207d")
1081 args = parser.parse_args() 953 args = parser.parse_args()
1082 assert not args.bad_user, ( 954 if args.admin_only:
1083 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \ 955 assert not args.bad_user, (
956 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \
1084 admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file' 957 admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file'
1085 % (args.bad_user, args.bad_user) 958 % (args.bad_user, args.bad_user)
1086 ) 959 )
1087 assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq" 960 assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
1088 assert ( 961 r = Tool_Factory(args)
1089 args.sysexe or args.packages
1090 ), "## Tool Factory wrapper expects an interpreter \
1091 or an executable package in --sysexe or --packages"
1092 print('Hello from',os.getcwd())
1093 r = ScriptRunner(args)
1094 r.writeShedyml() 962 r.writeShedyml()
1095 r.makeTool() 963 r.makeTool()
1096 r.makeToolTar() 964 r.makeToolTar()
1097 if args.run_test:
1098 if not args.packages or args.packages.strip() == "bash":
1099 r.run()
1100 r.makeToolTar()
1101 else:
1102 tt = ToolTester(report_dir=r.repdir, in_tool_archive=r.newtarpath, new_tool_archive=r.args.new_tool, galaxy_root=args.galaxy_root, include_tests=False)
1103 if args.install: 965 if args.install:
1104 #try: 966 #try:
1105 tcu = ToolConfUpdater(args=args, tool_dir=args.local_tools, 967 tcu = Tool_Conf_Updater(args=args, tool_dir=args.local_tools,
1106 new_tool_archive_path=r.newtarpath, tool_conf_path=args.tool_conf_path, 968 new_tool_archive_path=r.newtarpath, tool_conf_path=args.tool_conf_path,
1107 new_tool_name=r.tool_name) 969 new_tool_name=r.tool_name)
1108 #except Exception: 970 #except Exception:
1109 # print("### Unable to install the new tool. Are you sure you have all the required special settings?") 971 # print("### Unable to install the new tool. Are you sure you have all the required special settings?")
1110 972