Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 112:5509dc4c1cf2 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Sun, 29 Nov 2020 10:34:48 +0000
parents 223b78754735
children 6873c211b250
comparison
equal deleted inserted replaced
111:223b78754735 112:5509dc4c1cf2
12 # Decided to 12 # Decided to
13 # 1. Fix the toolfactory so it works - done for simplest case 13 # 1. Fix the toolfactory so it works - done for simplest case
14 # 2. Fix planemo so the toolfactory function works 14 # 2. Fix planemo so the toolfactory function works
15 # 3. Rewrite bits using galaxyxml functions where that makes sense - done 15 # 3. Rewrite bits using galaxyxml functions where that makes sense - done
16 # 16 #
17 # removed all the old complications including making the new tool use this same script 17 # uses planemo in a biodocker sort of image as a requirement
18 # galaxyxml now generates the tool xml https://github.com/hexylena/galaxyxml 18 # otherwise planemo seems to leak dependencies back into the
19 # No support for automatic HTML file creation from arbitrary outputs 19 # calling venv. Hilarity ensues.
20 # essential problem is to create two command lines - one for the tool xml and a different 20
21 # one to run the executable with the supplied test data and settings 21
22 # Be simpler to write the tool, then run it with planemo and soak up the test outputs.
23 # well well. sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox
24 # does the needful. Use GALAXY_TEST_SAVE /foo to save outputs - only the tar.gz - not the rest sadly
25 # GALAXY_TEST_NO_CLEANUP GALAXY_TEST_TMP_DIR=wherever
26 # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml
27 22
28 import argparse 23 import argparse
29 import copy 24 import copy
30 import datetime 25 import datetime
31 import grp 26 import grp
56 myversion = "V2.1 July 2020" 51 myversion = "V2.1 July 2020"
57 verbose = True 52 verbose = True
58 debug = True 53 debug = True
59 toolFactoryURL = "https://github.com/fubar2/toolfactory" 54 toolFactoryURL = "https://github.com/fubar2/toolfactory"
60 ourdelim = "~~~" 55 ourdelim = "~~~"
61 ALOT = 10000000 # srsly. command or test overrides use read() so just in case
62 STDIOXML = """<stdio>
63 <exit_code range="100:" level="debug" description="shite happens" />
64 </stdio>"""
65 56
66 # --input_files="$intab.input_files~~~$intab.input_CL~~~$intab.input_formats\ 57 # --input_files="$intab.input_files~~~$intab.input_CL~~~$intab.input_formats\
67 #~~~$intab.input_label~~~$intab.input_help" 58 #~~~$intab.input_label~~~$intab.input_help"
68 IPATHPOS = 0 59 IPATHPOS = 0
69 ICLPOS = 1 60 ICLPOS = 1
261 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""] 252 appendme = [p[IOCLPOS], p[ICLPOS], p[IPATHPOS], ""]
262 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] 253 xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""]
263 clsuffix.append(appendme) 254 clsuffix.append(appendme)
264 xclsuffix.append(xappendme) 255 xclsuffix.append(xappendme)
265 for i, p in enumerate(self.outfiles): 256 for i, p in enumerate(self.outfiles):
266 if p[OCLPOS] == "STDOUT": 257 if p[OOCLPOS] == "STDOUT":
267 self.lastclredirect = [">", p[ONAMEPOS]] 258 self.lastclredirect = [">", p[ONAMEPOS]]
268 self.lastxclredirect = [">", "$%s" % p[OCLPOS]] 259 self.lastxclredirect = [">", "$%s" % p[OCLPOS]]
269 else: 260 else: # for (o_v, k, v, koverride) in self.xclsuffix:
270 clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""]) 261 clsuffix.append([p[ONAMEPOS], p[ONAMEPOS], p[ONAMEPOS], ""])
271 xclsuffix.append([p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""]) 262 xclsuffix.append([p[ONAMEPOS], p[ONAMEPOS], "$%s" % p[ONAMEPOS], ""])
272 for p in self.addpar: 263 for p in self.addpar:
273 clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]]) 264 clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]])
274 xclsuffix.append( 265 xclsuffix.append(
275 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] 266 [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]]
276 ) 267 )
294 ) 285 )
295 tscript = open(self.sfile, "w") 286 tscript = open(self.sfile, "w")
296 tscript.write(self.script) 287 tscript.write(self.script)
297 tscript.close() 288 tscript.close()
298 self.escapedScript = [cheetah_escape(x) for x in rx] 289 self.escapedScript = [cheetah_escape(x) for x in rx]
299 self.spacedScript = [f" {x}" for x in rx] 290 self.spacedScript = [f" {x}" for x in rx if x.strip() > ""]
300 art = "%s.%s" % (self.tool_name, self.executeme) 291 art = "%s.%s" % (self.tool_name, self.executeme)
301 artifact = open(art, "wb") 292 artifact = open(art, "wb")
302 artifact.write(bytes('\n'.join(self.escapedScript),'utf8')) 293 artifact.write(bytes('\n'.join(self.escapedScript),'utf8'))
303 artifact.close() 294 artifact.close()
304 295
328 ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % ( 319 ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
329 p[OCLPOS], 320 p[OCLPOS],
330 p[ONAMEPOS], 321 p[ONAMEPOS],
331 ) 322 )
332 p.append(p[OCLPOS]) # keep copy 323 p.append(p[OCLPOS]) # keep copy
333 if p[OCLPOS].isdigit() or p[OCLPOS] == "STDOUT": 324 if p[OOCLPOS].isdigit() or p[OOCLPOS] == "STDOUT":
334 scl = p[ONAMEPOS] 325 scl = p[ONAMEPOS]
335 p[OCLPOS] = scl 326 p[OCLPOS] = scl
336 self.outfiles[i] = p 327 self.outfiles[i] = p
337 for i, p in enumerate(self.addpar): 328 for i, p in enumerate(self.addpar):
338 if self.args.parampass == "positional": 329 if self.args.parampass == "positional":
376 for (o_v, k, v, koverride) in self.xclsuffix: 367 for (o_v, k, v, koverride) in self.xclsuffix:
377 aXCL(v) 368 aXCL(v)
378 if self.lastxclredirect: 369 if self.lastxclredirect:
379 aXCL(self.lastxclredirect[0]) 370 aXCL(self.lastxclredirect[0])
380 aXCL(self.lastxclredirect[1]) 371 aXCL(self.lastxclredirect[1])
372 if os.path.exists(self.tlog):
373 tout = open(self.tlog, "a")
374 else:
375 tout = open(self.tlog, "w")
376 tout.write(f" #### clpositional: self.clsuffix = {self.clsuffix} and self.xclsuffix = {self.xclsuffix}")
377 tout.close()
378
381 379
382 def clargparse(self): 380 def clargparse(self):
383 """argparse style""" 381 """argparse style"""
384 aCL = self.cl.append 382 aCL = self.cl.append
385 aXCL = self.xmlcl.append 383 aXCL = self.xmlcl.append
386 # inputs then params in argparse named form 384 # inputs then params in argparse named form
385
387 for (o_v, k, v, koverride) in self.xclsuffix: 386 for (o_v, k, v, koverride) in self.xclsuffix:
388 if koverride > "": 387 if koverride > "":
389 k = koverride 388 k = koverride
390 elif len(k.strip()) == 1: 389 elif len(k.strip()) == 1:
391 k = "-%s" % k 390 k = "-%s" % k
400 k = "-%s" % k 399 k = "-%s" % k
401 else: 400 else:
402 k = "--%s" % k 401 k = "--%s" % k
403 aCL(k) 402 aCL(k)
404 aCL(v) 403 aCL(v)
404 if os.path.exists(self.tlog):
405 tout = open(self.tlog, "a")
406 else:
407 tout = open(self.tlog, "w")
408 tout.write(f" #### clargparse: self.clsuffix = {self.clsuffix} and self.xclsuffix = {self.xclsuffix}")
409 tout.close()
410
405 411
406 def getNdash(self, newname): 412 def getNdash(self, newname):
407 if self.is_positional: 413 if self.is_positional:
408 ndash = 0 414 ndash = 0
409 else: 415 else:
891 "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode) 897 "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
892 ) 898 )
893 tout.close() 899 tout.close()
894 return subp.returncode 900 return subp.returncode
895 901
896 def planemo_shedLoad(self): 902
897 """
898 planemo shed_create --shed_target testtoolshed
899 planemo shed_init --name=<name>
900 --owner=<shed_username>
901 --description=<short description>
902 [--remote_repository_url=<URL to .shed.yml on github>]
903 [--homepage_url=<Homepage for tool.>]
904 [--long_description=<long description>]
905 [--category=<category name>]*
906
907
908 planemo shed_update --check_diff --shed_target testtoolshed
909 """
910 if os.path.exists(self.tlog):
911 tout = open(self.tlog, "a")
912 else:
913 tout = open(self.tlog, "w")
914 ts = toolshed.ToolShedInstance(
915 url=self.args.toolshed_url, key=self.args.toolshed_api_key, verify=False
916 )
917 repos = ts.repositories.get_repositories()
918 rnames = [x.get("name", "?") for x in repos]
919 rids = [x.get("id", "?") for x in repos]
920 # cat = "ToolFactory generated tools"
921 if self.tool_name not in rnames:
922 cll = [
923 "planemo",
924 "shed_create",
925 "--shed_target",
926 "local",
927 "--owner",
928 "fubar",
929 "--name",
930 self.tool_name,
931 "--shed_key",
932 self.args.toolshed_api_key,
933 ]
934 try:
935 subp = subprocess.run(
936 cll,
937 env=self.ourenv,
938 shell=False,
939 cwd=self.tooloutdir,
940 stdout=tout,
941 stderr=tout,
942 )
943 except:
944 pass
945 if subp.returncode != 0:
946 tout.write("Repository %s exists\n" % self.tool_name)
947 else:
948 tout.write("initiated %s\n" % self.tool_name)
949 cll = [
950 "planemo",
951 "shed_upload",
952 "--shed_target",
953 "local",
954 "--owner",
955 "fubar",
956 "--name",
957 self.tool_name,
958 "--shed_key",
959 self.args.toolshed_api_key,
960 "--tar",
961 self.newtarpath,
962 ]
963 subp = subprocess.run(
964 cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout
965 )
966 tout.write("Ran %s got %d\n" % (" ".join(cll), subp.returncode))
967 tout.close()
968 return subp.returncode
969
970 def eph_test(self, genoutputs=True):
971 """problem getting jobid - ephemeris upload is the job before the one we want - but depends on how many inputs"""
972 if os.path.exists(self.tlog):
973 tout = open(self.tlog, "a")
974 else:
975 tout = open(self.tlog, "w")
976 cll = [
977 "shed-tools",
978 "test",
979 "-g",
980 self.args.galaxy_url,
981 "-a",
982 self.args.galaxy_api_key,
983 "--name",
984 self.tool_name,
985 "--owner",
986 "fubar",
987 ]
988 if genoutputs:
989 dummy, tfile = tempfile.mkstemp()
990 subp = subprocess.run(
991 cll,
992 env=self.ourenv,
993 cwd=self.ourcwd,
994 shell=False,
995 stderr=dummy,
996 stdout=dummy,
997 )
998
999 with open("tool_test_output.json", "rb") as f:
1000 s = json.loads(f.read())
1001 print("read %s" % s)
1002 cl = s["tests"][0]["data"]["job"]["command_line"].split()
1003 n = cl.index("--script_path")
1004 jobdir = cl[n + 1]
1005 jobdir = jobdir.replace('"', "")
1006 jobdir = jobdir.split("/configs")[0]
1007 print("jobdir=%s" % jobdir)
1008
1009 # "/home/ross/galthrow/database/jobs_directory/000/649/configs/tmptfxu51gs\"
1010 src = os.path.join(jobdir, "working", self.newtarpath)
1011 if os.path.exists(src):
1012 dest = os.path.join(self.testdir, self.newtarpath)
1013 shutil.copyfile(src, dest)
1014 else:
1015 tout.write(
1016 "No toolshed archive found after first ephemeris test - not a good sign"
1017 )
1018 ephouts = os.path.join(jobdir, "working", "tfout", "test-data")
1019 with os.scandir(ephouts) as outs:
1020 for entry in outs:
1021 if not entry.is_file():
1022 continue
1023 dest = os.path.join(self.tooloutdir, entry.name)
1024 src = os.path.join(ephouts, entry.name)
1025 shutil.copyfile(src, dest)
1026 else:
1027 subp = subprocess.run(
1028 cll,
1029 env=self.ourenv,
1030 cwd=self.ourcwd,
1031 shell=False,
1032 stderr=tout,
1033 stdout=tout,
1034 )
1035 tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))
1036 tout.close()
1037 return subp.returncode
1038
1039 def planemo_test_biocontainer(self, genoutputs=True):
1040 """planemo is a requirement so is available for testing but testing in a biocontainer
1041 requires some fiddling to use the hacked galaxy-central .venv
1042
1043 Planemo runs:
1044 python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file
1045 /export/galaxy-central/database/job_working_directory/000/17/working/TF_run_report_tempdir/tacrev_planemo_test_report.html
1046 --with-xunit --xunit-file /tmp/tmpt90p7f9h/xunit.xml --with-structureddata
1047 --structured-data-file
1048 /export/galaxy-central/database/job_working_directory/000/17/working/tfout/tool_test_output.json functional.test_toolbox
1049
1050
1051 for the planemo-biocontainer,
1052 planemo test --conda_dependency_resolution --skip_venv --galaxy_root /galthrow/ rgToolFactory2.xml
1053
1054 """
1055 xreal = "%s.xml" % self.tool_name
1056 tool_test_path = os.path.join(
1057 self.repdir, f"{self.tool_name}_planemo_test_report.html"
1058 )
1059 if os.path.exists(self.tlog):
1060 tout = open(self.tlog, "a")
1061 else:
1062 tout = open(self.tlog, "w")
1063 if genoutputs:
1064 dummy, tfile = tempfile.mkstemp()
1065 cll = [
1066 ".",
1067 os.path.join(self.args.galaxy_root, ".venv", "bin", "activate"),
1068 "&&",
1069 "planemo",
1070 "test",
1071 "--test_data",
1072 self.testdir,
1073 "--test_output",
1074 tool_test_path,
1075 "--skip_venv",
1076 "--galaxy_root",
1077 self.args.galaxy_root,
1078 "--update_test_data",
1079 xreal,
1080 ]
1081 subp = subprocess.run(
1082 cll,
1083 env=self.ourenv,
1084 shell=False,
1085 cwd=self.tooloutdir,
1086 stderr=dummy,
1087 stdout=dummy,
1088 )
1089
1090 else:
1091 cll = [
1092 ".",
1093 os.path.join(self.args.galaxy_root, ".venv", "bin", "activate"),
1094 "&&",
1095 "planemo",
1096 "test",
1097 "--test_data",
1098 os.path.self.testdir,
1099 "--test_output",
1100 os.path.tool_test_path,
1101 "--skip_venv",
1102 "--galaxy_root",
1103 self.args.galaxy_root,
1104 xreal,
1105 ]
1106 subp = subprocess.run(
1107 cll,
1108 env=self.ourenv,
1109 shell=False,
1110 cwd=self.tooloutdir,
1111 stderr=tout,
1112 stdout=tout,
1113 )
1114 tout.close()
1115 return subp.returncode
1116 903
1117 def writeShedyml(self): 904 def writeShedyml(self):
1118 """for planemo""" 905 """for planemo"""
1119 yuser = self.args.user_email.split("@")[0] 906 yuser = self.args.user_email.split("@")[0]
1120 yfname = os.path.join(self.tooloutdir, ".shed.yml") 907 yfname = os.path.join(self.tooloutdir, ".shed.yml")