Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/rgToolFactory2.py @ 63:b3254219d4fb draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Tue, 10 Nov 2020 02:33:46 +0000
parents 881bd8c81ac9
children ed1ecd161295
comparison
equal deleted inserted replaced
62:881bd8c81ac9 63:b3254219d4fb
21 # one to run the executable with the supplied test data and settings 21 # one to run the executable with the supplied test data and settings
22 # Be simpler to write the tool, then run it with planemo and soak up the test outputs. 22 # Be simpler to write the tool, then run it with planemo and soak up the test outputs.
23 23
24 24
25 import argparse 25 import argparse
26 import copy
26 import logging 27 import logging
27 import os 28 import os
28 import re 29 import re
29 import shutil 30 import shutil
30 import subprocess 31 import subprocess
31 import sys 32 import sys
32 import tarfile 33 import tarfile
33 import tempfile 34 import tempfile
34 import time 35 import time
36
37 from bioblend import galaxy
38 from bioblend import toolshed
35 39
36 import galaxyxml.tool as gxt 40 import galaxyxml.tool as gxt
37 import galaxyxml.tool.parameters as gxtp 41 import galaxyxml.tool.parameters as gxtp
38 42
39 import lxml 43 import lxml
152 self.lastclredirect = None 156 self.lastclredirect = None
153 self.lastxclredirect = None 157 self.lastxclredirect = None
154 self.cl = [] 158 self.cl = []
155 self.xmlcl = [] 159 self.xmlcl = []
156 self.is_positional = self.args.parampass == "positional" 160 self.is_positional = self.args.parampass == "positional"
157 if self.args.packages: 161 if self.args.sysexe:
158 self.executeme = self.args.packages.split(",")[0].split(":")[0]
159 else:
160 self.executeme = self.args.sysexe 162 self.executeme = self.args.sysexe
161 assert ( 163 else:
162 self.executeme is not None 164 if self.args.packages:
163 ), "No system or managed executable passed in. Cannot build" 165 self.executeme = self.args.packages.split(",")[0].split(":")[0]
166 else:
167 self.executeme = None
164 aCL = self.cl.append 168 aCL = self.cl.append
165 aXCL = self.xmlcl.append 169 aXCL = self.xmlcl.append
166 assert args.parampass in [ 170 assert args.parampass in [
167 "0", 171 "0",
168 "argparse", 172 "argparse",
475 additional parameter %s in makeXML' 479 additional parameter %s in makeXML'
476 % (newtype, newname) 480 % (newtype, newname)
477 ) 481 )
478 aparm.positional = self.is_positional 482 aparm.positional = self.is_positional
479 if self.is_positional: 483 if self.is_positional:
480 aninput.positional = int(oldcl) 484 aparm.positional = int(oldcl)
481 self.tinputs.append(aparm) 485 self.tinputs.append(aparm)
482 self.tparm = gxtp.TestParam(newname, value=newval) 486 tparm = gxtp.TestParam(newname, value=newval)
483 self.testparam.append(tparm) 487 self.testparam.append(tparm)
484 488
485 def doNoXMLparam(self): 489 def doNoXMLparam(self):
486 """filter style package - stdin to stdout""" 490 """filter style package - stdin to stdout"""
487 if len(self.infiles) > 0: 491 if len(self.infiles) > 0:
506 aninput.command_line_override = "< $%s" % newname 510 aninput.command_line_override = "< $%s" % newname
507 aninput.positional = self.is_positional 511 aninput.positional = self.is_positional
508 self.tinputs.append(aninput) 512 self.tinputs.append(aninput)
509 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) 513 tp = gxtp.TestParam(name=newname, value="%s_sample" % newname)
510 self.testparam.append(tp) 514 self.testparam.append(tp)
511 if len(self.outfiles > 0): 515 if len(self.outfiles) > 0:
512 newname = self.outfiles[0][OCLPOS] 516 newname = self.outfiles[0][OCLPOS]
513 newfmt = self.outfiles[0][OFMTPOS] 517 newfmt = self.outfiles[0][OFMTPOS]
514 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) 518 anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0)
515 anout.command_line_override = "> $%s" % newname 519 anout.command_line_override = "> $%s" % newname
516 anout.positional = self.is_positional 520 anout.positional = self.is_positional
529 else: 533 else:
530 self.newtool.command_override = self.xmlcl 534 self.newtool.command_override = self.xmlcl
531 if self.args.help_text: 535 if self.args.help_text:
532 helptext = open(self.args.help_text, "r").readlines() 536 helptext = open(self.args.help_text, "r").readlines()
533 safertext = [html_escape(x) for x in helptext] 537 safertext = [html_escape(x) for x in helptext]
534 if self.args.script_path: 538 if False and self.args.script_path:
535 scrpt = self.script.split('\n') 539 scrp = self.script.split('\n')
540 scrpt = [' %s' % x for x in scrp] # try to stop templating
536 scrpt.insert(0,"```\n") 541 scrpt.insert(0,"```\n")
537 if len(scrpt) > 300: 542 if len(scrpt) > 300:
538 safertext = safertext + scrpt[:100] + ['>500 lines - stuff deleted','......'] + scrpt[-100:] 543 safertext = safertext + scrpt[:100] + ['>500 lines - stuff deleted','......'] + scrpt[-100:]
539 else: 544 else:
540 safertext = safertext + scrpt 545 safertext = safertext + scrpt
591 ): # cannot do this inside galaxyxml as it expects lxml objects for tests 596 ): # cannot do this inside galaxyxml as it expects lxml objects for tests
592 part1 = exml.split("<tests>")[0] 597 part1 = exml.split("<tests>")[0]
593 part2 = exml.split("</tests>")[1] 598 part2 = exml.split("</tests>")[1]
594 fixed = "%s\n%s\n%s" % (part1, self.test_override, part2) 599 fixed = "%s\n%s\n%s" % (part1, self.test_override, part2)
595 exml = fixed 600 exml = fixed
601 exml = exml.replace('range="1:"', 'range="1000:"')
596 xf = open("%s.xml" % self.tool_name, "w") 602 xf = open("%s.xml" % self.tool_name, "w")
597 xf.write(exml) 603 xf.write(exml)
598 xf.write("\n") 604 xf.write("\n")
599 xf.close() 605 xf.close()
600 # ready for the tarball 606 # ready for the tarball
604 generate test outputs by running a command line 610 generate test outputs by running a command line
605 won't work if command or test override in play - planemo is the 611 won't work if command or test override in play - planemo is the
606 easiest way to generate test outputs for that case so is 612 easiest way to generate test outputs for that case so is
607 automagically selected 613 automagically selected
608 """ 614 """
609 s = "run cl=%s" % str(self.cl)
610 logging.debug(s)
611 scl = " ".join(self.cl) 615 scl = " ".join(self.cl)
612 err = None 616 err = None
613 if self.args.parampass != "0": 617 if self.args.parampass != "0":
614 if os.path.exists(self.elog): 618 if os.path.exists(self.elog):
615 ste = open(self.elog, "a") 619 ste = open(self.elog, "a")
621 if os.path.exists(self.tlog): 625 if os.path.exists(self.tlog):
622 sto = open(self.tlog, "a") 626 sto = open(self.tlog, "a")
623 else: 627 else:
624 sto = open(self.tlog, "w") 628 sto = open(self.tlog, "w")
625 sto.write( 629 sto.write(
626 "## Executing Toolfactory generated command line = %s\n" % scl 630 "## Executing Toolfactory generated command line = %s\n" % scl
627 ) 631 )
628 sto.flush() 632 sto.flush()
629 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste) 633 p = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)
630 sto.close() 634 sto.close()
631 ste.close() 635 ste.close()
632 retval = p.returncode 636 retval = p.returncode
633 else: # work around special case - stdin and write to stdout 637 else: # work around special case - stdin and write to stdout
634 if len(self.infiles) > 0: 638 if len(self.infiles) > 0:
635 sti = open(self.infiles[0][IPATHPOS], "rb") 639 sti = open(self.infiles[0][IPATHPOS], "rb")
636 else: 640 else:
637 sti = sys.stdin 641 sti = sys.stdin
638 if len(self.outfiles) > 0: 642 if len(self.outfiles) > 0:
639 sto = open(self.outfiles[0][ONAMEPOS], "wb") 643 sto = open(self.outfiles[0][ONAMEPOS], "wb")
640 else: 644 else:
641 sto = sys.stdout 645 sto = sys.stdout
642 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) 646 p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)
647 sto.write(
648 "## Executing Toolfactory generated command line = %s\n" % scl
649 )
643 retval = p.returncode 650 retval = p.returncode
644 sto.close() 651 sto.close()
645 sti.close() 652 sti.close()
646 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0: 653 if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
647 os.unlink(self.tlog) 654 os.unlink(self.tlog)
650 if retval != 0 and err: # problem 657 if retval != 0 and err: # problem
651 sys.stderr.write(err) 658 sys.stderr.write(err)
652 logging.debug("run done") 659 logging.debug("run done")
653 return retval 660 return retval
654 661
655 def planemo_shedload(self): 662
656 """ 663 def shedLoad(self):
657 planemo shed_create --shed_target testtoolshed 664 """
658 planemo shed_update --check_diff --shed_target testtoolshed 665 {'deleted': False,
666 'description': 'Tools for manipulating data',
667 'id': '175812cd7caaf439',
668 'model_class': 'Category',
669 'name': 'Text Manipulation',
670 'url': '/api/categories/175812cd7caaf439'}]
671
672
659 """ 673 """
660 if os.path.exists(self.tlog): 674 if os.path.exists(self.tlog):
661 tout = open(self.tlog, "a") 675 sto = open(self.tlog, "a")
662 else: 676 else:
663 tout = open(self.tlog, "w") 677 sto = open(self.tlog, "w")
664 cll = ["planemo", "shed_create", "--shed_target", "local"] 678
665 try: 679 ts = toolshed.ToolShedInstance(url=self.args.toolshed_url,key=self.args.toolshed_api_key,verify=False)
666 p = subprocess.run( 680 repos = ts.repositories.get_repositories()
667 cll, shell=True, cwd=self.tooloutdir, stdout=tout, stderr=tout 681 rnames = [x.get('name','?') for x in repos]
668 ) 682 rids = [x.get('id','?') for x in repos]
669 except: 683 sto.write(f'############names={rnames} rids={rids}')
670 pass 684 cat = 'ToolFactory generated tools'
671 if p.returncode != 0: 685 if self.args.tool_name not in rnames:
672 print("Repository %s exists" % self.args.tool_name) 686 tscat = ts.categories.get_categories()
673 else: 687 cnames = [x.get('name','?') for x in tscat]
674 print("initiated %s" % self.args.tool_name) 688 cids = [x.get('id','?') for x in tscat]
675 cll = [ 689 catID = None
676 "planemo", 690 if cat in cnames:
677 "shed_upload", 691 ci = cnames.index(cat)
678 "--shed_target", 692 catID = cids[ci]
679 "local", 693 res = ts.repositories.create_repository(name=self.args.tool_name, synopsis='Synopsis:%s' % self.args.tool_desc, description=self.args.tool_desc,
680 "--owner", 694 type='unrestricted', remote_repository_url=self.args.toolshed_url,
681 "fubar", 695 homepage_url=None, category_ids=catID)
682 "--name", 696 tid = res.get('id',None)
683 self.args.tool_name, 697 sto.write(f'##########create res={res}')
684 "--shed_key", 698 else:
685 self.args.toolshed_api_key, 699 i = rnames.index(self.args.tool_name)
686 "--tar", 700 tid = rids[i]
687 self.newtarpath, 701 res = ts.repositories.update_repository(id=tid, tar_ball_path=self.newtarpath, commit_message=None)
688 ] 702 sto.write(f'#####update res={res}')
689 p = subprocess.run(cll, shell=False) 703 sto.close()
690 print("Ran", " ".join(cll), "got", p.returncode) 704
691 tout.close()
692 return p.returncode
693
694 def planemo_test(self, genoutputs=True):
695 """planemo is a requirement so is available for testing
696 and for generating test outputs if command or test overrides are supplied
697 test outputs are sent to repdir for display
698 """
699 xreal = "%s.xml" % self.tool_name
700 if os.path.exists(self.tlog):
701 tout = open(self.tlog, "a")
702 else:
703 tout = open(self.tlog, "w")
704 if genoutputs:
705 cll = [
706 "planemo",
707 "test",
708 "--galaxy_root",
709 self.args.galaxy_root,
710 "--update_test_data",
711 xreal,
712 ]
713 else:
714 cll = ["planemo", "test", "--galaxy_root",
715 self.args.galaxy_root,
716 xreal,]
717 p = subprocess.run(
718 cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
719 )
720 if genoutputs:
721 with os.scandir(self.testdir) as outs:
722 for entry in outs:
723 if entry.is_file():
724 dest = os.path.join(self.repdir, entry.name)
725 src = os.path.join(self.testdir, entry.name)
726 shutil.copyfile(src, dest)
727 tout.write(
728 "Copied output %s to %s after planemo test\n" % (src, dest)
729 )
730 tout.close()
731 return p.returncode
732 705
733 def eph_galaxy_load(self): 706 def eph_galaxy_load(self):
734 """load the new tool from the local toolshed after planemo uploads it 707 """load the new tool from the local toolshed after planemo uploads it
735 """ 708 """
736 if os.path.exists(self.tlog): 709 if os.path.exists(self.tlog):
749 self.args.tool_name, 722 self.args.tool_name,
750 "--owner", 723 "--owner",
751 "fubar", 724 "fubar",
752 "--toolshed", 725 "--toolshed",
753 self.args.toolshed_url, 726 self.args.toolshed_url,
754 "--section_label", 727 "--section_label",
755 "Generated Tools", 728 "ToolFactory",
756 729
757 ] 730 ]
758 print("running\n", " ".join(cll)) 731 tout.write("running\n%s\n" % " ".join(cll))
759 p = subprocess.run(cll, shell=False, stderr=tout, stdout=tout) 732 p = subprocess.run(cll, shell=False, stderr=tout, stdout=tout)
760 if p.returncode != 0: 733 tout.write("installed %s - got retcode %d\n" % (self.args.tool_name,p.returncode))
761 print(
762 "Repository %s installation returned %d"
763 % (self.args.tool_name, p.returncode)
764 )
765 else:
766 print("installed %s" % self.args.tool_name)
767 tout.close() 734 tout.close()
768 return p.returncode 735 return p.returncode
736
737
738 def planemo_shedload(self):
739 """
740 planemo shed_create --shed_target testtoolshed
741 planemo shed_init --name=<name>
742 --owner=<shed_username>
743 --description=<short description>
744 [--remote_repository_url=<URL to .shed.yml on github>]
745 [--homepage_url=<Homepage for tool.>]
746 [--long_description=<long description>]
747 [--category=<category name>]*
748
749
750 planemo shed_update --check_diff --shed_target testtoolshed
751 """
752 if os.path.exists(self.tlog):
753 tout = open(self.tlog, "a")
754 else:
755 tout = open(self.tlog, "w")
756 ts = toolshed.ToolShedInstance(url=self.args.toolshed_url,key=self.args.toolshed_api_key,verify=False)
757 repos = ts.repositories.get_repositories()
758 rnames = [x.get('name','?') for x in repos]
759 rids = [x.get('id','?') for x in repos]
760 tout.write(f'############names={rnames} rids={rids}')
761 cat = 'ToolFactory generated tools'
762 if self.args.tool_name not in rnames:
763 cll = ["planemo", "shed_create", "--shed_target", "local",
764 "--owner","fubar","--name",
765 self.args.tool_name,"--shed_key",
766 self.args.toolshed_api_key,]
767 try:
768 p = subprocess.run(
769 cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
770 )
771 except:
772 pass
773 if p.returncode != 0:
774 tout.write("Repository %s exists" % self.args.tool_name)
775 else:
776 tout.write("initiated %s" % self.args.tool_name)
777 cll = [
778 "planemo",
779 "shed_upload",
780 "--shed_target",
781 "local",
782 "--owner",
783 "fubar",
784 "--name",
785 self.args.tool_name,
786 "--shed_key",
787 self.args.toolshed_api_key,
788 "--tar",
789 self.newtarpath,
790 ]
791 p = subprocess.run(cll, shell=False, stdout=tout, stderr=tout)
792 tout.write("Ran %s got %d" % (" ".join(cll), p.returncode))
793 tout.close()
794 return p.returncode
795
796
797 def eph_test(self):
798 """
799 """
800 if os.path.exists(self.tlog):
801 tout = open(self.tlog, "a")
802 else:
803 tout = open(self.tlog, "w")
804 cll = [
805 "shed-tools",
806 "test",
807 "-g",
808 self.args.galaxy_url,
809 "-a",
810 self.args.galaxy_api_key,
811 "--name",
812 self.args.tool_name,
813 "--owner",
814 "fubar",
815 ]
816 p = subprocess.run(
817 cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
818 )
819 tout.write("eph_test Ran %s got %d" % (" ".join(cll), p.returncode))
820 tout.close()
821 return p.returncode
822
823 def planemo_test(self, genoutputs=True):
824 """planemo is a requirement so is available for testing
825 and for generating test outputs if command or test overrides are supplied
826 test outputs are sent to repdir for display
827 """
828 xreal = "%s.xml" % self.tool_name
829 if os.path.exists(self.tlog):
830 tout = open(self.tlog, "a")
831 else:
832 tout = open(self.tlog, "w")
833 if genoutputs:
834 dummy,tfile = tempfile.mkstemp()
835 cll = [
836 "planemo",
837 "test",
838 "--galaxy_root",
839 self.args.galaxy_root,
840 "--update_test_data",
841 xreal,
842 ]
843 p = subprocess.run(
844 cll, shell=False, cwd=self.tooloutdir, stderr=dummy, stdout=dummy,
845 )
846 dummy.close() # throw all the log away as it will be rerun after outputs are generated
847 else:
848 cll = ["planemo", "test", "--galaxy_root",
849 self.args.galaxy_root,
850 xreal,]
851 p = subprocess.run(
852 cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
853 )
854 tout.close()
855 return p.returncode
856
857
769 858
770 def writeShedyml(self): 859 def writeShedyml(self):
771 """for planemo 860 """for planemo
772 """ 861 """
773 yuser = self.args.user_email.split("@")[0] 862 yuser = self.args.user_email.split("@")[0]
870 a("--edit_additional_parameters", action="store_true", default=False) 959 a("--edit_additional_parameters", action="store_true", default=False)
871 a("--parampass", default="positional") 960 a("--parampass", default="positional")
872 a("--tfout", default="./tfout") 961 a("--tfout", default="./tfout")
873 a("--new_tool", default="new_tool") 962 a("--new_tool", default="new_tool")
874 a("--galaxy_url", default="http://localhost:8080") 963 a("--galaxy_url", default="http://localhost:8080")
964 a("--toolshed_url", default="http://localhost:9009") # make sure this is NOT 127.0.0.1 - it won't work if tool_sheds_conf.xml has localhost
965 #a("--galaxy_api_key", default="1e62ddad74fe9bf112859f4e9efea48b")
966 #a("--toolshed_api_key", default="9154c91f2a162bf12fda15764f43846c")
967
968 a("--toolshed_api_key", default="fakekey")
875 a("--galaxy_api_key", default="fakekey") 969 a("--galaxy_api_key", default="fakekey")
876 a("--toolshed_url", default="http://localhost:9009")
877 a("--toolshed_api_key", default="fakekey")
878 a("--galaxy_root", default="/galaxy-central") 970 a("--galaxy_root", default="/galaxy-central")
971
879 972
880 args = parser.parse_args() 973 args = parser.parse_args()
881 assert not args.bad_user, ( 974 assert not args.bad_user, (
882 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' 975 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file'
883 % (args.bad_user, args.bad_user) 976 % (args.bad_user, args.bad_user)
891 for i, x in enumerate(args.additional_parameters): 984 for i, x in enumerate(args.additional_parameters):
892 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "") 985 args.additional_parameters[i] = args.additional_parameters[i].replace('"', "")
893 r = ScriptRunner(args) 986 r = ScriptRunner(args)
894 r.writeShedyml() 987 r.writeShedyml()
895 r.makeTool() 988 r.makeTool()
896 if args.command_override or args.test_override: 989 retcode = r.planemo_test(genoutputs=True) # this fails :( - see PR
897 retcode = r.planemo_test(genoutputs=True) # this fails :( - see PR 990 r.moveRunOutputs()
898 r.moveRunOutputs() 991 r.makeToolTar()
899 r.makeToolTar() 992 retcode = r.planemo_test(genoutputs=False)
900 retcode = r.planemo_test(genoutputs=False) 993 r.moveRunOutputs()
901 r.moveRunOutputs() 994 if args.make_Tool == "gentestinstall":
902 if args.make_Tool == "gentestinstall": 995 retcode = r.planemo_shedload() #r.shedLoad()
903 r.planemo_shedload() 996 print(f'planemo_shedload returned {retcode}')
904 r.eph_galaxy_load() 997 r.eph_galaxy_load()
905 else: 998
906 retcode = r.run()
907 if retcode:
908 sys.stderr.write(
909 "## Run failed with return code %d. Cannot build yet. Please fix and retry"
910 % retcode
911 )
912 sys.exit(1)
913 else:
914 r.moveRunOutputs()
915 r.makeToolTar()
916 if args.make_Tool in ["gentestinstall", "gentest"]:
917 r.planemo_test(genoutputs=False)
918 r.moveRunOutputs()
919 if args.make_Tool == "gentestinstall":
920 r.planemo_shedload()
921 r.eph_galaxy_load()
922 999
923 1000
924 if __name__ == "__main__": 1001 if __name__ == "__main__":
925 main() 1002 main()