changeset 101:557d5f06f213 draft

Uploaded

--- a/toolfactory/rgToolFactory2.py	Mon Nov 23 03:12:37 2020 +0000
+++ b/toolfactory/rgToolFactory2.py	Thu Nov 26 06:17:12 2020 +0000
@@ -26,6 +26,7 @@
 # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml
 
 import argparse
+import copy
 import datetime
 import json
 import logging
@@ -42,7 +43,7 @@
 from bioblend import ConnectionError
 from bioblend import toolshed
 
-# import docker
+import docker
 
 import galaxyxml.tool as gxt
 import galaxyxml.tool.parameters as gxtp
@@ -153,6 +154,8 @@
         prepare command line cl for running the tool here
         and prepare elements needed for galaxyxml tool generation
         """
+        self.ourcwd = os.getcwd()
+        self.ourenv = copy.deepcopy(os.environ)
         self.infiles = [x.split(ourdelim) for x in args.input_files]
         self.outfiles = [x.split(ourdelim) for x in args.output_files]
         self.addpar = [x.split(ourdelim) for x in args.additional_parameters]
@@ -180,7 +183,7 @@
         self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
         self.tool_id = self.tool_name
         self.newtool = gxt.Tool(
-            self.args.tool_name,
+            self.tool_name,
             self.tool_id,
             self.args.tool_version,
             self.args.tool_desc,
@@ -645,7 +648,7 @@
                     "## Executing Toolfactory generated command line = %s\n" % scl
                 )
             sto.flush()
-            subp = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)
+            subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste)
             sto.close()
             ste.close()
             retval = subp.returncode
@@ -658,7 +661,7 @@
                 sto = open(self.outfiles[0][ONAMEPOS], "wb")
             else:
                 sto = sys.stdout
-            subp = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)
+            subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti)
             sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
             retval = subp.returncode
             sto.close()
@@ -673,33 +676,221 @@
         return retval
 
 
-    def gal_tool_test(self):
+    def copy_to_container(self, src, dest, container):
+        """ Recreate the src directory tree at dest - full path included
+        """
+        idir = os.getcwd()
+        workdir = os.path.dirname(src)
+        os.chdir(workdir)
+        _, tfname = tempfile.mkstemp(suffix=".tar")
+        tar = tarfile.open(tfname, mode='w')
+        srcb = os.path.basename(src)
+        tar.add(srcb)
+        tar.close()
+        data = open(tfname, 'rb').read()
+        container.put_archive(dest, data)
+        os.unlink(tfname)
+        os.chdir(idir)
+
+
+    def copy_from_container(self, src, dest, container):
+        """ recreate the src directory tree at dest using docker sdk
+        """
+        os.makedirs(dest,exist_ok=True)
+        _, tfname = tempfile.mkstemp(suffix=".tar")
+        tf = open(tfname,'wb')
+        bits, stat = container.get_archive(src)
+        for chunk in bits:
+            tf.write(chunk)
+        tf.close()
+        tar = tarfile.open(tfname,'r')
+        tar.extractall(dest)
+        tar.close()
+        os.unlink(tfname)
+
+
+
+    def planemo_biodocker_test(self):
+        """planemo currently leaks dependencies if used in the same container and gets unhappy after a
+        first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
+
+        Docker biocontainer has planemo with caches filled to save repeated downloads
+
+
         """
-        This handy script writes test outputs even if they don't exist
-        galaxy-tool-test [-h] [-u GALAXY_URL] [-k KEY] [-a ADMIN_KEY] [--force_path_paste] [-t TOOL_ID] [--tool-version TOOL_VERSION]
-        [-i TEST_INDEX] [-o OUTPUT] [--append] [-j OUTPUT_JSON] [--verbose] [-c CLIENT_TEST_CONFIG]
-        galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose
-        handy - just leaves outputs in -o
+        if os.path.exists(self.tlog):
+            tout = open(self.tlog, "a")
+        else:
+            tout = open(self.tlog, "w")
+        planemoimage = "quay.io/fubar2/planemo-biocontainer"
+        xreal = "%s.xml" % self.tool_name
+        destdir = "/tmp/tfout"
+        repname = f"{self.tool_name}_planemo_test_report.html"
+        imrep  = os.path.join(destdir,repname)
+        ptestrep_path = os.path.join(self.repdir,repname)
+        tool_name = self.tool_name
+        client = docker.from_env()
+        container = client.containers.run(planemoimage,'sleep 10000m', detach=True)
+        rlog = container.exec_run(f"mkdir -p {destdir}")
+        slogl = str(rlog).split('\\n')
+        slog = '\n'.join(slogl)
+        tout.write(f"## got rlog {slog} from mkdir {destdir}")
+        ptestpath = os.path.join(destdir,xreal)
+        self.copy_to_container(self.tooloutdir,'/tmp',container)
+        rlog = container.exec_run(f"ls -la {destdir}")
+        ptestcl = f"planemo test  --update_test_data  --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+        try:
+            rlog = container.exec_run(ptestcl)
+        except:
+            e = sys.exc_info()[0]
+            tout.write(f"#### error: {e} from {ptestcl}")
+        # fails - used to generate test outputs
+        ptestcl = f"planemo test  --test_output {imrep} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+        try:
+            rlog = container.exec_run(ptestcl)
+        except:
+            pass
+        slogl = str(rlog).split('\\n')
+        slog = '\n'.join(slogl)
+        tout.write(f"## got rlog {slog} from mkdir {destdir}")
+        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".")
+        self.copy_from_container(destdir,testouts,container)
+        try:
+            shutil.rmtree(os.path.join(testouts,self.tooloutdir,'test-data','test-data'))
+        except:
+            e = sys.exc_info()[0]
+            tout.write(f"#### error: {e} from {ptestcl}")
+        shutil.copytree(os.path.join(testouts,self.tooloutdir), self.tooloutdir, dirs_exist_ok=True)
+        tout.close()
+        container.stop()
+        container.remove()
+        shutil.rmtree(testouts)
+
+
+    def planemo_biodocker_vol_test(self):
+        """planemo currently leaks dependencies if used in the same container and gets unhappy after a
+        first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
+
+        Docker biocontainer has planemo with caches filled to save repeated downloads
+        Cannot get volumes to work right in this version
+
         """
         if os.path.exists(self.tlog):
             tout = open(self.tlog, "a")
         else:
             tout = open(self.tlog, "w")
-        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp")
+        planemoimage = "quay.io/fubar2/planemo-biocontainer"
+        xreal = "%s.xml" % self.tool_name
+        repname = f"{self.tool_name}_planemo_test_report.html"
+        ptestrep_path = os.path.join(self.repdir,repname)
+        tool_name = self.tool_name
+        workdir = "export"
+        aworkdir = os.path.abspath(workdir)
+        os.makedirs(workdir, exist_ok=True)
+        os.chmod(workdir,0o777)
+        imworkdir = "/export"
+        # must be mounted as a volume
+        tooldir = os.path.join(workdir,self.tool_name)
+        testdir = os.path.join(tooldir,'test-data')
+        imtooldir = os.path.join(imworkdir,self.tool_name)
+        imtestdir = os.path.join(imtooldir,'test-data')
+        for d in [tooldir,testdir]:
+            if not os.path.exists(d):
+                os.mkdir(d)
+        with os.scandir(self.testdir) as outs:
+            for entry in outs:
+                if not entry.is_file():
+                    continue
+                src = os.path.join(self.testdir, entry.name)
+                dest = os.path.join(testdir, entry.name)
+                shutil.copyfile(src, dest)
+        shutil.copyfile(xreal,os.path.join(tooldir,xreal))
+        client = docker.from_env()
+        # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],)
+        atestcl = "ls -lt /export"
+        container = client.containers.run(planemoimage,atestcl,
+           volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
+        tout.write(f"## Ran {atestcl} and got {container}")
+        ptestpath = os.path.join(imtooldir,xreal)
+        ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+        try:
+            container = client.containers.run(planemoimage,ptestcl,
+               volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
+        except:
+            pass
+        tout.write(f"## Ran {ptestcl}")
+        with os.scandir(testdir) as outs:
+            for entry in outs:
+                if not entry.is_file():
+                    continue
+                src = os.path.join(testdir, entry.name)
+                dest = os.path.join(self.testdir, entry.name)
+                shutil.copyfile(src, dest)
+        imrep_path = os.path.join(imtooldir,repname)
+        ptestcl = f"planemo test --job_output_files {imtooldir}  --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+        try:
+            container = client.containers.run(planemoimage,ptestcl,
+               volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
+        except:
+            pass
+        tout.write(f"## Ran {ptestcl}")
+        if os.path.isfile(imrep_path):
+            shutil.copyfile(imrep_path,ptestrep_path)
+        else:
+            tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found")
+        tout.close()
+        #shutil.rmtree(workdir)
+
+
+
+    def gal_tool_test(self):
+        """
+        On path should be a handy script writes test outputs even if they don't exist
+
+        galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose
+
+       leaves outputs in -o !
+        """
+        gttscript = f"""#!{self.args.galaxy_venv}/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from galaxy.tool_util.verify.script import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())
+
+        """
+        galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
+        fakeenv = copy.copy(os.environ)
+        fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+        fakeenv ["PYTHONPATH"] = f"{galaxy_lib}"
+        if os.path.exists(self.tlog):
+            tout = open(self.tlog, "a")
+        else:
+            tout = open(self.tlog, "w")
+        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
+        tout.write(f"#### using {testouts} as tempdir\n")
         dummy, tfile = tempfile.mkstemp()
+        gtt = 'galaxy-tool-test'
+        gttf = open(gtt,'w')
+        gttf.write(gttscript)
+        gttf.write('\n')
+        gttf.close()
+        os.chmod(gtt,0o744)
         cll = [
-            os.path.join(self.args.tool_dir,"galaxy-tool-test"),
+            os.path.abspath(gtt),
             "-u",
             self.args.galaxy_url,
             "-k",
             self.args.galaxy_api_key,
             "-t",
-            self.args.tool_name,
+            self.tool_name,
             "-o",
             testouts,
         ]
         subp = subprocess.run(
-           cll, shell=False, stderr=dummy, stdout=dummy
+           cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout
         )
         outfiles = []
         for p in self.outfiles:
@@ -712,13 +903,13 @@
                 dest = os.path.join(self.tooloutdir, entry.name)
                 src = os.path.join(testouts, entry.name)
                 shutil.copyfile(src, dest)
-                dest = os.path.join(self.testdir, entry.name)
+                testdest = os.path.join(self.testdir, entry.name)
                 src = os.path.join(testouts, entry.name)
-                shutil.copyfile(src, dest)
+                shutil.copyfile(src, testdest)
                 dest = os.path.join(self.repdir,f"{entry.name}_sample")
-                tout.write(f"## found and moved output {entry.name} to {dest}\n")
+                tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n")
         tout.close()
-        shutil.rmtree(testouts)
+        #shutil.rmtree(testouts)
         return subp.returncode
 
     def gal_test(self):
@@ -729,25 +920,26 @@
         && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox
 
         """
-        testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp")
+        testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
         tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html"
         if os.path.exists(self.tlog):
             tout = open(self.tlog, "a")
         else:
             tout = open(self.tlog, "w")
 
-        ourenv = os.environ
-        ourenv["GALAXY_TEST_SAVE"] = testdir
-        ourenv["GALAXY_TEST_NO_CLEANUP"] = "1"
-        ourenv["GALAXY_TEST_TMP_DIR"] = testdir
+        fakeenv  = copy.copy(os.environ)
+        fakeenv["GALAXY_TEST_SAVE"] = testdir
+        fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1"
+        fakeenv["GALAXY_TEST_TMP_DIR"] = testdir
+        galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
 
         cll = [
-       "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.args.tool_name,
+       "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name,
        "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox",
         ]
         subp = subprocess.run(
-            cll, env = ourenv,
-            shell=False, cwd=self.args.galaxy_root, stderr=tout, stdout=tout
+            cll, env = fakeenv ,
+            shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout
         )
         src = os.path.join(testdir, tool_test_rep)
         if os.path.isfile(src):
@@ -784,7 +976,7 @@
         sto.write(f"############names={rnames} rids={rids}\n")
         sto.write(f"############names={repos}\n")
         tfcat = "ToolFactory generated tools"
-        if self.args.tool_name not in rnames:
+        if self.tool_name not in rnames:
             tscat = ts.categories.get_categories()
             cnames = [x.get("name", "?").strip() for x in tscat]
             cids = [x.get("id", "?") for x in tscat]
@@ -804,7 +996,7 @@
             tid = res.get("id", None)
             sto.write(f"##########create res={res}\n")
         else:
-            i = rnames.index(self.args.tool_name)
+            i = rnames.index(self.tool_name)
             tid = rids[i]
         try:
             res = ts.repositories.update_repository(
@@ -829,7 +1021,7 @@
             "-a",
             self.args.galaxy_api_key,
             "--name",
-            self.args.tool_name,
+            self.tool_name,
             "--owner",
             "fubar",
             "--toolshed",
@@ -838,9 +1030,9 @@
             "ToolFactory",
         ]
         tout.write("running\n%s\n" % " ".join(cll))
-        subp = subprocess.run(cll, shell=False, stderr=tout, stdout=tout)
+        subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)
         tout.write(
-            "installed %s - got retcode %d\n" % (self.args.tool_name, subp.returncode)
+            "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
         )
         tout.close()
         return subp.returncode
@@ -870,7 +1062,7 @@
         rnames = [x.get("name", "?") for x in repos]
         rids = [x.get("id", "?") for x in repos]
         #cat = "ToolFactory generated tools"
-        if self.args.tool_name not in rnames:
+        if self.tool_name not in rnames:
             cll = [
                 "planemo",
                 "shed_create",
@@ -879,20 +1071,20 @@
                 "--owner",
                 "fubar",
                 "--name",
-                self.args.tool_name,
+                self.tool_name,
                 "--shed_key",
                 self.args.toolshed_api_key,
             ]
             try:
                 subp = subprocess.run(
-                    cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
+                    cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
                 )
             except:
                 pass
             if subp.returncode != 0:
-                tout.write("Repository %s exists\n" % self.args.tool_name)
+                tout.write("Repository %s exists\n" % self.tool_name)
             else:
-                tout.write("initiated %s\n" % self.args.tool_name)
+                tout.write("initiated %s\n" % self.tool_name)
         cll = [
             "planemo",
             "shed_upload",
@@ -901,13 +1093,13 @@
             "--owner",
             "fubar",
             "--name",
-            self.args.tool_name,
+            self.tool_name,
             "--shed_key",
             self.args.toolshed_api_key,
             "--tar",
             self.newtarpath,
         ]
-        subp = subprocess.run(cll, shell=False, stdout=tout, stderr=tout)
+        subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout)
         tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode))
         tout.close()
         return subp.returncode
@@ -927,14 +1119,14 @@
             "-a",
             self.args.galaxy_api_key,
             "--name",
-            self.args.tool_name,
+            self.tool_name,
             "--owner",
             "fubar",
         ]
         if genoutputs:
             dummy, tfile = tempfile.mkstemp()
             subp = subprocess.run(
-               cll, shell=False, stderr=dummy, stdout=dummy
+               cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=dummy, stdout=dummy
             )
 
             with open('tool_test_output.json','rb') as f:
@@ -964,7 +1156,7 @@
                     shutil.copyfile(src, dest)
         else:
             subp = subprocess.run(
-               cll, shell=False,  stderr=tout, stdout=tout)
+               cll, env=self.ourenv, cwd=self.ourcwd, shell=False,  stderr=tout, stdout=tout)
             tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))
         tout.close()
         return subp.returncode
@@ -1007,6 +1199,7 @@
             ]
             subp = subprocess.run(
                 cll,
+                env=self.ourenv,
                 shell=False,
                 cwd=self.tooloutdir,
                 stderr=dummy,
@@ -1026,7 +1219,7 @@
                 xreal,
             ]
             subp = subprocess.run(
-                cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
+                cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
             )
         tout.close()
         return subp.returncode
@@ -1068,6 +1261,7 @@
             ]
             subp = subprocess.run(
                 cll,
+                env=self.ourenv,
                 shell=False,
                 cwd=self.testdir,
                 stderr=dummy,
@@ -1085,7 +1279,7 @@
                 xreal,
             ]
             subp = subprocess.run(
-                cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
+                cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
             )
         tout.close()
         return subp.returncode
@@ -1124,13 +1318,13 @@
 
     def makeToolTar(self):
         """move outputs into test-data and prepare the tarball"""
-        excludeme = "tool_test_output"
+        excludeme = "_planemo_test_report.html"
 
         def exclude_function(tarinfo):
             filename = tarinfo.name
             return (
                 None
-                if filename.startswith(excludeme)
+                if filename.endswith(excludeme)
                 else tarinfo
             )
 
@@ -1171,7 +1365,7 @@
                 shutil.copyfile(src, dest)
         with os.scandir(self.testdir) as outs:
             for entry in outs:
-                if not entry.is_file():
+                if (not entry.is_file()) or entry.name.endswith('_sample') or entry.name.endswith("_planemo_test_report.html"):
                     continue
                 if "." in entry.name:
                     nayme, ext = os.path.splitext(entry.name)
@@ -1223,6 +1417,7 @@
     a("--toolshed_api_key", default="fakekey")
     a("--galaxy_api_key", default="fakekey")
     a("--galaxy_root", default="/galaxy-central")
+    a("--galaxy_venv", default="/galaxy_venv")
     args = parser.parse_args()
     assert not args.bad_user, (
         'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file'
@@ -1240,23 +1435,16 @@
     r.writeShedyml()
     r.makeTool()
     if args.make_Tool == "generate":
-        retcode = r.run()
+        retcode = r.run() # for testing toolfactory itself
         r.moveRunOutputs()
         r.makeToolTar()
     else:
-        r.makeToolTar()
-        r.planemo_shedLoad()
-        r.shedLoad()
-        r.eph_galaxy_load()
-        retcode = r.gal_tool_test()  # writes outputs
-        r.makeToolTar()
-        r.planemo_shedLoad()
-        r.shedLoad()
-        r.eph_galaxy_load()
-        retcode = r.gal_test()
+        r.planemo_biodocker_test() # test to make outputs and then test
         r.moveRunOutputs()
         r.makeToolTar()
-        print(f"second galaxy_test returned {retcode}")
+        if args.make_Tool == "gentestinstall":
+            r.shedLoad()
+            r.eph_galaxy_load()
 
 
 if __name__ == "__main__":

--- a/toolfactory/rgToolFactory2.xml	Mon Nov 23 03:12:37 2020 +0000
+++ b/toolfactory/rgToolFactory2.xml	Thu Nov 26 06:17:12 2020 +0000
@@ -135,7 +135,8 @@
    <requirement type="package" version="0.4.11">galaxyxml</requirement>
    <requirement type="package" version="0.14.0">bioblend</requirement>
    <requirement type="package" version="0.10.6">ephemeris</requirement>
-   <requirement type="package" version="0.72.0">planemo</requirement>
+   <requirement type="package" version="4.4.0">docker-py</requirement>
+   <container type="docker">quay.io/fubar2/planemo-biocontainer:latest</container>
 </requirements>
 
   <command ><![CDATA[