Mercurial > repos > fubar > tool_factory_2

changeset 106:6de08f44e551 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Sat, 28 Nov 2020 00:50:29 +0000
parents 373e1f9cb609
children 18e616939a91
files toolfactory/rgToolFactory2.py
diffstat 1 files changed, 133 insertions(+), 92 deletions(-) [+]
line wrap: on
line diff
--- a/toolfactory/rgToolFactory2.py	Fri Nov 27 23:17:10 2020 +0000
+++ b/toolfactory/rgToolFactory2.py	Sat Nov 28 00:50:29 2020 +0000
@@ -557,8 +557,10 @@
                 scrpt.insert(0, "```\n")
                 if len(scrpt) > 300:
                     safertext = (
-                        safertext + scrpt[:100] + \
-                        [">500 lines - stuff deleted", "......"] + scrpt[-100:]
+                        safertext
+                        + scrpt[:100]
+                        + [">500 lines - stuff deleted", "......"]
+                        + scrpt[-100:]
                     )
                 else:
                     safertext = safertext + scrpt
@@ -649,7 +651,9 @@
                     "## Executing Toolfactory generated command line = %s\n" % scl
                 )
             sto.flush()
-            subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste)
+            subp = subprocess.run(
+                self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste
+            )
             sto.close()
             ste.close()
             retval = subp.returncode
@@ -662,7 +666,9 @@
                 sto = open(self.outfiles[0][ONAMEPOS], "wb")
             else:
                 sto = sys.stdout
-            subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti)
+            subp = subprocess.run(
+                self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti
+            )
             sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
             retval = subp.returncode
             sto.close()
@@ -676,42 +682,35 @@
         logging.debug("run done")
         return retval
 
-
     def copy_to_container(self, src, dest, container):
-        """ Recreate the src directory tree at dest - full path included
-        """
+        """Recreate the src directory tree at dest - full path included"""
         idir = os.getcwd()
         workdir = os.path.dirname(src)
         os.chdir(workdir)
         _, tfname = tempfile.mkstemp(suffix=".tar")
-        tar = tarfile.open(tfname, mode='w')
+        tar = tarfile.open(tfname, mode="w")
         srcb = os.path.basename(src)
         tar.add(srcb)
         tar.close()
-        data = open(tfname, 'rb').read()
+        data = open(tfname, "rb").read()
         container.put_archive(dest, data)
         os.unlink(tfname)
         os.chdir(idir)
 
-
     def copy_from_container(self, src, dest, container):
-        """ recreate the src directory tree at dest using docker sdk
-        """
-        os.makedirs(dest,exist_ok=True)
+        """recreate the src directory tree at dest using docker sdk"""
+        os.makedirs(dest, exist_ok=True)
         _, tfname = tempfile.mkstemp(suffix=".tar")
-        tf = open(tfname,'wb')
+        tf = open(tfname, "wb")
         bits, stat = container.get_archive(src)
         for chunk in bits:
             tf.write(chunk)
         tf.close()
-        tar = tarfile.open(tfname,'r')
+        tar = tarfile.open(tfname, "r")
         tar.extractall(dest)
         tar.close()
         os.unlink(tfname)
 
-
-
-
     def planemo_biodocker_test(self):
         """planemo currently leaks dependencies if used in the same container and gets unhappy after a
         first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
@@ -720,13 +719,14 @@
 
 
         """
-        def prun(container,tout,cl,user="biodocker"):
-            rlog = container.exec_run(cl,user=user)
-            slogl = str(rlog).split('\\n')
-            slog = '\n'.join(slogl)
+
+        def prun(container, tout, cl, user="biodocker"):
+            rlog = container.exec_run(cl, user=user)
+            slogl = str(rlog).split("\\n")
+            slog = "\n".join(slogl)
             tout.write(f"## got rlog {slog} from {cl}\n")
 
-        dgroup = grp.getgrnam('docker')[2]
+        dgroup = grp.getgrnam("docker")[2]
         if os.path.exists(self.tlog):
             tout = open(self.tlog, "a")
         else:
@@ -734,25 +734,30 @@
         planemoimage = "quay.io/fubar2/planemo-biocontainer"
         xreal = "%s.xml" % self.tool_name
         repname = f"{self.tool_name}_planemo_test_report.html"
-        ptestrep_path = os.path.join(self.repdir,repname)
+        ptestrep_path = os.path.join(self.repdir, repname)
         tool_name = self.tool_name
         client = docker.from_env()
         tvol = client.volumes.create()
         tvolname = tvol.name
         destdir = "/toolfactory/ptest"
-        imrep  = os.path.join(destdir,repname)
+        imrep = os.path.join(destdir, repname)
         # need to keep the container running so sleep a while - we stop and destroy it when we are done
-        container = client.containers.run(planemoimage,'sleep 30m', detach=True, user="biodocker",
-            network="host", volumes={f"{tvolname}": {'bind': '/toolfactory', 'mode': 'rw'}})
+        container = client.containers.run(
+            planemoimage,
+            "sleep 30m",
+            detach=True,
+            user="biodocker",
+            volumes={f"{tvolname}": {"bind": "/toolfactory", "mode": "rw"}},
+        )
         cl = f"groupmod -g {dgroup} docker"
         prun(container, tout, cl, user="root")
         cl = f"mkdir -p {destdir}"
         prun(container, tout, cl, user="root")
         cl = f"rm -rf {destdir}/*"
         prun(container, tout, cl, user="root")
-        ptestpath = os.path.join(destdir,'tfout',xreal)
-        self.copy_to_container(self.tooloutdir,destdir,container)
-        cl ='chmod -R a+rwx /toolfactory'
+        ptestpath = os.path.join(destdir, "tfout", xreal)
+        self.copy_to_container(self.tooloutdir, destdir, container)
+        cl = "chmod -R a+rwx /toolfactory"
         prun(container, tout, cl, user="root")
         rlog = container.exec_run(f"ls -la {destdir}")
         ptestcl = f"planemo test  --update_test_data  --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
@@ -764,24 +769,24 @@
         # fails - used to generate test outputs
         cl = f"planemo test  --test_output {imrep} --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
         try:
-            prun(container,tout,cl)
+            prun(container, tout, cl)
         except:
             pass
-        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".")
-        self.copy_from_container(destdir,testouts,container)
-        src = os.path.join(testouts,'ptest')
+        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp", dir=".")
+        self.copy_from_container(destdir, testouts, container)
+        src = os.path.join(testouts, "ptest")
         if os.path.isdir(src):
-            shutil.copytree(src, '.', dirs_exist_ok=True)
+            shutil.copytree(src, ".", dirs_exist_ok=True)
             src = repname
             if os.path.isfile(repname):
-                shutil.copyfile(src,ptestrep_path)
+                shutil.copyfile(src, ptestrep_path)
         else:
             tout.write(f"No output from run to shutil.copytree in {src}\n")
         tout.close()
         container.stop()
         container.remove()
         tvol.remove()
-        #shutil.rmtree(testouts)
+        # shutil.rmtree(testouts)
 
     def shedLoad(self):
         """
@@ -832,10 +837,13 @@
             tid = rids[i]
         try:
             res = ts.repositories.update_repository(
-                id=tid, tar_ball_path=self.newtarpath, commit_message=None)
+                id=tid, tar_ball_path=self.newtarpath, commit_message=None
+            )
             sto.write(f"#####update res={res}\n")
         except ConnectionError:
-            sto.write("Probably no change to repository - bioblend shed upload failed\n")
+            sto.write(
+                "Probably no change to repository - bioblend shed upload failed\n"
+            )
         sto.close()
 
     def eph_galaxy_load(self):
@@ -862,7 +870,9 @@
             "ToolFactory",
         ]
         tout.write("running\n%s\n" % " ".join(cll))
-        subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)
+        subp = subprocess.run(
+            cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout
+        )
         tout.write(
             "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
         )
@@ -893,7 +903,7 @@
         repos = ts.repositories.get_repositories()
         rnames = [x.get("name", "?") for x in repos]
         rids = [x.get("id", "?") for x in repos]
-        #cat = "ToolFactory generated tools"
+        # cat = "ToolFactory generated tools"
         if self.tool_name not in rnames:
             cll = [
                 "planemo",
@@ -909,7 +919,12 @@
             ]
             try:
                 subp = subprocess.run(
-                    cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
+                    cll,
+                    env=self.ourenv,
+                    shell=False,
+                    cwd=self.tooloutdir,
+                    stdout=tout,
+                    stderr=tout,
                 )
             except:
                 pass
@@ -931,14 +946,15 @@
             "--tar",
             self.newtarpath,
         ]
-        subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout)
-        tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode))
+        subp = subprocess.run(
+            cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout
+        )
+        tout.write("Ran %s got %d\n" % (" ".join(cll), subp.returncode))
         tout.close()
         return subp.returncode
 
     def eph_test(self, genoutputs=True):
-        """problem getting jobid - ephemeris upload is the job before the one we want - but depends on how many inputs
-        """
+        """problem getting jobid - ephemeris upload is the job before the one we want - but depends on how many inputs"""
         if os.path.exists(self.tlog):
             tout = open(self.tlog, "a")
         else:
@@ -958,27 +974,34 @@
         if genoutputs:
             dummy, tfile = tempfile.mkstemp()
             subp = subprocess.run(
-               cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=dummy, stdout=dummy
+                cll,
+                env=self.ourenv,
+                cwd=self.ourcwd,
+                shell=False,
+                stderr=dummy,
+                stdout=dummy,
             )
 
-            with open('tool_test_output.json','rb') as f:
+            with open("tool_test_output.json", "rb") as f:
                 s = json.loads(f.read())
-                print('read %s' % s)
-                cl = s['tests'][0]['data']['job']['command_line'].split()
-                n = cl.index('--script_path')
-                jobdir = cl[n+1]
-                jobdir = jobdir.replace('"','')
-                jobdir = jobdir.split('/configs')[0]
-                print('jobdir=%s' % jobdir)
+                print("read %s" % s)
+                cl = s["tests"][0]["data"]["job"]["command_line"].split()
+                n = cl.index("--script_path")
+                jobdir = cl[n + 1]
+                jobdir = jobdir.replace('"', "")
+                jobdir = jobdir.split("/configs")[0]
+                print("jobdir=%s" % jobdir)
 
-                #"/home/ross/galthrow/database/jobs_directory/000/649/configs/tmptfxu51gs\"
-            src = os.path.join(jobdir,'working',self.newtarpath)
+                # "/home/ross/galthrow/database/jobs_directory/000/649/configs/tmptfxu51gs\"
+            src = os.path.join(jobdir, "working", self.newtarpath)
             if os.path.exists(src):
                 dest = os.path.join(self.testdir, self.newtarpath)
                 shutil.copyfile(src, dest)
             else:
-                tout.write('No toolshed archive found after first ephemeris test - not a good sign')
-            ephouts = os.path.join(jobdir,'working','tfout','test-data')
+                tout.write(
+                    "No toolshed archive found after first ephemeris test - not a good sign"
+                )
+            ephouts = os.path.join(jobdir, "working", "tfout", "test-data")
             with os.scandir(ephouts) as outs:
                 for entry in outs:
                     if not entry.is_file():
@@ -988,29 +1011,37 @@
                     shutil.copyfile(src, dest)
         else:
             subp = subprocess.run(
-               cll, env=self.ourenv, cwd=self.ourcwd, shell=False,  stderr=tout, stdout=tout)
+                cll,
+                env=self.ourenv,
+                cwd=self.ourcwd,
+                shell=False,
+                stderr=tout,
+                stdout=tout,
+            )
             tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))
         tout.close()
         return subp.returncode
 
     def planemo_test_biocontainer(self, genoutputs=True):
         """planemo is a requirement so is available for testing but testing in a biocontainer
-        requires some fiddling to use the hacked galaxy-central .venv
+                requires some fiddling to use the hacked galaxy-central .venv
 
-        Planemo runs:
-python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file
-/export/galaxy-central/database/job_working_directory/000/17/working/TF_run_report_tempdir/tacrev_planemo_test_report.html
---with-xunit --xunit-file /tmp/tmpt90p7f9h/xunit.xml --with-structureddata
---structured-data-file
-/export/galaxy-central/database/job_working_directory/000/17/working/tfout/tool_test_output.json functional.test_toolbox
+                Planemo runs:
+        python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file
+        /export/galaxy-central/database/job_working_directory/000/17/working/TF_run_report_tempdir/tacrev_planemo_test_report.html
+        --with-xunit --xunit-file /tmp/tmpt90p7f9h/xunit.xml --with-structureddata
+        --structured-data-file
+        /export/galaxy-central/database/job_working_directory/000/17/working/tfout/tool_test_output.json functional.test_toolbox
 
 
-        for the planemo-biocontainer,
-        planemo test --conda_dependency_resolution --skip_venv --galaxy_root /galthrow/ rgToolFactory2.xml
+                for the planemo-biocontainer,
+                planemo test --conda_dependency_resolution --skip_venv --galaxy_root /galthrow/ rgToolFactory2.xml
 
         """
         xreal = "%s.xml" % self.tool_name
-        tool_test_path = os.path.join(self.repdir,f"{self.tool_name}_planemo_test_report.html")
+        tool_test_path = os.path.join(
+            self.repdir, f"{self.tool_name}_planemo_test_report.html"
+        )
         if os.path.exists(self.tlog):
             tout = open(self.tlog, "a")
         else:
@@ -1018,11 +1049,15 @@
         if genoutputs:
             dummy, tfile = tempfile.mkstemp()
             cll = [
-                ".", os.path.join(self.args.galaxy_root,'.venv','bin','activate'),"&&",
+                ".",
+                os.path.join(self.args.galaxy_root, ".venv", "bin", "activate"),
+                "&&",
                 "planemo",
                 "test",
-                "--test_data", self.testdir,
-                "--test_output", tool_test_path,
+                "--test_data",
+                self.testdir,
+                "--test_output",
+                tool_test_path,
                 "--skip_venv",
                 "--galaxy_root",
                 self.args.galaxy_root,
@@ -1040,23 +1075,31 @@
 
         else:
             cll = [
-                ".", os.path.join(self.args.galaxy_root,'.venv','bin','activate'),"&&",
+                ".",
+                os.path.join(self.args.galaxy_root, ".venv", "bin", "activate"),
+                "&&",
                 "planemo",
                 "test",
-                "--test_data", os.path.self.testdir,
-                "--test_output", os.path.tool_test_path,
+                "--test_data",
+                os.path.self.testdir,
+                "--test_output",
+                os.path.tool_test_path,
                 "--skip_venv",
                 "--galaxy_root",
                 self.args.galaxy_root,
                 xreal,
             ]
             subp = subprocess.run(
-                cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
+                cll,
+                env=self.ourenv,
+                shell=False,
+                cwd=self.tooloutdir,
+                stderr=tout,
+                stdout=tout,
             )
         tout.close()
         return subp.returncode
 
-
     def writeShedyml(self):
         """for planemo"""
         yuser = self.args.user_email.split("@")[0]
@@ -1094,17 +1137,13 @@
 
         def exclude_function(tarinfo):
             filename = tarinfo.name
-            return (
-                None
-                if filename.endswith(excludeme)
-                else tarinfo
-            )
+            return None if filename.endswith(excludeme) else tarinfo
 
         for p in self.outfiles:
             oname = p[ONAMEPOS]
             tdest = os.path.join(self.testdir, "%s_sample" % oname)
             if not os.path.isfile(tdest):
-                src = os.path.join(self.testdir,oname)
+                src = os.path.join(self.testdir, oname)
                 if os.path.isfile(src):
                     shutil.copyfile(src, tdest)
                     dest = os.path.join(self.repdir, "%s.sample" % (oname))
@@ -1127,8 +1166,8 @@
                     continue
                 if "." in entry.name:
                     nayme, ext = os.path.splitext(entry.name)
-                    if ext in ['.yml','.xml','.json','.yaml']:
-                        ext = f'{ext}.txt'
+                    if ext in [".yml", ".xml", ".json", ".yaml"]:
+                        ext = f"{ext}.txt"
                 else:
                     ext = ".txt"
                 ofn = "%s%s" % (entry.name.replace(".", "_"), ext)
@@ -1137,7 +1176,11 @@
                 shutil.copyfile(src, dest)
         with os.scandir(self.testdir) as outs:
             for entry in outs:
-                if (not entry.is_file()) or entry.name.endswith('_sample') or entry.name.endswith("_planemo_test_report.html"):
+                if (
+                    (not entry.is_file())
+                    or entry.name.endswith("_sample")
+                    or entry.name.endswith("_planemo_test_report.html")
+                ):
                     continue
                 if "." in entry.name:
                     nayme, ext = os.path.splitext(entry.name)
@@ -1149,7 +1192,6 @@
                 shutil.copyfile(src, dest)
 
 
-
 def main():
     """
     This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as:
@@ -1183,8 +1225,7 @@
     a("--tfout", default="./tfout")
     a("--new_tool", default="new_tool")
     a("--galaxy_url", default="http://localhost:8080")
-    a(
-        "--toolshed_url", default="http://localhost:9009")
+    a("--toolshed_url", default="http://localhost:9009")
     # make sure this is identical to tool_sheds_conf.xml  localhost != 127.0.0.1 so validation fails
     a("--toolshed_api_key", default="fakekey")
     a("--galaxy_api_key", default="fakekey")
@@ -1207,11 +1248,11 @@
     r.writeShedyml()
     r.makeTool()
     if args.make_Tool == "generate":
-        retcode = r.run() # for testing toolfactory itself
+        retcode = r.run()  # for testing toolfactory itself
         r.moveRunOutputs()
         r.makeToolTar()
     else:
-        r.planemo_biodocker_test() # test to make outputs and then test
+        r.planemo_biodocker_test()  # test to make outputs and then test
         r.moveRunOutputs()
         r.makeToolTar()
         if args.make_Tool == "gentestinstall":