comparison toolfactory/rgToolFactory2.py @ 102:c632db66f8c0 draft

Uploaded

101:557d5f06f213

        tout.close()
        container.stop()
        container.remove()
        shutil.rmtree(testouts)


    def planemo_biodocker_vol_test(self):
        """planemo currently leaks dependencies if used in the same container and gets unhappy after a
        first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930

        Docker biocontainer has planemo with caches filled to save repeated downloads
        Cannot get volumes to work right in this version

        """
        if os.path.exists(self.tlog):
            tout = open(self.tlog, "a")
        else:
            tout = open(self.tlog, "w")
        planemoimage = "quay.io/fubar2/planemo-biocontainer"
        xreal = "%s.xml" % self.tool_name
        repname = f"{self.tool_name}_planemo_test_report.html"
        ptestrep_path = os.path.join(self.repdir,repname)
        tool_name = self.tool_name
        workdir = "export"
        aworkdir = os.path.abspath(workdir)
        os.makedirs(workdir, exist_ok=True)
        os.chmod(workdir,0o777)
        imworkdir = "/export"
        # must be mounted as a volume
        tooldir = os.path.join(workdir,self.tool_name)
        testdir = os.path.join(tooldir,'test-data')
        imtooldir = os.path.join(imworkdir,self.tool_name)
        imtestdir = os.path.join(imtooldir,'test-data')
        for d in [tooldir,testdir]:
            if not os.path.exists(d):
                os.mkdir(d)
        with os.scandir(self.testdir) as outs:
            for entry in outs:
                if not entry.is_file():
                    continue
                src = os.path.join(self.testdir, entry.name)
                dest = os.path.join(testdir, entry.name)
                shutil.copyfile(src, dest)
        shutil.copyfile(xreal,os.path.join(tooldir,xreal))
        client = docker.from_env()
        # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],)
        atestcl = "ls -lt /export"
        container = client.containers.run(planemoimage,atestcl,
           volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
        tout.write(f"## Ran {atestcl} and got {container}")
        ptestpath = os.path.join(imtooldir,xreal)
        ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
        try:
            container = client.containers.run(planemoimage,ptestcl,
               volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
        except:
            pass
        tout.write(f"## Ran {ptestcl}")
        with os.scandir(testdir) as outs:
            for entry in outs:
                if not entry.is_file():
                    continue
                src = os.path.join(testdir, entry.name)
                dest = os.path.join(self.testdir, entry.name)
                shutil.copyfile(src, dest)
        imrep_path = os.path.join(imtooldir,repname)
        ptestcl = f"planemo test --job_output_files {imtooldir}  --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
        try:
            container = client.containers.run(planemoimage,ptestcl,
               volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
        except:
            pass
        tout.write(f"## Ran {ptestcl}")
        if os.path.isfile(imrep_path):
            shutil.copyfile(imrep_path,ptestrep_path)
        else:
            tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found")
        tout.close()
        #shutil.rmtree(workdir)



    def gal_tool_test(self):
        """
        On path should be a handy script writes test outputs even if they don't exist

        galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose

       leaves outputs in -o !
        """
        gttscript = f"""#!{self.args.galaxy_venv}/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from galaxy.tool_util.verify.script import main
if __name__ == '__main__':
    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
    sys.exit(main())

        """
        galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
        fakeenv = copy.copy(os.environ)
        fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        fakeenv ["PYTHONPATH"] = f"{galaxy_lib}"
        if os.path.exists(self.tlog):
            tout = open(self.tlog, "a")
        else:
            tout = open(self.tlog, "w")
        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
        tout.write(f"#### using {testouts} as tempdir\n")
        dummy, tfile = tempfile.mkstemp()
        gtt = 'galaxy-tool-test'
        gttf = open(gtt,'w')
        gttf.write(gttscript)
        gttf.write('\n')
        gttf.close()
        os.chmod(gtt,0o744)
        cll = [
            os.path.abspath(gtt),
            "-u",
            self.args.galaxy_url,
            "-k",
            self.args.galaxy_api_key,
            "-t",
            self.tool_name,
            "-o",
            testouts,
        ]
        subp = subprocess.run(
           cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout
        )
        outfiles = []
        for p in self.outfiles:
            oname = p[ONAMEPOS]
            outfiles.append(oname)
        with os.scandir(testouts) as outs:
            for entry in outs:
                if not entry.is_file():
                    continue
                dest = os.path.join(self.tooloutdir, entry.name)
                src = os.path.join(testouts, entry.name)
                shutil.copyfile(src, dest)
                testdest = os.path.join(self.testdir, entry.name)
                src = os.path.join(testouts, entry.name)
                shutil.copyfile(src, testdest)
                dest = os.path.join(self.repdir,f"{entry.name}_sample")
                tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n")
        tout.close()
        #shutil.rmtree(testouts)
        return subp.returncode

    def gal_test(self):
        """
        Uses the built in galaxy tool tester run_test.sh

        export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \
        && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox

        """
        testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
        tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html"
        if os.path.exists(self.tlog):
            tout = open(self.tlog, "a")
        else:
            tout = open(self.tlog, "w")

        fakeenv  = copy.copy(os.environ)
        fakeenv["GALAXY_TEST_SAVE"] = testdir
        fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1"
        fakeenv["GALAXY_TEST_TMP_DIR"] = testdir
        galaxy_lib = os.path.join(self.args.galaxy_root,'lib')

        cll = [
       "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name,
       "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox",
        ]
        subp = subprocess.run(
            cll, env = fakeenv ,
            shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout
        )
        src = os.path.join(testdir, tool_test_rep)
        if os.path.isfile(src):
            dest = os.path.join(self.repdir, tool_test_rep)
            shutil.copyfile(src, dest)
        else:
            tout.write(f"### {src} not found\n")
        tout.close()
        return subp.returncode


    def shedLoad(self):
        """
        {'deleted': False,
              'description': 'Tools for manipulating data',
              'id': '175812cd7caaf439',

                self.args.galaxy_root,
                xreal,
            ]
            subp = subprocess.run(
                cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
            )
        tout.close()
        return subp.returncode

    def planemo_test(self, genoutputs=True):
        """planemo is a requirement so is available for testing but needs a different call if
        in the biocontainer - see above
        and for generating test outputs if command or test overrides are supplied
        test outputs are sent to repdir for display
        planemo test --engine docker_galaxy  --galaxy_root /galaxy-central pyrevpos/pyrevpos.xml

        Planemo runs:
python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file
/export/galaxy-central/database/job_working_directory/000/17/working/TF_run_report_tempdir/tacrev_planemo_test_report.html
--with-xunit --xunit-file /tmp/tmpt90p7f9h/xunit.xml --with-structureddata
--structured-data-file
/export/galaxy-central/database/job_working_directory/000/17/working/tfout/tool_test_output.json functional.test_toolbox


        for the planemo-biocontainer,
        planemo test --conda_dependency_resolution --skip_venv --galaxy_root /galthrow/ rgToolFactory2.xml

        """
        xreal = "%s.xml" % self.tool_name
        tool_test_path = os.path.join(self.repdir,f"{self.tool_name}_planemo_test_report.html")
        if os.path.exists(self.tlog):
            tout = open(self.tlog, "a")
        else:
            tout = open(self.tlog, "w")
        if genoutputs:
            dummy, tfile = tempfile.mkstemp()
            cll = [
                "planemo",
                "test",
                "--galaxy_root",
                self.args.galaxy_root,
                "--update_test_data",
                xreal,
            ]
            subp = subprocess.run(
                cll,
                env=self.ourenv,
                shell=False,
                cwd=self.testdir,
                stderr=dummy,
                stdout=dummy,
            )

        else:
            cll = [
                "planemo",
                "test",
                "--test_data", self.testdir,
                "--test_output",tool_test_path,
                "--galaxy_root",
                self.args.galaxy_root,
                xreal,
            ]
            subp = subprocess.run(
                cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
            )
        tout.close()
        return subp.returncode

102:c632db66f8c0

        tout.close()
        container.stop()
        container.remove()
        shutil.rmtree(testouts)

    def shedLoad(self):
        """
        {'deleted': False,
              'description': 'Tools for manipulating data',
              'id': '175812cd7caaf439',

                self.args.galaxy_root,
                xreal,
            ]
            subp = subprocess.run(
                cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
            )
        tout.close()
        return subp.returncode