# HG changeset patch # User fubar # Date 1606371988 0 # Node ID c632db66f8c0e026160c0b7f167d9c7d38b7d1cf # Parent 557d5f06f213df8e0d0a2d309b96281c69b40acb Uploaded diff -r 557d5f06f213 -r c632db66f8c0 toolfactory/rgToolFactory2.py --- a/toolfactory/rgToolFactory2.py Thu Nov 26 06:17:12 2020 +0000 +++ b/toolfactory/rgToolFactory2.py Thu Nov 26 06:26:28 2020 +0000 @@ -766,191 +766,6 @@ container.remove() shutil.rmtree(testouts) - - def planemo_biodocker_vol_test(self): - """planemo currently leaks dependencies if used in the same container and gets unhappy after a - first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930 - - Docker biocontainer has planemo with caches filled to save repeated downloads - Cannot get volumes to work right in this version - - """ - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - planemoimage = "quay.io/fubar2/planemo-biocontainer" - xreal = "%s.xml" % self.tool_name - repname = f"{self.tool_name}_planemo_test_report.html" - ptestrep_path = os.path.join(self.repdir,repname) - tool_name = self.tool_name - workdir = "export" - aworkdir = os.path.abspath(workdir) - os.makedirs(workdir, exist_ok=True) - os.chmod(workdir,0o777) - imworkdir = "/export" - # must be mounted as a volume - tooldir = os.path.join(workdir,self.tool_name) - testdir = os.path.join(tooldir,'test-data') - imtooldir = os.path.join(imworkdir,self.tool_name) - imtestdir = os.path.join(imtooldir,'test-data') - for d in [tooldir,testdir]: - if not os.path.exists(d): - os.mkdir(d) - with os.scandir(self.testdir) as outs: - for entry in outs: - if not entry.is_file(): - continue - src = os.path.join(self.testdir, entry.name) - dest = os.path.join(testdir, entry.name) - shutil.copyfile(src, dest) - shutil.copyfile(xreal,os.path.join(tooldir,xreal)) - client = docker.from_env() - # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],) - atestcl = "ls -lt /export" - container = client.containers.run(planemoimage,atestcl, - volumes={aworkdir:{'bind':'/export','mode':'rw'}}, ) - tout.write(f"## Ran {atestcl} and got {container}") - ptestpath = os.path.join(imtooldir,xreal) - ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}" - try: - container = client.containers.run(planemoimage,ptestcl, - volumes={aworkdir:{'bind':'/export','mode':'rw'}}, ) - except: - pass - tout.write(f"## Ran {ptestcl}") - with os.scandir(testdir) as outs: - for entry in outs: - if not entry.is_file(): - continue - src = os.path.join(testdir, entry.name) - dest = os.path.join(self.testdir, entry.name) - shutil.copyfile(src, dest) - imrep_path = os.path.join(imtooldir,repname) - ptestcl = f"planemo test --job_output_files {imtooldir} --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}" - try: - container = client.containers.run(planemoimage,ptestcl, - volumes={aworkdir:{'bind':'/export','mode':'rw'}}, ) - except: - pass - tout.write(f"## Ran {ptestcl}") - if os.path.isfile(imrep_path): - shutil.copyfile(imrep_path,ptestrep_path) - else: - tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found") - tout.close() - #shutil.rmtree(workdir) - - - - def gal_tool_test(self): - """ - On path should be a handy script writes test outputs even if they don't exist - - galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose - - leaves outputs in -o ! - """ - gttscript = f"""#!{self.args.galaxy_venv}/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from galaxy.tool_util.verify.script import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) - - """ - galaxy_lib = os.path.join(self.args.galaxy_root,'lib') - fakeenv = copy.copy(os.environ) - fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - fakeenv ["PYTHONPATH"] = f"{galaxy_lib}" - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp") - tout.write(f"#### using {testouts} as tempdir\n") - dummy, tfile = tempfile.mkstemp() - gtt = 'galaxy-tool-test' - gttf = open(gtt,'w') - gttf.write(gttscript) - gttf.write('\n') - gttf.close() - os.chmod(gtt,0o744) - cll = [ - os.path.abspath(gtt), - "-u", - self.args.galaxy_url, - "-k", - self.args.galaxy_api_key, - "-t", - self.tool_name, - "-o", - testouts, - ] - subp = subprocess.run( - cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout - ) - outfiles = [] - for p in self.outfiles: - oname = p[ONAMEPOS] - outfiles.append(oname) - with os.scandir(testouts) as outs: - for entry in outs: - if not entry.is_file(): - continue - dest = os.path.join(self.tooloutdir, entry.name) - src = os.path.join(testouts, entry.name) - shutil.copyfile(src, dest) - testdest = os.path.join(self.testdir, entry.name) - src = os.path.join(testouts, entry.name) - shutil.copyfile(src, testdest) - dest = os.path.join(self.repdir,f"{entry.name}_sample") - tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n") - tout.close() - #shutil.rmtree(testouts) - return subp.returncode - - def gal_test(self): - """ - Uses the built in galaxy tool tester run_test.sh - - export GALAXY_TEST_SAVE="./foo" && export GALAXY_TEST_NO_CLEANUP="1" \ - && export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox - - """ - testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp") - tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html" - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - - fakeenv = copy.copy(os.environ) - fakeenv["GALAXY_TEST_SAVE"] = testdir - fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1" - fakeenv["GALAXY_TEST_TMP_DIR"] = testdir - galaxy_lib = os.path.join(self.args.galaxy_root,'lib') - - cll = [ - "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name, - "--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox", - ] - subp = subprocess.run( - cll, env = fakeenv , - shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout - ) - src = os.path.join(testdir, tool_test_rep) - if os.path.isfile(src): - dest = os.path.join(self.repdir, tool_test_rep) - shutil.copyfile(src, dest) - else: - tout.write(f"### {src} not found\n") - tout.close() - return subp.returncode - - def shedLoad(self): """ {'deleted': False, @@ -1224,66 +1039,6 @@ tout.close() return subp.returncode - def planemo_test(self, genoutputs=True): - """planemo is a requirement so is available for testing but needs a different call if - in the biocontainer - see above - and for generating test outputs if command or test overrides are supplied - test outputs are sent to repdir for display - planemo test --engine docker_galaxy --galaxy_root /galaxy-central pyrevpos/pyrevpos.xml - - Planemo runs: -python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file -/export/galaxy-central/database/job_working_directory/000/17/working/TF_run_report_tempdir/tacrev_planemo_test_report.html ---with-xunit --xunit-file /tmp/tmpt90p7f9h/xunit.xml --with-structureddata ---structured-data-file -/export/galaxy-central/database/job_working_directory/000/17/working/tfout/tool_test_output.json functional.test_toolbox - - - for the planemo-biocontainer, - planemo test --conda_dependency_resolution --skip_venv --galaxy_root /galthrow/ rgToolFactory2.xml - - """ - xreal = "%s.xml" % self.tool_name - tool_test_path = os.path.join(self.repdir,f"{self.tool_name}_planemo_test_report.html") - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - if genoutputs: - dummy, tfile = tempfile.mkstemp() - cll = [ - "planemo", - "test", - "--galaxy_root", - self.args.galaxy_root, - "--update_test_data", - xreal, - ] - subp = subprocess.run( - cll, - env=self.ourenv, - shell=False, - cwd=self.testdir, - stderr=dummy, - stdout=dummy, - ) - - else: - cll = [ - "planemo", - "test", - "--test_data", self.testdir, - "--test_output",tool_test_path, - "--galaxy_root", - self.args.galaxy_root, - xreal, - ] - subp = subprocess.run( - cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout - ) - tout.close() - return subp.returncode - def writeShedyml(self): """for planemo"""