# HG changeset patch
# User fubar
# Date 1606371432 0
# Node ID 557d5f06f213df8e0d0a2d309b96281c69b40acb
# Parent c749364c2283efc9d533f10cc5da88ce372bba2a
Uploaded
diff -r c749364c2283 -r 557d5f06f213 toolfactory/rgToolFactory2.py
--- a/toolfactory/rgToolFactory2.py Mon Nov 23 03:12:37 2020 +0000
+++ b/toolfactory/rgToolFactory2.py Thu Nov 26 06:17:12 2020 +0000
@@ -26,6 +26,7 @@
# planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml
import argparse
+import copy
import datetime
import json
import logging
@@ -42,7 +43,7 @@
from bioblend import ConnectionError
from bioblend import toolshed
-# import docker
+import docker
import galaxyxml.tool as gxt
import galaxyxml.tool.parameters as gxtp
@@ -153,6 +154,8 @@
prepare command line cl for running the tool here
and prepare elements needed for galaxyxml tool generation
"""
+ self.ourcwd = os.getcwd()
+ self.ourenv = copy.deepcopy(os.environ)
self.infiles = [x.split(ourdelim) for x in args.input_files]
self.outfiles = [x.split(ourdelim) for x in args.output_files]
self.addpar = [x.split(ourdelim) for x in args.additional_parameters]
@@ -180,7 +183,7 @@
self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
self.tool_id = self.tool_name
self.newtool = gxt.Tool(
- self.args.tool_name,
+ self.tool_name,
self.tool_id,
self.args.tool_version,
self.args.tool_desc,
@@ -645,7 +648,7 @@
"## Executing Toolfactory generated command line = %s\n" % scl
)
sto.flush()
- subp = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)
+ subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste)
sto.close()
ste.close()
retval = subp.returncode
@@ -658,7 +661,7 @@
sto = open(self.outfiles[0][ONAMEPOS], "wb")
else:
sto = sys.stdout
- subp = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)
+ subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti)
sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
retval = subp.returncode
sto.close()
@@ -673,33 +676,221 @@
return retval
- def gal_tool_test(self):
+ def copy_to_container(self, src, dest, container):
+ """ Recreate the src directory tree at dest - full path included
+ """
+ idir = os.getcwd()
+ workdir = os.path.dirname(src)
+ os.chdir(workdir)
+ _, tfname = tempfile.mkstemp(suffix=".tar")
+ tar = tarfile.open(tfname, mode='w')
+ srcb = os.path.basename(src)
+ tar.add(srcb)
+ tar.close()
+ data = open(tfname, 'rb').read()
+ container.put_archive(dest, data)
+ os.unlink(tfname)
+ os.chdir(idir)
+
+
+ def copy_from_container(self, src, dest, container):
+ """ recreate the src directory tree at dest using docker sdk
+ """
+ os.makedirs(dest,exist_ok=True)
+ _, tfname = tempfile.mkstemp(suffix=".tar")
+ tf = open(tfname,'wb')
+ bits, stat = container.get_archive(src)
+ for chunk in bits:
+ tf.write(chunk)
+ tf.close()
+ tar = tarfile.open(tfname,'r')
+ tar.extractall(dest)
+ tar.close()
+ os.unlink(tfname)
+
+
+
+ def planemo_biodocker_test(self):
+ """planemo currently leaks dependencies if used in the same container and gets unhappy after a
+ first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
+
+ Docker biocontainer has planemo with caches filled to save repeated downloads
+
+
"""
- This handy script writes test outputs even if they don't exist
- galaxy-tool-test [-h] [-u GALAXY_URL] [-k KEY] [-a ADMIN_KEY] [--force_path_paste] [-t TOOL_ID] [--tool-version TOOL_VERSION]
- [-i TEST_INDEX] [-o OUTPUT] [--append] [-j OUTPUT_JSON] [--verbose] [-c CLIENT_TEST_CONFIG]
- galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose
- handy - just leaves outputs in -o
+ if os.path.exists(self.tlog):
+ tout = open(self.tlog, "a")
+ else:
+ tout = open(self.tlog, "w")
+ planemoimage = "quay.io/fubar2/planemo-biocontainer"
+ xreal = "%s.xml" % self.tool_name
+ destdir = "/tmp/tfout"
+ repname = f"{self.tool_name}_planemo_test_report.html"
+ imrep = os.path.join(destdir,repname)
+ ptestrep_path = os.path.join(self.repdir,repname)
+ tool_name = self.tool_name
+ client = docker.from_env()
+ container = client.containers.run(planemoimage,'sleep 10000m', detach=True)
+ rlog = container.exec_run(f"mkdir -p {destdir}")
+ slogl = str(rlog).split('\\n')
+ slog = '\n'.join(slogl)
+ tout.write(f"## got rlog {slog} from mkdir {destdir}")
+ ptestpath = os.path.join(destdir,xreal)
+ self.copy_to_container(self.tooloutdir,'/tmp',container)
+ rlog = container.exec_run(f"ls -la {destdir}")
+ ptestcl = f"planemo test --update_test_data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+ try:
+ rlog = container.exec_run(ptestcl)
+ except:
+ e = sys.exc_info()[0]
+ tout.write(f"#### error: {e} from {ptestcl}")
+ # fails - used to generate test outputs
+ ptestcl = f"planemo test --test_output {imrep} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+ try:
+ rlog = container.exec_run(ptestcl)
+ except:
+ pass
+ slogl = str(rlog).split('\\n')
+ slog = '\n'.join(slogl)
+ tout.write(f"## got rlog {slog} from mkdir {destdir}")
+ testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir=".")
+ self.copy_from_container(destdir,testouts,container)
+ try:
+ shutil.rmtree(os.path.join(testouts,self.tooloutdir,'test-data','test-data'))
+ except:
+ e = sys.exc_info()[0]
+ tout.write(f"#### error: {e} from {ptestcl}")
+ shutil.copytree(os.path.join(testouts,self.tooloutdir), self.tooloutdir, dirs_exist_ok=True)
+ tout.close()
+ container.stop()
+ container.remove()
+ shutil.rmtree(testouts)
+
+
+ def planemo_biodocker_vol_test(self):
+ """planemo currently leaks dependencies if used in the same container and gets unhappy after a
+ first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930
+
+ Docker biocontainer has planemo with caches filled to save repeated downloads
+ Cannot get volumes to work right in this version
+
"""
if os.path.exists(self.tlog):
tout = open(self.tlog, "a")
else:
tout = open(self.tlog, "w")
- testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp")
+ planemoimage = "quay.io/fubar2/planemo-biocontainer"
+ xreal = "%s.xml" % self.tool_name
+ repname = f"{self.tool_name}_planemo_test_report.html"
+ ptestrep_path = os.path.join(self.repdir,repname)
+ tool_name = self.tool_name
+ workdir = "export"
+ aworkdir = os.path.abspath(workdir)
+ os.makedirs(workdir, exist_ok=True)
+ os.chmod(workdir,0o777)
+ imworkdir = "/export"
+ # must be mounted as a volume
+ tooldir = os.path.join(workdir,self.tool_name)
+ testdir = os.path.join(tooldir,'test-data')
+ imtooldir = os.path.join(imworkdir,self.tool_name)
+ imtestdir = os.path.join(imtooldir,'test-data')
+ for d in [tooldir,testdir]:
+ if not os.path.exists(d):
+ os.mkdir(d)
+ with os.scandir(self.testdir) as outs:
+ for entry in outs:
+ if not entry.is_file():
+ continue
+ src = os.path.join(self.testdir, entry.name)
+ dest = os.path.join(testdir, entry.name)
+ shutil.copyfile(src, dest)
+ shutil.copyfile(xreal,os.path.join(tooldir,xreal))
+ client = docker.from_env()
+ # mnt = docker.types.Mount(source='workdir', target=imworkdir) # mounts=[mnt],)
+ atestcl = "ls -lt /export"
+ container = client.containers.run(planemoimage,atestcl,
+ volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
+ tout.write(f"## Ran {atestcl} and got {container}")
+ ptestpath = os.path.join(imtooldir,xreal)
+ ptestcll = f"planemo test --job_output_files {imtooldir} --update_test_data --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+ try:
+ container = client.containers.run(planemoimage,ptestcl,
+ volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
+ except:
+ pass
+ tout.write(f"## Ran {ptestcl}")
+ with os.scandir(testdir) as outs:
+ for entry in outs:
+ if not entry.is_file():
+ continue
+ src = os.path.join(testdir, entry.name)
+ dest = os.path.join(self.testdir, entry.name)
+ shutil.copyfile(src, dest)
+ imrep_path = os.path.join(imtooldir,repname)
+ ptestcl = f"planemo test --job_output_files {imtooldir} --test_output {imrep_path} --test_data {imtestdir} --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
+ try:
+ container = client.containers.run(planemoimage,ptestcl,
+ volumes={aworkdir:{'bind':'/export','mode':'rw'}}, )
+ except:
+ pass
+ tout.write(f"## Ran {ptestcl}")
+ if os.path.isfile(imrep_path):
+ shutil.copyfile(imrep_path,ptestrep_path)
+ else:
+ tout.write(f"## planemo_biodocker_test - no test report {imrep_path} found")
+ tout.close()
+ #shutil.rmtree(workdir)
+
+
+
+ def gal_tool_test(self):
+ """
+ On path should be a handy script writes test outputs even if they don't exist
+
+ galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hello -j /tmp/foo/hello.json --verbose
+
+ leaves outputs in -o !
+ """
+ gttscript = f"""#!{self.args.galaxy_venv}/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from galaxy.tool_util.verify.script import main
+if __name__ == '__main__':
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+ sys.exit(main())
+
+ """
+ galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
+ fakeenv = copy.copy(os.environ)
+ fakeenv ["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ fakeenv ["PYTHONPATH"] = f"{galaxy_lib}"
+ if os.path.exists(self.tlog):
+ tout = open(self.tlog, "a")
+ else:
+ tout = open(self.tlog, "w")
+ testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
+ tout.write(f"#### using {testouts} as tempdir\n")
dummy, tfile = tempfile.mkstemp()
+ gtt = 'galaxy-tool-test'
+ gttf = open(gtt,'w')
+ gttf.write(gttscript)
+ gttf.write('\n')
+ gttf.close()
+ os.chmod(gtt,0o744)
cll = [
- os.path.join(self.args.tool_dir,"galaxy-tool-test"),
+ os.path.abspath(gtt),
"-u",
self.args.galaxy_url,
"-k",
self.args.galaxy_api_key,
"-t",
- self.args.tool_name,
+ self.tool_name,
"-o",
testouts,
]
subp = subprocess.run(
- cll, shell=False, stderr=dummy, stdout=dummy
+ cll, env=fakeenv, cwd=galaxy_lib, shell=True, stderr=tout, stdout=tout
)
outfiles = []
for p in self.outfiles:
@@ -712,13 +903,13 @@
dest = os.path.join(self.tooloutdir, entry.name)
src = os.path.join(testouts, entry.name)
shutil.copyfile(src, dest)
- dest = os.path.join(self.testdir, entry.name)
+ testdest = os.path.join(self.testdir, entry.name)
src = os.path.join(testouts, entry.name)
- shutil.copyfile(src, dest)
+ shutil.copyfile(src, testdest)
dest = os.path.join(self.repdir,f"{entry.name}_sample")
- tout.write(f"## found and moved output {entry.name} to {dest}\n")
+ tout.write(f"## found and moved output {entry.name} to {dest} and {testdest}\n")
tout.close()
- shutil.rmtree(testouts)
+ #shutil.rmtree(testouts)
return subp.returncode
def gal_test(self):
@@ -729,25 +920,26 @@
&& export GALAXY_TEST_TMP_DIR=./foo && sh run_tests.sh --id rgtf2 --report_file tool_tests_tool_conf.html functional.test_toolbox
"""
- testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp")
+ testdir = tempfile.mkdtemp(suffix=None, prefix="tftemp",dir="/tmp")
tool_test_rep = f"{self.tool_name}_galaxy_test_report_html.html"
if os.path.exists(self.tlog):
tout = open(self.tlog, "a")
else:
tout = open(self.tlog, "w")
- ourenv = os.environ
- ourenv["GALAXY_TEST_SAVE"] = testdir
- ourenv["GALAXY_TEST_NO_CLEANUP"] = "1"
- ourenv["GALAXY_TEST_TMP_DIR"] = testdir
+ fakeenv = copy.copy(os.environ)
+ fakeenv["GALAXY_TEST_SAVE"] = testdir
+ fakeenv["GALAXY_TEST_NO_CLEANUP"] = "1"
+ fakeenv["GALAXY_TEST_TMP_DIR"] = testdir
+ galaxy_lib = os.path.join(self.args.galaxy_root,'lib')
cll = [
- "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.args.tool_name,
+ "sh", f"{self.args.galaxy_root}/run_tests.sh", "--id", self.tool_name,
"--report_file", os.path.join(testdir,tool_test_rep), "functional.test_toolbox",
]
subp = subprocess.run(
- cll, env = ourenv,
- shell=False, cwd=self.args.galaxy_root, stderr=tout, stdout=tout
+ cll, env = fakeenv ,
+ shell=False, cwd=galaxy_lib, stderr=tout, stdout=tout
)
src = os.path.join(testdir, tool_test_rep)
if os.path.isfile(src):
@@ -784,7 +976,7 @@
sto.write(f"############names={rnames} rids={rids}\n")
sto.write(f"############names={repos}\n")
tfcat = "ToolFactory generated tools"
- if self.args.tool_name not in rnames:
+ if self.tool_name not in rnames:
tscat = ts.categories.get_categories()
cnames = [x.get("name", "?").strip() for x in tscat]
cids = [x.get("id", "?") for x in tscat]
@@ -804,7 +996,7 @@
tid = res.get("id", None)
sto.write(f"##########create res={res}\n")
else:
- i = rnames.index(self.args.tool_name)
+ i = rnames.index(self.tool_name)
tid = rids[i]
try:
res = ts.repositories.update_repository(
@@ -829,7 +1021,7 @@
"-a",
self.args.galaxy_api_key,
"--name",
- self.args.tool_name,
+ self.tool_name,
"--owner",
"fubar",
"--toolshed",
@@ -838,9 +1030,9 @@
"ToolFactory",
]
tout.write("running\n%s\n" % " ".join(cll))
- subp = subprocess.run(cll, shell=False, stderr=tout, stdout=tout)
+ subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)
tout.write(
- "installed %s - got retcode %d\n" % (self.args.tool_name, subp.returncode)
+ "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
)
tout.close()
return subp.returncode
@@ -870,7 +1062,7 @@
rnames = [x.get("name", "?") for x in repos]
rids = [x.get("id", "?") for x in repos]
#cat = "ToolFactory generated tools"
- if self.args.tool_name not in rnames:
+ if self.tool_name not in rnames:
cll = [
"planemo",
"shed_create",
@@ -879,20 +1071,20 @@
"--owner",
"fubar",
"--name",
- self.args.tool_name,
+ self.tool_name,
"--shed_key",
self.args.toolshed_api_key,
]
try:
subp = subprocess.run(
- cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
+ cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stdout=tout, stderr=tout
)
except:
pass
if subp.returncode != 0:
- tout.write("Repository %s exists\n" % self.args.tool_name)
+ tout.write("Repository %s exists\n" % self.tool_name)
else:
- tout.write("initiated %s\n" % self.args.tool_name)
+ tout.write("initiated %s\n" % self.tool_name)
cll = [
"planemo",
"shed_upload",
@@ -901,13 +1093,13 @@
"--owner",
"fubar",
"--name",
- self.args.tool_name,
+ self.tool_name,
"--shed_key",
self.args.toolshed_api_key,
"--tar",
self.newtarpath,
]
- subp = subprocess.run(cll, shell=False, stdout=tout, stderr=tout)
+ subp = subprocess.run(cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stdout=tout, stderr=tout)
tout.write("Ran %s got %d\n" % (" ".join(cll),subp.returncode))
tout.close()
return subp.returncode
@@ -927,14 +1119,14 @@
"-a",
self.args.galaxy_api_key,
"--name",
- self.args.tool_name,
+ self.tool_name,
"--owner",
"fubar",
]
if genoutputs:
dummy, tfile = tempfile.mkstemp()
subp = subprocess.run(
- cll, shell=False, stderr=dummy, stdout=dummy
+ cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=dummy, stdout=dummy
)
with open('tool_test_output.json','rb') as f:
@@ -964,7 +1156,7 @@
shutil.copyfile(src, dest)
else:
subp = subprocess.run(
- cll, shell=False, stderr=tout, stdout=tout)
+ cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)
tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))
tout.close()
return subp.returncode
@@ -1007,6 +1199,7 @@
]
subp = subprocess.run(
cll,
+ env=self.ourenv,
shell=False,
cwd=self.tooloutdir,
stderr=dummy,
@@ -1026,7 +1219,7 @@
xreal,
]
subp = subprocess.run(
- cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
+ cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout
)
tout.close()
return subp.returncode
@@ -1068,6 +1261,7 @@
]
subp = subprocess.run(
cll,
+ env=self.ourenv,
shell=False,
cwd=self.testdir,
stderr=dummy,
@@ -1085,7 +1279,7 @@
xreal,
]
subp = subprocess.run(
- cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
+ cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout
)
tout.close()
return subp.returncode
@@ -1124,13 +1318,13 @@
def makeToolTar(self):
"""move outputs into test-data and prepare the tarball"""
- excludeme = "tool_test_output"
+ excludeme = "_planemo_test_report.html"
def exclude_function(tarinfo):
filename = tarinfo.name
return (
None
- if filename.startswith(excludeme)
+ if filename.endswith(excludeme)
else tarinfo
)
@@ -1171,7 +1365,7 @@
shutil.copyfile(src, dest)
with os.scandir(self.testdir) as outs:
for entry in outs:
- if not entry.is_file():
+ if (not entry.is_file()) or entry.name.endswith('_sample') or entry.name.endswith("_planemo_test_report.html"):
continue
if "." in entry.name:
nayme, ext = os.path.splitext(entry.name)
@@ -1223,6 +1417,7 @@
a("--toolshed_api_key", default="fakekey")
a("--galaxy_api_key", default="fakekey")
a("--galaxy_root", default="/galaxy-central")
+ a("--galaxy_venv", default="/galaxy_venv")
args = parser.parse_args()
assert not args.bad_user, (
'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file'
@@ -1240,23 +1435,16 @@
r.writeShedyml()
r.makeTool()
if args.make_Tool == "generate":
- retcode = r.run()
+ retcode = r.run() # for testing toolfactory itself
r.moveRunOutputs()
r.makeToolTar()
else:
- r.makeToolTar()
- r.planemo_shedLoad()
- r.shedLoad()
- r.eph_galaxy_load()
- retcode = r.gal_tool_test() # writes outputs
- r.makeToolTar()
- r.planemo_shedLoad()
- r.shedLoad()
- r.eph_galaxy_load()
- retcode = r.gal_test()
+ r.planemo_biodocker_test() # test to make outputs and then test
r.moveRunOutputs()
r.makeToolTar()
- print(f"second galaxy_test returned {retcode}")
+ if args.make_Tool == "gentestinstall":
+ r.shedLoad()
+ r.eph_galaxy_load()
if __name__ == "__main__":
diff -r c749364c2283 -r 557d5f06f213 toolfactory/rgToolFactory2.xml
--- a/toolfactory/rgToolFactory2.xml Mon Nov 23 03:12:37 2020 +0000
+++ b/toolfactory/rgToolFactory2.xml Thu Nov 26 06:17:12 2020 +0000
@@ -135,7 +135,8 @@
galaxyxml
bioblend
ephemeris
- planemo
+ docker-py
+ quay.io/fubar2/planemo-biocontainer:latest