| Previous changeset 100:c749364c2283 (2020-11-23) Next changeset 102:c632db66f8c0 (2020-11-26) |
|
Commit message:
Uploaded |
|
modified:
toolfactory/rgToolFactory2.py toolfactory/rgToolFactory2.xml |
| b |
| diff -r c749364c2283 -r 557d5f06f213 toolfactory/rgToolFactory2.py --- a/toolfactory/rgToolFactory2.py Mon Nov 23 03:12:37 2020 +0000 +++ b/toolfactory/rgToolFactory2.py Thu Nov 26 06:17:12 2020 +0000 |
| [ |
| b'@@ -26,6 +26,7 @@\n # planemo test --engine docker_galaxy --test_data ./test-data/ --docker_extra_volume ./test-data rgToolFactory2.xml\n \n import argparse\n+import copy\n import datetime\n import json\n import logging\n@@ -42,7 +43,7 @@\n from bioblend import ConnectionError\n from bioblend import toolshed\n \n-# import docker\n+import docker\n \n import galaxyxml.tool as gxt\n import galaxyxml.tool.parameters as gxtp\n@@ -153,6 +154,8 @@\n prepare command line cl for running the tool here\n and prepare elements needed for galaxyxml tool generation\n """\n+ self.ourcwd = os.getcwd()\n+ self.ourenv = copy.deepcopy(os.environ)\n self.infiles = [x.split(ourdelim) for x in args.input_files]\n self.outfiles = [x.split(ourdelim) for x in args.output_files]\n self.addpar = [x.split(ourdelim) for x in args.additional_parameters]\n@@ -180,7 +183,7 @@\n self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)\n self.tool_id = self.tool_name\n self.newtool = gxt.Tool(\n- self.args.tool_name,\n+ self.tool_name,\n self.tool_id,\n self.args.tool_version,\n self.args.tool_desc,\n@@ -645,7 +648,7 @@\n "## Executing Toolfactory generated command line = %s\\n" % scl\n )\n sto.flush()\n- subp = subprocess.run(self.cl, shell=False, stdout=sto, stderr=ste)\n+ subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste)\n sto.close()\n ste.close()\n retval = subp.returncode\n@@ -658,7 +661,7 @@\n sto = open(self.outfiles[0][ONAMEPOS], "wb")\n else:\n sto = sys.stdout\n- subp = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti)\n+ subp = subprocess.run(self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti)\n sto.write("## Executing Toolfactory generated command line = %s\\n" % scl)\n retval = subp.returncode\n sto.close()\n@@ -673,33 +676,221 @@\n return retval\n \n \n- def gal_tool_test(self):\n+ def copy_to_container(self, src, dest, container):\n+ """ Recreate the src directory tree at dest - full path included\n+ """\n+ idir = os.getcwd()\n+ workdir = os.path.dirname(src)\n+ os.chdir(workdir)\n+ _, tfname = tempfile.mkstemp(suffix=".tar")\n+ tar = tarfile.open(tfname, mode=\'w\')\n+ srcb = os.path.basename(src)\n+ tar.add(srcb)\n+ tar.close()\n+ data = open(tfname, \'rb\').read()\n+ container.put_archive(dest, data)\n+ os.unlink(tfname)\n+ os.chdir(idir)\n+\n+\n+ def copy_from_container(self, src, dest, container):\n+ """ recreate the src directory tree at dest using docker sdk\n+ """\n+ os.makedirs(dest,exist_ok=True)\n+ _, tfname = tempfile.mkstemp(suffix=".tar")\n+ tf = open(tfname,\'wb\')\n+ bits, stat = container.get_archive(src)\n+ for chunk in bits:\n+ tf.write(chunk)\n+ tf.close()\n+ tar = tarfile.open(tfname,\'r\')\n+ tar.extractall(dest)\n+ tar.close()\n+ os.unlink(tfname)\n+\n+\n+\n+ def planemo_biodocker_test(self):\n+ """planemo currently leaks dependencies if used in the same container and gets unhappy after a\n+ first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930\n+\n+ Docker biocontainer has planemo with caches filled to save repeated downloads\n+\n+\n """\n- This handy script writes test outputs even if they don\'t exist\n- galaxy-tool-test [-h] [-u GALAXY_URL] [-k KEY] [-a ADMIN_KEY] [--force_path_paste] [-t TOOL_ID] [--tool-version TOOL_VERSION]\n- [-i TEST_INDEX] [-o OUTPUT] [--append] [-j OUTPUT_JSON] [--verbose] [-c CLIENT_TEST_CONFIG]\n- galaxy-tool-test -u http://localhost:8080 -a 3c9afe09f1b7892449d266109639c104 -o /tmp/foo -t hel'..b' cll, shell=False, stderr=dummy, stdout=dummy\n+ cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=dummy, stdout=dummy\n )\n \n with open(\'tool_test_output.json\',\'rb\') as f:\n@@ -964,7 +1156,7 @@\n shutil.copyfile(src, dest)\n else:\n subp = subprocess.run(\n- cll, shell=False, stderr=tout, stdout=tout)\n+ cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout)\n tout.write("eph_test Ran %s got %d" % (" ".join(cll), subp.returncode))\n tout.close()\n return subp.returncode\n@@ -1007,6 +1199,7 @@\n ]\n subp = subprocess.run(\n cll,\n+ env=self.ourenv,\n shell=False,\n cwd=self.tooloutdir,\n stderr=dummy,\n@@ -1026,7 +1219,7 @@\n xreal,\n ]\n subp = subprocess.run(\n- cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout\n+ cll, env=self.ourenv, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout\n )\n tout.close()\n return subp.returncode\n@@ -1068,6 +1261,7 @@\n ]\n subp = subprocess.run(\n cll,\n+ env=self.ourenv,\n shell=False,\n cwd=self.testdir,\n stderr=dummy,\n@@ -1085,7 +1279,7 @@\n xreal,\n ]\n subp = subprocess.run(\n- cll, shell=False, cwd=self.testdir, stderr=tout, stdout=tout\n+ cll, env=self.ourenv, shell=False, cwd=self.testdir, stderr=tout, stdout=tout\n )\n tout.close()\n return subp.returncode\n@@ -1124,13 +1318,13 @@\n \n def makeToolTar(self):\n """move outputs into test-data and prepare the tarball"""\n- excludeme = "tool_test_output"\n+ excludeme = "_planemo_test_report.html"\n \n def exclude_function(tarinfo):\n filename = tarinfo.name\n return (\n None\n- if filename.startswith(excludeme)\n+ if filename.endswith(excludeme)\n else tarinfo\n )\n \n@@ -1171,7 +1365,7 @@\n shutil.copyfile(src, dest)\n with os.scandir(self.testdir) as outs:\n for entry in outs:\n- if not entry.is_file():\n+ if (not entry.is_file()) or entry.name.endswith(\'_sample\') or entry.name.endswith("_planemo_test_report.html"):\n continue\n if "." in entry.name:\n nayme, ext = os.path.splitext(entry.name)\n@@ -1223,6 +1417,7 @@\n a("--toolshed_api_key", default="fakekey")\n a("--galaxy_api_key", default="fakekey")\n a("--galaxy_root", default="/galaxy-central")\n+ a("--galaxy_venv", default="/galaxy_venv")\n args = parser.parse_args()\n assert not args.bad_user, (\n \'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file\'\n@@ -1240,23 +1435,16 @@\n r.writeShedyml()\n r.makeTool()\n if args.make_Tool == "generate":\n- retcode = r.run()\n+ retcode = r.run() # for testing toolfactory itself\n r.moveRunOutputs()\n r.makeToolTar()\n else:\n- r.makeToolTar()\n- r.planemo_shedLoad()\n- r.shedLoad()\n- r.eph_galaxy_load()\n- retcode = r.gal_tool_test() # writes outputs\n- r.makeToolTar()\n- r.planemo_shedLoad()\n- r.shedLoad()\n- r.eph_galaxy_load()\n- retcode = r.gal_test()\n+ r.planemo_biodocker_test() # test to make outputs and then test\n r.moveRunOutputs()\n r.makeToolTar()\n- print(f"second galaxy_test returned {retcode}")\n+ if args.make_Tool == "gentestinstall":\n+ r.shedLoad()\n+ r.eph_galaxy_load()\n \n \n if __name__ == "__main__":\n' |
| b |
| diff -r c749364c2283 -r 557d5f06f213 toolfactory/rgToolFactory2.xml --- a/toolfactory/rgToolFactory2.xml Mon Nov 23 03:12:37 2020 +0000 +++ b/toolfactory/rgToolFactory2.xml Thu Nov 26 06:17:12 2020 +0000 |
| [ |
| @@ -135,7 +135,8 @@ <requirement type="package" version="0.4.11">galaxyxml</requirement> <requirement type="package" version="0.14.0">bioblend</requirement> <requirement type="package" version="0.10.6">ephemeris</requirement> - <requirement type="package" version="0.72.0">planemo</requirement> + <requirement type="package" version="4.4.0">docker-py</requirement> + <container type="docker">quay.io/fubar2/planemo-biocontainer:latest</container> </requirements> <command ><![CDATA[ |