| Previous changeset 1:315a7e9ed6eb (2016-07-09) |
|
Commit message:
planemo upload for repository https://github.com/mvdbeek/docker_scriptrunner/ commit dded837d19aeb3f06b84e5076282cedeeaf713fa |
|
modified:
README.md dockerfiles/build.sh dockerfiles/scriptrunner/Dockerfile planemo_serve_script.sh scriptrunner.py scriptrunner.xml test-data/tf2_test.html test.sh |
|
removed:
tool_dependencies.xml |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 README.md --- a/README.md Sat Jul 09 17:00:06 2016 -0400 +++ b/README.md Sun Jul 22 13:38:01 2018 -0400 |
| [ |
| @@ -1,6 +1,35 @@ +[](https://travis-ci.org/ARTbio/docker-scriptrunner) + +You need to have docker Docker scriptrunner for galaxy ----------------------------- This is a [galaxy](https://github.com/galaxyproject/galaxy) tool that allows users to submit random scripts. -You can install it on your galaxy server from the [galaxy toolshed]() +You can install it on your galaxy server from the [galaxy toolshed] (https://toolshed.g2.bx.psu.edu/view/mvdbeek/docker_scriptrunner/). + + +This tool is heavily inspired by Ross Lazarus' tool factory +(https://www.ncbi.nlm.nih.gov/pubmed/23024011), but removes the ability to +create galaxy tools. (If you are looking at creating galaxy tools, you may want +to use [planemo](https://planemo.readthedocs.io/en/latest/)). + + +You need to have docker installed on any machine that can run +galaxy jobs, or route this tool to a dedicated docker host +in galaxy's job_conf.xml. Note that the tool itself talks +to the docker daemon and bypasses galaxy's docker configuration. +Making use of galaxy's docker capabilities is on the roadmap. + + +The tool comes with two docker images that can be easily extended. +"artbio/scriptrunner" is a base image, that has very few dependencies installed. +You can extend the image by following the example in the dockerfiles/r-bioperl-python +folder. + + +All security relies on docker; the container only mounts those files that the user +has selected as input files, and the script itself drops privileged inside +the container. This should be secure, but caution should be taken on public servers. +By default the container has networking enabled, so make sure your firewall rules +forbid traffic to the local network. |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 dockerfiles/build.sh --- a/dockerfiles/build.sh Sat Jul 09 17:00:06 2016 -0400 +++ b/dockerfiles/build.sh Sun Jul 22 13:38:01 2018 -0400 |
| b |
| @@ -1,3 +1,4 @@ #!/usr/bin/env bash +cd "${0%/*}" docker build -t artbio/scriptrunner scriptrunner && \ docker build -t artbio/scriptrunner-r-bioperl-python r-bioperl-python |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 dockerfiles/scriptrunner/Dockerfile --- a/dockerfiles/scriptrunner/Dockerfile Sat Jul 09 17:00:06 2016 -0400 +++ b/dockerfiles/scriptrunner/Dockerfile Sun Jul 22 13:38:01 2018 -0400 |
| b |
| @@ -26,8 +26,8 @@ RUN curl https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -L -o miniconda.sh RUN bash miniconda.sh -b -p "$CONDA_PREFIX" && rm miniconda.sh -RUN bash -c "conda create -y -q -c bioconda --name default samtools==0.1.19 bcftools && \ - conda config --add channels r && \ +RUN bash -c "conda create -y -q -c bioconda --name default samtools bcftools && \ + conda config --add channels conda-forge && \ conda config --add channels bioconda && \ conda config --add channels iuc && \ . activate default && \ |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 planemo_serve_script.sh --- a/planemo_serve_script.sh Sat Jul 09 17:00:06 2016 -0400 +++ b/planemo_serve_script.sh Sun Jul 22 13:38:01 2018 -0400 |
| b |
| @@ -1,1 +1,1 @@ -planemo serve --galaxy_branch dev --conda_prefix=/conda --conda_dependency_resolution --profile tf --port 80 --host 0.0.0.0 --job_config_file ~/mydisk/job_conf.xml --tool_data_table tool_data_table_conf.xml.sample.test +planemo serve --tool_data_table tool_data_table_conf.xml.sample.test "$@" |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 scriptrunner.py --- a/scriptrunner.py Sat Jul 09 17:00:06 2016 -0400 +++ b/scriptrunner.py Sun Jul 22 13:38:01 2018 -0400 |
| [ |
| b'@@ -1,213 +1,195 @@\n # DockerToolFactory.py\n # see https://github.com/mvdbeek/scriptrunner\n \n-import sys \n-import shutil \n-import subprocess \n-import os \n-import time \n-import tempfile \n+from __future__ import print_function\n+import sys\n+import shutil\n+import subprocess\n+import os\n+import time\n+import tempfile\n import argparse\n-import getpass\n-import tarfile\n-import re\n-import shutil\n import math\n-import fileinput\n-from os.path import abspath \n-\n+from os.path import abspath\n \n-progname = os.path.split(sys.argv[0])[1] \n-verbose = False \n+progname = os.path.split(sys.argv[0])[1]\n+verbose = False\n debug = False\n \n-def timenow():\n- """return current time as a string\n- """\n- return time.strftime(\'%d/%m/%Y %H:%M:%S\', time.localtime(time.time()))\n-\n html_escape_table = {\n- "&": "&",\n- ">": ">",\n- "<": "<",\n- "$": "\\$"\n- }\n+ "&": "&",\n+ ">": ">",\n+ "<": "<",\n+ "$": "\\$"\n+}\n+\n+\n+def timenow():\n+ """Return current time as a string."""\n+ return time.strftime(\'%d/%m/%Y %H:%M:%S\', time.localtime(time.time()))\n+\n \n def html_escape(text):\n- """Produce entities within text."""\n- return "".join(html_escape_table.get(c,c) for c in text)\n+ """Produce entities within text."""\n+ return "".join(html_escape_table.get(c, c) for c in text)\n+\n \n def cmd_exists(cmd):\n- return subprocess.call("type " + cmd, shell=True, \n- stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0\n+ return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0\n+\n \n def construct_bind(host_path, container_path=False, binds=None, ro=True):\n- #TODO remove container_path if it\'s alwyas going to be the same as host_path\n- \'\'\'build or extend binds dictionary with container path. binds is used\n- to mount all files using the docker-py client.\'\'\'\n+ # TODO remove container_path if it\'s alwyas going to be the same as host_path\n+ """Build or extend binds dictionary with container path. binds is used\n+ to mount all files using the docker-py client."""\n if not binds:\n- binds={}\n+ binds = {}\n if isinstance(host_path, list):\n- for k,v in enumerate(host_path):\n+ for k, v in enumerate(host_path):\n if not container_path:\n- container_path=host_path[k]\n- binds[host_path[k]]={\'bind\':container_path, \'ro\':ro}\n- container_path=False #could be more elegant\n+ container_path = host_path[k]\n+ binds[host_path[k]] = {\'bind\': container_path, \'ro\': ro}\n+ container_path = False # could be more elegant\n return binds\n else:\n if not container_path:\n- container_path=host_path\n- binds[host_path]={\'bind\':container_path, \'ro\':ro}\n+ container_path = host_path\n+ binds[host_path] = {\'bind\': container_path, \'ro\': ro}\n return binds\n \n+\n def switch_to_docker(opts):\n- import docker #need local import, as container does not have docker-py\n+ import docker # need local import, as container does not have docker-py\n user_id = os.getuid()\n group_id = os.getgid()\n- docker_client=docker.Client()\n- toolfactory_path=abspath(sys.argv[0])\n- binds=construct_bind(host_path=opts.script_path, ro=False)\n- binds=construct_bind(binds=binds, host_path=abspath(opts.output_dir), ro=False)\n- if len(opts.input_tab)>0:\n- binds=construct_bind(binds=binds, host_path=opts.input_tab, ro=True)\n- if not opts.output_tab == \'None\':\n- binds=construct_bind(binds=binds, host_path=opts.output_tab, ro=False)\n+ docker_client = docker.APIClient()\n+ toolfactory_path = abspath(sys.argv[0])\n+ binds = construct_bind(host_path=opts.script_path, ro=False)\n+ binds = construct_bind(binds=binds, host_path=abspath(opts.output_dir), ro=False)\n+ if len(opts.input_file) > 0:\n+ binds = construct_bind(binds=binds, host_path=opts.input_file, ro=True)\n+ if not opts.output'..b'file=sys.stderr) # same problem, need to capture docker stdin/stdout\n if self.opts.make_HTML:\n self.makeHtml()\n return retval\n@@ -359,19 +338,19 @@\n """\n if self.opts.output_dir:\n s = \'## Toolfactory generated command line = %s\\n\' % \' \'.join(self.cl)\n- sto = open(self.tlog,\'w\')\n+ sto = open(self.tlog, \'w\')\n sto.write(s)\n sto.flush()\n- p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=sto,cwd=self.opts.output_dir)\n+ p = subprocess.Popen(self.cl, shell=False, stdout=sto, stderr=sto, cwd=self.opts.output_dir)\n else:\n- p = subprocess.Popen(self.cl,shell=False) \n+ p = subprocess.Popen(self.cl, shell=False)\n retval = p.wait()\n if self.opts.output_dir:\n sto.close()\n if self.opts.make_HTML:\n self.makeHtml()\n return retval\n- \n+\n \n def change_user_id(new_uid, new_gid):\n """\n@@ -386,42 +365,37 @@\n \n \n def main():\n- u = """\n- This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as:\n- <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript"\n- </command>\n- """\n op = argparse.ArgumentParser()\n a = op.add_argument\n- a(\'--docker_image\',default=None)\n- a(\'--script_path\',default=None)\n- a(\'--tool_name\',default=None)\n- a(\'--interpreter\',default=None)\n- a(\'--output_dir\',default=\'./\')\n- a(\'--output_html\',default=None)\n- a(\'--input_tab\',default=\'None\', nargs=\'*\')\n- a(\'--output_tab\',default=\'None\')\n- a(\'--user_email\',default=\'Unknown\')\n- a(\'--bad_user\',default=None)\n- a(\'--make_HTML\',default=None)\n- a(\'--new_tool\',default=None)\n- a(\'--dockerized\',default=0)\n- a(\'--group_id\',default=None)\n- a(\'--user_id\',default=None)\n+ a(\'--docker_image\', default=None)\n+ a(\'--script_path\', default=None)\n+ a(\'--tool_name\', default=None)\n+ a(\'--interpreter\', default=None)\n+ a(\'--output_dir\', default=\'./\')\n+ a(\'--output_html\', default=None)\n+ a(\'--input_file\', default=\'None\', nargs=\'*\')\n+ a(\'--output_file\', default=\'None\')\n+ a(\'--user_email\', default=\'Unknown\')\n+ a(\'--bad_user\', default=None)\n+ a(\'--make_HTML\', default=None)\n+ a(\'--new_tool\', default=None)\n+ a(\'--dockerized\', default=0)\n+ a(\'--group_id\', default=None)\n+ a(\'--user_id\', default=None)\n a(\'--output_format\', default=\'tabular\')\n a(\'--input_format\', dest=\'input_formats\', action=\'append\', default=[])\n a(\'--additional_parameters\', dest=\'additional_parameters\', action=\'append\', default=[])\n opts = op.parse_args()\n- assert not opts.bad_user,\'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to admin_users in universe_wsgi.ini\' % (opts.bad_user,opts.bad_user)\n- assert os.path.isfile(opts.script_path),\'## Tool Factory wrapper expects a script path - eg --script_path=foo.R\'\n+ assert not opts.bad_user, \'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to admin_users in universe_wsgi.ini\' % (opts.bad_user, opts.bad_user)\n+ assert os.path.isfile(opts.script_path), \'## Tool Factory wrapper expects a script path - eg --script_path=foo.R\'\n if opts.output_dir:\n try:\n os.makedirs(opts.output_dir)\n- except:\n+ except Exception:\n pass\n- if opts.dockerized==0:\n- switch_to_docker(opts)\n- return\n+ if opts.dockerized == 0:\n+ retcode = switch_to_docker(opts)\n+ sys.exit(retcode)\n change_user_id(opts.user_id, opts.group_id)\n os.setgid(int(opts.group_id))\n os.setuid(int(opts.user_id))\n@@ -429,7 +403,7 @@\n retcode = r.run()\n os.unlink(r.sfile)\n if retcode:\n- sys.exit(retcode) # indicate failure to job runner\n+ sys.exit(retcode) # indicate failure to job runner\n \n \n if __name__ == "__main__":\n' |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 scriptrunner.xml --- a/scriptrunner.xml Sat Jul 09 17:00:06 2016 -0400 +++ b/scriptrunner.xml Sun Jul 22 13:38:01 2018 -0400 |
| [ |
| @@ -1,40 +1,40 @@ -<tool id="docker_scriptrunner" name="docker scriptrunner" version="1.1.6"> +<tool id="docker_scriptrunner" name="docker scriptrunner" version="0.1.7"> <description>Runs scripts using docker</description> <macros> <import>macros.xml</import> </macros> <requirements> - <requirement type="package" version="1.8.1">docker-py</requirement> + <requirement type="package" version="3.4.1">docker-py</requirement> </requirements> - <command> - python "$__tool_directory__/scriptrunner.py" - --script_path "$runme" - --interpreter "$interpreter" - --user_email "$__user_email__" - #if $generate_simple_output.make_TAB=="yes": - --output_tab "$tab_file" - #end if - #if $make_HTML.value=="yes": - --output_dir "$html_file.files_path" --output_html "$html_file" --make_HTML "yes" - #else: - --output_dir "." - #end if - #if $additional_parameters != 'None': - #for i in $additional_parameters: - --additional_parameters - "$i.param_name, $i.param_value" - #end for - #end if - #if $input_files != 'None': - --input_tab - #for i in $input_files: - $i.input - #end for - #for i in $input_files: - --input_format "Any" - #end for - #end if - --docker_image "$docker_image" + <command detect_errors="exit_code"> +python '$__tool_directory__/scriptrunner.py' +--script_path '$runme' +--interpreter '$interpreter' +--user_email '$__user_email__' +#if $generate_simple_output.make_file=="yes": + --output_file "$output_file" +#end if +#if $make_HTML.value=="yes": + --output_dir "$html_file.files_path" --output_html "$html_file" --make_HTML "yes" +#else: + --output_dir "." +#end if +#if $additional_parameters != 'None': + #for i in $additional_parameters: + --additional_parameters + "$i.param_name, $i.param_value" + #end for +#end if +#if $input_files != 'None': + --input_file + #for i in $input_files: + $i.input + #end for + #for i in $input_files: + --input_format "Any" + #end for +#end if +--docker_image "$docker_image" </command> <configfiles> <configfile name="runme">$dynScript</configfile> @@ -64,8 +64,8 @@ <option value="" selected="true">No, no HTML output file thanks</option> </param> <conditional name="generate_simple_output"> - <param name="make_TAB" type="select" label="Create a new history output alongside the HTML file specified above" - help="This is useful if your script creates a single new tabular file you want to appear in the history after the tool executes"> + <param name="make_file" type="select" label="Create a new history output alongside the HTML file specified above" + help="This is useful if your script creates a single new file that you want to appear in the history after the tool executes"> <option value="yes" selected="true">My script writes to a new history output</option> <option value="">I do not want a new history output file</option> </param> @@ -89,7 +89,7 @@ <options from_data_table="docker_scriptrunner_images"/> </param> <param name="dynScript" label="Copy and paste the script to be executed here" type="text" value="" area="True" size="8x120" - help="Script must deal with two command line parameters: Path to input tabular file path (or 'None' if none selected) and path to output tabular history file (or 'None')."> + help="Script must deal with two command line parameters: Path to input file path (or 'None' if none selected) and path to output tabular history file (or 'None')."> <sanitizer> <valid initial="string.printable"> </valid> @@ -98,8 +98,8 @@ </param> </inputs> <outputs> - <data format_source="input" name="tab_file"> - <filter>generate_simple_output['make_TAB'] == "yes"</filter> + <data format_source="input" name="output_file"> + <filter>generate_simple_output['make_file'] == "yes"</filter> <actions> <action type="format"> <option type="from_param" name="generate_simple_output.out_format" /> @@ -112,14 +112,14 @@ </outputs> <tests> <test> - <param name='input_tab' value='tf2_test_in.xls' ftype='tabular' /> - <param name="make_TAB" value="yes" /> - <param name="make_HTML" value="yes" /> - <param name="out_format" value="tabular" /> - <param name="interpreter" value='python' /> + <param name="input_file" value="tf2_test_in.xls" ftype="tabular"/> + <param name="make_file" value="yes"/> + <param name="make_HTML" value="yes"/> + <param name="out_format" value="tabular"/> + <param name="interpreter" value="python"/> <param name="runme" value="tf2_test_runme.py"/> - <output name='output1' file='tf2_test_out.xls' compare='diff' lines_diff = '10'/> - <output name='html_file' file="tf2_test.html" compare='diff' lines_diff = '10'/> + <output name="output_file" file="tf2_test_out.xls" compare="diff" lines_diff="10"/> + <output name="html_file" file="tf2_test.html" compare="diff" lines_diff="10"/> </test> </tests> <expand macro="help_macro" /> |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 test-data/tf2_test.html --- a/test-data/tf2_test.html Sat Jul 09 17:00:06 2016 -0400 +++ b/test-data/tf2_test.html Sun Jul 22 13:38:01 2018 -0400 |
| b |
| @@ -1,25 +1,25 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> - <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <meta name="generator" content="Galaxy scriptrunner.py tool output - see http://g2.trac.bx.psu.edu/" /> - <title></title> - <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> - </head> - <body> - <div class="toolFormBody"> - -<div class="infomessage">Galaxy Tool "script" run at 09/07/2016 11:37:54</div><br/> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <meta name="generator" content="Galaxy scriptrunner.py tool output - see http://g2.trac.bx.psu.edu/" /> + <title></title> + <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> + </head> + <body> + <div class="toolFormBody"> + +<div class="infomessage">Galaxy Tool "script" run at 22/07/2018 17:09:47</div><br/> <div class="toolFormTitle">script log output</div> script_error.log is empty<br/> <div class="toolFormTitle">Other log output</div> -/tmp/tmpG7m9zp/job_working_directory/000/1/dataset_2_files/script_runner.log is empty<br/> +/private/var/folders/df/6xqpqpcd7h73b6jpx9t6cwhw0000gn/T/tmptx2hJF/job_working_directory/000/1/dataset_2_files/script_runner.log is empty<br/> <div class="toolFormTitle">All output files available for downloading</div> <div><table class="colored" cellpadding="3" cellspacing="3"><tr><th>Output File Name (click to view)</th><th>Size</th></tr> <tr><td><a href="script.python">script.python</a></td><td>0 B</td></tr> <tr class="odd_row"><td><a href="script_error.log">script_error.log</a></td><td>0 B</td></tr> -<tr><td><a href="script_runner.log">script_runner.log</a></td><td>100 B</td></tr> +<tr><td><a href="script_runner.log">script_runner.log</a></td><td>140 B</td></tr> </table></div><br/> </div></body></html> |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 test.sh --- a/test.sh Sat Jul 09 17:00:06 2016 -0400 +++ b/test.sh Sun Jul 22 13:38:01 2018 -0400 |
| b |
| @@ -1,7 +1,2 @@ #!/usr/bin/env bash -planemo test --galaxy_branch dev \ - --conda_auto_init \ - --conda_dependency_resolution \ - --conda_auto_install \ - --conda_ensure_channels scrapinghub \ - --tool_data_table tool_data_table_conf.xml.sample.test +planemo test --tool_data_table tool_data_table_conf.xml.sample.test "$@" |
| b |
| diff -r 315a7e9ed6eb -r 495946ffc2d6 tool_dependencies.xml --- a/tool_dependencies.xml Sat Jul 09 17:00:06 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,14 +0,0 @@ -<?xml version="1.0"?> -<tool_dependency> - <package name="docker-py" version="1.8.1"> - <install version="1.0"> - <actions> - <action type="setup_virtualenv">docker-py==1.8.1 - </action> - </actions> - </install> - </package> - <readme> - Only Admins can use this tool generator but please do NOT install o - </readme> -</tool_dependency> |