Mercurial > repos > mvdbeek > docker_scriptrunner
changeset 2:495946ffc2d6 draft default tip
planemo upload for repository https://github.com/mvdbeek/docker_scriptrunner/ commit dded837d19aeb3f06b84e5076282cedeeaf713fa
| author | mvdbeek |
|---|---|
| date | Sun, 22 Jul 2018 13:38:01 -0400 |
| parents | 315a7e9ed6eb |
| children | |
| files | README.md dockerfiles/build.sh dockerfiles/scriptrunner/Dockerfile planemo_serve_script.sh scriptrunner.py scriptrunner.xml test-data/tf2_test.html test.sh tool_dependencies.xml |
| diffstat | 9 files changed, 284 insertions(+), 299 deletions(-) [+] |
line wrap: on
line diff
--- a/README.md Sat Jul 09 17:00:06 2016 -0400 +++ b/README.md Sun Jul 22 13:38:01 2018 -0400 @@ -1,6 +1,35 @@ +[](https://travis-ci.org/ARTbio/docker-scriptrunner) + +You need to have docker Docker scriptrunner for galaxy ----------------------------- This is a [galaxy](https://github.com/galaxyproject/galaxy) tool that allows users to submit random scripts. -You can install it on your galaxy server from the [galaxy toolshed]() +You can install it on your galaxy server from the [galaxy toolshed] (https://toolshed.g2.bx.psu.edu/view/mvdbeek/docker_scriptrunner/). + + +This tool is heavily inspired by Ross Lazarus' tool factory +(https://www.ncbi.nlm.nih.gov/pubmed/23024011), but removes the ability to +create galaxy tools. (If you are looking at creating galaxy tools, you may want +to use [planemo](https://planemo.readthedocs.io/en/latest/)). + + +You need to have docker installed on any machine that can run +galaxy jobs, or route this tool to a dedicated docker host +in galaxy's job_conf.xml. Note that the tool itself talks +to the docker daemon and bypasses galaxy's docker configuration. +Making use of galaxy's docker capabilities is on the roadmap. + + +The tool comes with two docker images that can be easily extended. +"artbio/scriptrunner" is a base image, that has very few dependencies installed. +You can extend the image by following the example in the dockerfiles/r-bioperl-python +folder. + + +All security relies on docker; the container only mounts those files that the user +has selected as input files, and the script itself drops privileged inside +the container. This should be secure, but caution should be taken on public servers. +By default the container has networking enabled, so make sure your firewall rules +forbid traffic to the local network.
--- a/dockerfiles/build.sh Sat Jul 09 17:00:06 2016 -0400 +++ b/dockerfiles/build.sh Sun Jul 22 13:38:01 2018 -0400 @@ -1,3 +1,4 @@ #!/usr/bin/env bash +cd "${0%/*}" docker build -t artbio/scriptrunner scriptrunner && \ docker build -t artbio/scriptrunner-r-bioperl-python r-bioperl-python
--- a/dockerfiles/scriptrunner/Dockerfile Sat Jul 09 17:00:06 2016 -0400 +++ b/dockerfiles/scriptrunner/Dockerfile Sun Jul 22 13:38:01 2018 -0400 @@ -26,8 +26,8 @@ RUN curl https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -L -o miniconda.sh RUN bash miniconda.sh -b -p "$CONDA_PREFIX" && rm miniconda.sh -RUN bash -c "conda create -y -q -c bioconda --name default samtools==0.1.19 bcftools && \ - conda config --add channels r && \ +RUN bash -c "conda create -y -q -c bioconda --name default samtools bcftools && \ + conda config --add channels conda-forge && \ conda config --add channels bioconda && \ conda config --add channels iuc && \ . activate default && \
--- a/planemo_serve_script.sh Sat Jul 09 17:00:06 2016 -0400 +++ b/planemo_serve_script.sh Sun Jul 22 13:38:01 2018 -0400 @@ -1,1 +1,1 @@ -planemo serve --galaxy_branch dev --conda_prefix=/conda --conda_dependency_resolution --profile tf --port 80 --host 0.0.0.0 --job_config_file ~/mydisk/job_conf.xml --tool_data_table tool_data_table_conf.xml.sample.test +planemo serve --tool_data_table tool_data_table_conf.xml.sample.test "$@"
--- a/scriptrunner.py Sat Jul 09 17:00:06 2016 -0400 +++ b/scriptrunner.py Sun Jul 22 13:38:01 2018 -0400 @@ -1,213 +1,195 @@ # DockerToolFactory.py # see https://github.com/mvdbeek/scriptrunner -import sys -import shutil -import subprocess -import os -import time -import tempfile +from __future__ import print_function +import sys +import shutil +import subprocess +import os +import time +import tempfile import argparse -import getpass -import tarfile -import re -import shutil import math -import fileinput -from os.path import abspath - +from os.path import abspath -progname = os.path.split(sys.argv[0])[1] -verbose = False +progname = os.path.split(sys.argv[0])[1] +verbose = False debug = False -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) - html_escape_table = { - "&": "&", - ">": ">", - "<": "<", - "$": "\$" - } + "&": "&", + ">": ">", + "<": "<", + "$": "\$" +} + + +def timenow(): + """Return current time as a string.""" + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + def html_escape(text): - """Produce entities within text.""" - return "".join(html_escape_table.get(c,c) for c in text) + """Produce entities within text.""" + return "".join(html_escape_table.get(c, c) for c in text) + def cmd_exists(cmd): - return subprocess.call("type " + cmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + def construct_bind(host_path, container_path=False, binds=None, ro=True): - #TODO remove container_path if it's alwyas going to be the same as host_path - '''build or extend binds dictionary with container path. binds is used - to mount all files using the docker-py client.''' + # TODO remove container_path if it's alwyas going to be the same as host_path + """Build or extend binds dictionary with container path. binds is used + to mount all files using the docker-py client.""" if not binds: - binds={} + binds = {} if isinstance(host_path, list): - for k,v in enumerate(host_path): + for k, v in enumerate(host_path): if not container_path: - container_path=host_path[k] - binds[host_path[k]]={'bind':container_path, 'ro':ro} - container_path=False #could be more elegant + container_path = host_path[k] + binds[host_path[k]] = {'bind': container_path, 'ro': ro} + container_path = False # could be more elegant return binds else: if not container_path: - container_path=host_path - binds[host_path]={'bind':container_path, 'ro':ro} + container_path = host_path + binds[host_path] = {'bind': container_path, 'ro': ro} return binds + def switch_to_docker(opts): - import docker #need local import, as container does not have docker-py + import docker # need local import, as container does not have docker-py user_id = os.getuid() group_id = os.getgid() - docker_client=docker.Client() - toolfactory_path=abspath(sys.argv[0]) - binds=construct_bind(host_path=opts.script_path, ro=False) - binds=construct_bind(binds=binds, host_path=abspath(opts.output_dir), ro=False) - if len(opts.input_tab)>0: - binds=construct_bind(binds=binds, host_path=opts.input_tab, ro=True) - if not opts.output_tab == 'None': - binds=construct_bind(binds=binds, host_path=opts.output_tab, ro=False) + docker_client = docker.APIClient() + toolfactory_path = abspath(sys.argv[0]) + binds = construct_bind(host_path=opts.script_path, ro=False) + binds = construct_bind(binds=binds, host_path=abspath(opts.output_dir), ro=False) + if len(opts.input_file) > 0: + binds = construct_bind(binds=binds, host_path=opts.input_file, ro=True) + if not opts.output_file == 'None': + binds = construct_bind(binds=binds, host_path=opts.output_file, ro=False) if opts.make_HTML: - binds=construct_bind(binds=binds, host_path=opts.output_html, ro=False) - binds=construct_bind(binds=binds, host_path=toolfactory_path) - volumes=binds.keys() - sys.argv=[abspath(opts.output_dir) if sys.argv[i-1]=='--output_dir' else arg for i,arg in enumerate(sys.argv)] ##inject absolute path of working_dir - cmd=['python', '-u']+sys.argv+['--dockerized', '1', "--user_id", str(user_id), "--group_id", str(group_id)] - image_exists = [ True for image in docker_client.images() if opts.docker_image in image['RepoTags'] ] + binds = construct_bind(binds=binds, host_path=opts.output_html, ro=False) + binds = construct_bind(binds=binds, host_path=toolfactory_path) + volumes = list(binds.keys()) + sys.argv = [abspath(opts.output_dir) if sys.argv[i - 1] == '--output_dir' else arg for i, arg in enumerate(sys.argv)] # inject absolute path of working_dir + cmd = ['python', '-u'] + sys.argv + ['--dockerized', '1', "--user_id", str(user_id), "--group_id", str(group_id)] + image_exists = [True for image in docker_client.images() if opts.docker_image in image['RepoTags']] if not image_exists: docker_client.pull(opts.docker_image) - container=docker_client.create_container( + container = docker_client.create_container( image=opts.docker_image, volumes=volumes, - command=cmd - ) - docker_client.start(container=container[u'Id'], binds=binds) - docker_client.wait(container=container[u'Id']) - logs=docker_client.logs(container=container[u'Id']) - print "".join([log for log in logs]) + command=cmd, + host_config=docker_client.create_host_config(binds=binds)) + docker_client.start(container=container[u'Id']) + exit_code = docker_client.wait(container=container[u'Id'])['StatusCode'] + logs = docker_client.logs(container=container[u'Id']) + print(logs, end="", file=sys.stderr) docker_client.remove_container(container[u'Id']) + return exit_code + class ScriptRunner: """class is a wrapper for an arbitrary script """ - def __init__(self,opts=None,treatbashSpecial=True, image_tag='base'): + def __init__(self, opts=None, treatbashSpecial=True, image_tag='base'): """ cleanup inputs, setup some outputs - """ self.opts = opts self.scriptname = 'script' - self.useIM = cmd_exists('convert') - self.useGS = cmd_exists('gs') - self.temp_warned = False # we want only one warning if $TMP not set self.treatbashSpecial = treatbashSpecial self.image_tag = image_tag os.chdir(abspath(opts.output_dir)) self.thumbformat = 'png' - s = open(self.opts.script_path,'r').readlines() - s = [x.rstrip() for x in s] # remove pesky dos line endings if needed + s = open(self.opts.script_path, 'r').readlines() + s = [x.rstrip() for x in s] # remove pesky dos line endings if needed self.script = '\n'.join(s) - fhandle,self.sfile = tempfile.mkstemp(prefix='script',suffix=".%s" % (opts.interpreter)) - tscript = open(self.sfile,'w') # use self.sfile as script source for Popen + fhandle, self.sfile = tempfile.mkstemp(prefix='script', suffix=".%s" % (opts.interpreter)) + tscript = open(self.sfile, 'w') # use self.sfile as script source for Popen tscript.write(self.script) tscript.close() - self.indentedScript = '\n'.join([' %s' % html_escape(x) for x in s]) # for restructured text in help + self.indentedScript = '\n'.join([' %s' % html_escape(x) for x in s]) # for restructured text in help self.escapedScript = '\n'.join([html_escape(x) for x in s]) - self.elog = os.path.join(self.opts.output_dir,"%s_error.log" % self.scriptname) - if opts.output_dir: # may not want these complexities - self.tlog = os.path.join(self.opts.output_dir,"%s_runner.log" % self.scriptname) - art = '%s.%s' % (self.scriptname,opts.interpreter) - artpath = os.path.join(self.opts.output_dir,art) # need full path - artifact = open(artpath,'w') # use self.sfile as script source for Popen + self.elog = os.path.join(self.opts.output_dir, "%s_error.log" % self.scriptname) + if opts.output_dir: # may not want these complexities + self.tlog = os.path.join(self.opts.output_dir, "%s_runner.log" % self.scriptname) + art = '%s.%s' % (self.scriptname, opts.interpreter) + artpath = os.path.join(self.opts.output_dir, art) # need full path + artifact = open(artpath, 'w') # use self.sfile as script source for Popen artifact.write(self.script) artifact.close() self.cl = [] self.html = [] - a = self.cl.append - a(opts.interpreter) - if self.treatbashSpecial and opts.interpreter in ['bash','sh']: - a(self.sfile) + self.cl.append(opts.interpreter) + if self.treatbashSpecial and opts.interpreter in ['bash', 'sh']: + self.cl.append(self.sfile) else: - a('-') # stdin - for input in opts.input_tab: - a(input) - if opts.output_tab == 'None': #If tool generates only HTML, set output name to toolname - a(str(self.scriptname)+'.out') - a(opts.output_tab) - for param in opts.additional_parameters: - param, value=param.split(',') - a('--'+param) - a(value) + self.cl.append('-') # stdin + for input in opts.input_file: + self.cl.append(input) + if opts.output_file == 'None': # If tool generates only HTML, set output name to toolname + self.cl.append(str(self.scriptname) + '.out') + self.cl.append(opts.output_file) + for param in opts.additional_parameters: + param, value = param.split(',') + self.cl.append('--' + param) + self.cl.append(value) self.outFormats = opts.output_format self.inputFormats = [formats for formats in opts.input_formats] self.test1Input = '%s_test1_input.xls' % self.scriptname self.test1Output = '%s_test1_output.xls' % self.scriptname self.test1HTML = '%s_test1_output.html' % self.scriptname - - def compressPDF(self,inpdf=None,thumbformat='png'): - """need absolute path to pdf - note that GS gets confoozled if no $TMP or $TEMP - so we set it + def compressPDF(self, inpdf=None, thumbformat='png'): + """ + inpdf is absolute path to PDF """ - assert os.path.isfile(inpdf), "## Input %s supplied to %s compressPDF not found" % (inpdf,self.myName) - hlog = os.path.join(self.opts.output_dir,"compress_%s.txt" % os.path.basename(inpdf)) - sto = open(hlog,'a') + assert os.path.isfile(inpdf), "## Input %s supplied to %s compressPDF not found" % (inpdf, self.myName) + hlog = os.path.join(self.opts.output_dir, "compress_%s.txt" % os.path.basename(inpdf)) + sto = open(hlog, 'a') our_env = os.environ.copy() - our_tmp = our_env.get('TMP',None) + our_tmp = our_env.get('TMP', None) if not our_tmp: - our_tmp = our_env.get('TEMP',None) - if not (our_tmp and os.path.exists(our_tmp)): - newtmp = os.path.join(self.opts.output_dir,'tmp') - try: - os.mkdir(newtmp) - except: - sto.write('## WARNING - cannot make %s - it may exist or permissions need fixing\n' % newtmp) - our_env['TEMP'] = newtmp - if not self.temp_warned: - sto.write('## WARNING - no $TMP or $TEMP!!! Please fix - using %s temporarily\n' % newtmp) - self.temp_warned = True + our_env['TMP'] = tempfile.gettempdir() outpdf = '%s_compressed' % inpdf - cl = ["gs", "-sDEVICE=pdfwrite", "-dNOPAUSE", "-dUseCIEColor", "-dBATCH","-dPDFSETTINGS=/printer", "-sOutputFile=%s" % outpdf,inpdf] - x = subprocess.Popen(cl,stdout=sto,stderr=sto,cwd=self.opts.output_dir,env=our_env) + cl = ["gs", "-sDEVICE=pdfwrite", "-dNOPAUSE", "-dUseCIEColor", "-dBATCH", "-dPDFSETTINGS=/printer", "-sOutputFile=%s" % outpdf, inpdf] + x = subprocess.Popen(cl, stdout=sto, stderr=sto, cwd=self.opts.output_dir, env=our_env) retval1 = x.wait() sto.close() if retval1 == 0: os.unlink(inpdf) - shutil.move(outpdf,inpdf) + shutil.move(outpdf, inpdf) os.unlink(hlog) - hlog = os.path.join(self.opts.output_dir,"thumbnail_%s.txt" % os.path.basename(inpdf)) - sto = open(hlog,'w') - outpng = '%s.%s' % (os.path.splitext(inpdf)[0],thumbformat) + hlog = os.path.join(self.opts.output_dir, "thumbnail_%s.txt" % os.path.basename(inpdf)) + sto = open(hlog, 'w') + outpng = '%s.%s' % (os.path.splitext(inpdf)[0], thumbformat) cl2 = ['convert', inpdf, outpng] - x = subprocess.Popen(cl2,stdout=sto,stderr=sto,cwd=self.opts.output_dir,env=our_env) + x = subprocess.Popen(cl2, stdout=sto, stderr=sto, cwd=self.opts.output_dir, env=our_env) retval2 = x.wait() sto.close() if retval2 == 0: - os.unlink(hlog) + os.unlink(hlog) retval = retval1 or retval2 return retval - - def getfSize(self,fpath,outpath): + def getfSize(self, fpath, outpath): """ format a nice file size string """ size = '' - fp = os.path.join(outpath,fpath) + fp = os.path.join(outpath, fpath) if os.path.isfile(fp): size = '0 B' n = float(os.path.getsize(fp)) if n > 2**20: - size = '%1.1f MB' % (n/2**20) + size = '%1.1f MB' % (n / 2**20) elif n > 2**10: - size = '%1.1f KB' % (n/2**10) + size = '%1.1f KB' % (n / 2**10) elif n > 0: size = '%d B' % (int(n)) return size @@ -216,92 +198,91 @@ """ Create an HTML file content to list all the artifacts found in the output_dir """ - galhtmlprefix = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> - <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> - <title></title> - <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> - </head> - <body> - <div class="toolFormBody"> - """ - galhtmlattr = """<hr/><div class="infomessage">This tool (%s) was generated by the <a href="https://bitbucket.org/fubar/galaxytoolfactory/overview">Galaxy Tool Factory</a></div><br/>""" + galhtmlprefix = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> + <title></title> + <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> + </head> + <body> + <div class="toolFormBody"> +""" galhtmlpostfix = """</div></body></html>\n""" flist = os.listdir(self.opts.output_dir) - flist = [x for x in flist if x <> 'Rplots.pdf'] + flist = [x for x in flist if x != 'Rplots.pdf'] flist.sort() html = [] html.append(galhtmlprefix % progname) - html.append('<div class="infomessage">Galaxy Tool "%s" run at %s</div><br/>' % (self.scriptname,timenow())) + html.append('<div class="infomessage">Galaxy Tool "%s" run at %s</div><br/>' % (self.scriptname, timenow())) fhtml = [] if len(flist) > 0: - logfiles = [x for x in flist if x.lower().endswith('.log')] # log file names determine sections + logfiles = [x for x in flist if x.lower().endswith('.log')] # log file names determine sections logfiles.sort() - logfiles = [x for x in logfiles if abspath(x) <> abspath(self.tlog)] - logfiles.append(abspath(self.tlog)) # make it the last one + logfiles = [x for x in logfiles if abspath(x) != abspath(self.tlog)] + logfiles.append(abspath(self.tlog)) # make it the last one pdflist = [] npdf = len([x for x in flist if os.path.splitext(x)[-1].lower() == '.pdf']) - for rownum,fname in enumerate(flist): - dname,e = os.path.splitext(fname) - sfsize = self.getfSize(fname,self.opts.output_dir) - if e.lower() == '.pdf' : # compress and make a thumbnail - thumb = '%s.%s' % (dname,self.thumbformat) - pdff = os.path.join(self.opts.output_dir,fname) - retval = self.compressPDF(inpdf=pdff,thumbformat=self.thumbformat) + for rownum, fname in enumerate(flist): + dname, e = os.path.splitext(fname) + sfsize = self.getfSize(fname, self.opts.output_dir) + if e.lower() == '.pdf': # compress and make a thumbnail + thumb = '%s.%s' % (dname, self.thumbformat) + pdff = os.path.join(self.opts.output_dir, fname) + retval = self.compressPDF(inpdf=pdff, thumbformat=self.thumbformat) if retval == 0: - pdflist.append((fname,thumb)) + pdflist.append((fname, thumb)) else: - pdflist.append((fname,fname)) - if (rownum+1) % 2 == 0: - fhtml.append('<tr class="odd_row"><td><a href="%s">%s</a></td><td>%s</td></tr>' % (fname,fname,sfsize)) + pdflist.append((fname, fname)) + if (rownum + 1) % 2 == 0: + fhtml.append('<tr class="odd_row"><td><a href="%s">%s</a></td><td>%s</td></tr>' % (fname, fname, sfsize)) else: - fhtml.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>' % (fname,fname,sfsize)) - for logfname in logfiles: # expect at least tlog - if more - if abspath(logfname) == abspath(self.tlog): # handled later + fhtml.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>' % (fname, fname, sfsize)) + for logfname in logfiles: # expect at least tlog - if more + if abspath(logfname) == abspath(self.tlog): # handled later sectionname = 'All tool run' if (len(logfiles) > 1): sectionname = 'Other' ourpdfs = pdflist else: realname = os.path.basename(logfname) - sectionname = os.path.splitext(realname)[0].split('_')[0] # break in case _ added to log + sectionname = os.path.splitext(realname)[0].split('_')[0] # break in case _ added to log ourpdfs = [x for x in pdflist if os.path.basename(x[0]).split('_')[0] == sectionname] - pdflist = [x for x in pdflist if os.path.basename(x[0]).split('_')[0] <> sectionname] # remove + pdflist = [x for x in pdflist if os.path.basename(x[0]).split('_')[0] != sectionname] # remove nacross = 1 npdf = len(ourpdfs) if npdf > 0: - nacross = math.sqrt(npdf) ## int(round(math.log(npdf,2))) + nacross = math.sqrt(npdf) if int(nacross)**2 != npdf: nacross += 1 nacross = int(nacross) - width = min(400,int(1200/nacross)) + width = min(400, int(1200 / nacross)) html.append('<div class="toolFormTitle">%s images and outputs</div>' % sectionname) html.append('(Click on a thumbnail image to download the corresponding original PDF image)<br/>') - ntogo = nacross # counter for table row padding with empty cells + ntogo = nacross # counter for table row padding with empty cells html.append('<div><table class="simple" cellpadding="2" cellspacing="2">\n<tr>') - for i,paths in enumerate(ourpdfs): - fname,thumb = paths - s= """<td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="5" width="%d" - alt="Image called %s"/></a></td>\n""" % (fname,thumb,fname,width,fname) - if ((i+1) % nacross == 0): + for i, paths in enumerate(ourpdfs): + fname, thumb = paths + s = """<td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="5" width="%d" + alt="Image called %s"/></a></td>\n""" % (fname, thumb, fname, width, fname) + if ((i + 1) % nacross == 0): s += '</tr>\n' ntogo = 0 - if i < (npdf - 1): # more to come - s += '<tr>' - ntogo = nacross + if i < (npdf - 1): # more to come + s += '<tr>' + ntogo = nacross else: ntogo -= 1 html.append(s) if html[-1].strip().endswith('</tr>'): html.append('</table></div>\n') else: - if ntogo > 0: # pad - html.append('<td> </td>'*ntogo) + if ntogo > 0: # pad + html.append('<td> </td>' * ntogo) html.append('</tr></table></div>\n') - logt = open(logfname,'r').readlines() + logt = open(logfname, 'r').readlines() logtext = [x for x in logt if x.strip() > ''] html.append('<div class="toolFormTitle">%s log output</div>' % sectionname) if len(logtext) > 1: @@ -311,44 +292,42 @@ else: html.append('%s is empty<br/>' % logfname) if len(fhtml) > 0: - fhtml.insert(0,'<div><table class="colored" cellpadding="3" cellspacing="3"><tr><th>Output File Name (click to view)</th><th>Size</th></tr>\n') - fhtml.append('</table></div><br/>') - html.append('<div class="toolFormTitle">All output files available for downloading</div>\n') - html += fhtml # add all non-pdf files to the end of the display + fhtml.insert(0, '<div><table class="colored" cellpadding="3" cellspacing="3"><tr><th>Output File Name (click to view)</th><th>Size</th></tr>\n') + fhtml.append('</table></div><br/>') + html.append('<div class="toolFormTitle">All output files available for downloading</div>\n') + html += fhtml # add all non-pdf files to the end of the display else: html.append('<div class="warningmessagelarge">### Error - %s returned no files - please confirm that parameters are sane</div>' % self.opts.interpreter) html.append(galhtmlpostfix) - htmlf = file(self.opts.output_html,'w') - htmlf.write('\n'.join(html)) - htmlf.write('\n') - htmlf.close() + with open(self.opts.output_html, 'w') as htmlf: + htmlf.write('\n'.join(html)) + htmlf.write('\n') self.html = html - def run(self): """ scripts must be small enough not to fill the pipe! """ - if self.treatbashSpecial and self.opts.interpreter in ['bash','sh']: - retval = self.runBash() + if self.treatbashSpecial and self.opts.interpreter in ['bash', 'sh']: + retval = self.runBash() else: if self.opts.output_dir: - ste = open(self.elog,'w') - sto = open(self.tlog,'w') + ste = open(self.elog, 'w') + sto = open(self.tlog, 'w') sto.write('## Toolfactory generated command line = %s\n' % ' '.join(self.cl)) sto.flush() - p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=ste,stdin=subprocess.PIPE,cwd=self.opts.output_dir) + p = subprocess.Popen(self.cl, shell=False, stdout=sto, stderr=ste, stdin=subprocess.PIPE, cwd=self.opts.output_dir) else: - p = subprocess.Popen(self.cl,shell=False,stdin=subprocess.PIPE) + p = subprocess.Popen(self.cl, shell=False, stdin=subprocess.PIPE) p.stdin.write(self.script) p.stdin.close() retval = p.wait() if self.opts.output_dir: sto.close() ste.close() - err = open(self.elog,'r').readlines() - if retval <> 0 and err: # problem - print >> sys.stderr,err #same problem, need to capture docker stdin/stdout + err = open(self.elog, 'r').readlines() + if retval != 0 and err: # problem + print(err, end="", file=sys.stderr) # same problem, need to capture docker stdin/stdout if self.opts.make_HTML: self.makeHtml() return retval @@ -359,19 +338,19 @@ """ if self.opts.output_dir: s = '## Toolfactory generated command line = %s\n' % ' '.join(self.cl) - sto = open(self.tlog,'w') + sto = open(self.tlog, 'w') sto.write(s) sto.flush() - p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=sto,cwd=self.opts.output_dir) + p = subprocess.Popen(self.cl, shell=False, stdout=sto, stderr=sto, cwd=self.opts.output_dir) else: - p = subprocess.Popen(self.cl,shell=False) + p = subprocess.Popen(self.cl, shell=False) retval = p.wait() if self.opts.output_dir: sto.close() if self.opts.make_HTML: self.makeHtml() return retval - + def change_user_id(new_uid, new_gid): """ @@ -386,42 +365,37 @@ def main(): - u = """ - This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: - <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" - </command> - """ op = argparse.ArgumentParser() a = op.add_argument - a('--docker_image',default=None) - a('--script_path',default=None) - a('--tool_name',default=None) - a('--interpreter',default=None) - a('--output_dir',default='./') - a('--output_html',default=None) - a('--input_tab',default='None', nargs='*') - a('--output_tab',default='None') - a('--user_email',default='Unknown') - a('--bad_user',default=None) - a('--make_HTML',default=None) - a('--new_tool',default=None) - a('--dockerized',default=0) - a('--group_id',default=None) - a('--user_id',default=None) + a('--docker_image', default=None) + a('--script_path', default=None) + a('--tool_name', default=None) + a('--interpreter', default=None) + a('--output_dir', default='./') + a('--output_html', default=None) + a('--input_file', default='None', nargs='*') + a('--output_file', default='None') + a('--user_email', default='Unknown') + a('--bad_user', default=None) + a('--make_HTML', default=None) + a('--new_tool', default=None) + a('--dockerized', default=0) + a('--group_id', default=None) + a('--user_id', default=None) a('--output_format', default='tabular') a('--input_format', dest='input_formats', action='append', default=[]) a('--additional_parameters', dest='additional_parameters', action='append', default=[]) opts = op.parse_args() - assert not opts.bad_user,'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to admin_users in universe_wsgi.ini' % (opts.bad_user,opts.bad_user) - assert os.path.isfile(opts.script_path),'## Tool Factory wrapper expects a script path - eg --script_path=foo.R' + assert not opts.bad_user, 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to admin_users in universe_wsgi.ini' % (opts.bad_user, opts.bad_user) + assert os.path.isfile(opts.script_path), '## Tool Factory wrapper expects a script path - eg --script_path=foo.R' if opts.output_dir: try: os.makedirs(opts.output_dir) - except: + except Exception: pass - if opts.dockerized==0: - switch_to_docker(opts) - return + if opts.dockerized == 0: + retcode = switch_to_docker(opts) + sys.exit(retcode) change_user_id(opts.user_id, opts.group_id) os.setgid(int(opts.group_id)) os.setuid(int(opts.user_id)) @@ -429,7 +403,7 @@ retcode = r.run() os.unlink(r.sfile) if retcode: - sys.exit(retcode) # indicate failure to job runner + sys.exit(retcode) # indicate failure to job runner if __name__ == "__main__":
--- a/scriptrunner.xml Sat Jul 09 17:00:06 2016 -0400 +++ b/scriptrunner.xml Sun Jul 22 13:38:01 2018 -0400 @@ -1,40 +1,40 @@ -<tool id="docker_scriptrunner" name="docker scriptrunner" version="1.1.6"> +<tool id="docker_scriptrunner" name="docker scriptrunner" version="0.1.7"> <description>Runs scripts using docker</description> <macros> <import>macros.xml</import> </macros> <requirements> - <requirement type="package" version="1.8.1">docker-py</requirement> + <requirement type="package" version="3.4.1">docker-py</requirement> </requirements> - <command> - python "$__tool_directory__/scriptrunner.py" - --script_path "$runme" - --interpreter "$interpreter" - --user_email "$__user_email__" - #if $generate_simple_output.make_TAB=="yes": - --output_tab "$tab_file" - #end if - #if $make_HTML.value=="yes": - --output_dir "$html_file.files_path" --output_html "$html_file" --make_HTML "yes" - #else: - --output_dir "." - #end if - #if $additional_parameters != 'None': - #for i in $additional_parameters: - --additional_parameters - "$i.param_name, $i.param_value" - #end for - #end if - #if $input_files != 'None': - --input_tab - #for i in $input_files: - $i.input - #end for - #for i in $input_files: - --input_format "Any" - #end for - #end if - --docker_image "$docker_image" + <command detect_errors="exit_code"> +python '$__tool_directory__/scriptrunner.py' +--script_path '$runme' +--interpreter '$interpreter' +--user_email '$__user_email__' +#if $generate_simple_output.make_file=="yes": + --output_file "$output_file" +#end if +#if $make_HTML.value=="yes": + --output_dir "$html_file.files_path" --output_html "$html_file" --make_HTML "yes" +#else: + --output_dir "." +#end if +#if $additional_parameters != 'None': + #for i in $additional_parameters: + --additional_parameters + "$i.param_name, $i.param_value" + #end for +#end if +#if $input_files != 'None': + --input_file + #for i in $input_files: + $i.input + #end for + #for i in $input_files: + --input_format "Any" + #end for +#end if +--docker_image "$docker_image" </command> <configfiles> <configfile name="runme">$dynScript</configfile> @@ -64,8 +64,8 @@ <option value="" selected="true">No, no HTML output file thanks</option> </param> <conditional name="generate_simple_output"> - <param name="make_TAB" type="select" label="Create a new history output alongside the HTML file specified above" - help="This is useful if your script creates a single new tabular file you want to appear in the history after the tool executes"> + <param name="make_file" type="select" label="Create a new history output alongside the HTML file specified above" + help="This is useful if your script creates a single new file that you want to appear in the history after the tool executes"> <option value="yes" selected="true">My script writes to a new history output</option> <option value="">I do not want a new history output file</option> </param> @@ -89,7 +89,7 @@ <options from_data_table="docker_scriptrunner_images"/> </param> <param name="dynScript" label="Copy and paste the script to be executed here" type="text" value="" area="True" size="8x120" - help="Script must deal with two command line parameters: Path to input tabular file path (or 'None' if none selected) and path to output tabular history file (or 'None')."> + help="Script must deal with two command line parameters: Path to input file path (or 'None' if none selected) and path to output tabular history file (or 'None')."> <sanitizer> <valid initial="string.printable"> </valid> @@ -98,8 +98,8 @@ </param> </inputs> <outputs> - <data format_source="input" name="tab_file"> - <filter>generate_simple_output['make_TAB'] == "yes"</filter> + <data format_source="input" name="output_file"> + <filter>generate_simple_output['make_file'] == "yes"</filter> <actions> <action type="format"> <option type="from_param" name="generate_simple_output.out_format" /> @@ -112,14 +112,14 @@ </outputs> <tests> <test> - <param name='input_tab' value='tf2_test_in.xls' ftype='tabular' /> - <param name="make_TAB" value="yes" /> - <param name="make_HTML" value="yes" /> - <param name="out_format" value="tabular" /> - <param name="interpreter" value='python' /> + <param name="input_file" value="tf2_test_in.xls" ftype="tabular"/> + <param name="make_file" value="yes"/> + <param name="make_HTML" value="yes"/> + <param name="out_format" value="tabular"/> + <param name="interpreter" value="python"/> <param name="runme" value="tf2_test_runme.py"/> - <output name='output1' file='tf2_test_out.xls' compare='diff' lines_diff = '10'/> - <output name='html_file' file="tf2_test.html" compare='diff' lines_diff = '10'/> + <output name="output_file" file="tf2_test_out.xls" compare="diff" lines_diff="10"/> + <output name="html_file" file="tf2_test.html" compare="diff" lines_diff="10"/> </test> </tests> <expand macro="help_macro" />
--- a/test-data/tf2_test.html Sat Jul 09 17:00:06 2016 -0400 +++ b/test-data/tf2_test.html Sun Jul 22 13:38:01 2018 -0400 @@ -1,25 +1,25 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> - <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <meta name="generator" content="Galaxy scriptrunner.py tool output - see http://g2.trac.bx.psu.edu/" /> - <title></title> - <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> - </head> - <body> - <div class="toolFormBody"> - -<div class="infomessage">Galaxy Tool "script" run at 09/07/2016 11:37:54</div><br/> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <meta name="generator" content="Galaxy scriptrunner.py tool output - see http://g2.trac.bx.psu.edu/" /> + <title></title> + <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> + </head> + <body> + <div class="toolFormBody"> + +<div class="infomessage">Galaxy Tool "script" run at 22/07/2018 17:09:47</div><br/> <div class="toolFormTitle">script log output</div> script_error.log is empty<br/> <div class="toolFormTitle">Other log output</div> -/tmp/tmpG7m9zp/job_working_directory/000/1/dataset_2_files/script_runner.log is empty<br/> +/private/var/folders/df/6xqpqpcd7h73b6jpx9t6cwhw0000gn/T/tmptx2hJF/job_working_directory/000/1/dataset_2_files/script_runner.log is empty<br/> <div class="toolFormTitle">All output files available for downloading</div> <div><table class="colored" cellpadding="3" cellspacing="3"><tr><th>Output File Name (click to view)</th><th>Size</th></tr> <tr><td><a href="script.python">script.python</a></td><td>0 B</td></tr> <tr class="odd_row"><td><a href="script_error.log">script_error.log</a></td><td>0 B</td></tr> -<tr><td><a href="script_runner.log">script_runner.log</a></td><td>100 B</td></tr> +<tr><td><a href="script_runner.log">script_runner.log</a></td><td>140 B</td></tr> </table></div><br/> </div></body></html>
--- a/test.sh Sat Jul 09 17:00:06 2016 -0400 +++ b/test.sh Sun Jul 22 13:38:01 2018 -0400 @@ -1,7 +1,2 @@ #!/usr/bin/env bash -planemo test --galaxy_branch dev \ - --conda_auto_init \ - --conda_dependency_resolution \ - --conda_auto_install \ - --conda_ensure_channels scrapinghub \ - --tool_data_table tool_data_table_conf.xml.sample.test +planemo test --tool_data_table tool_data_table_conf.xml.sample.test "$@"
--- a/tool_dependencies.xml Sat Jul 09 17:00:06 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -<?xml version="1.0"?> -<tool_dependency> - <package name="docker-py" version="1.8.1"> - <install version="1.0"> - <actions> - <action type="setup_virtualenv">docker-py==1.8.1 - </action> - </actions> - </install> - </package> - <readme> - Only Admins can use this tool generator but please do NOT install o - </readme> -</tool_dependency>