# HG changeset patch
# User fubar
# Date 1618700314 0
# Node ID 290f552d7e05a4147d466c5377241135927911c1
# Parent 9fd3d83e1bacfe3444993aa01987e5d64fe3f13a
Uploaded
diff -r 9fd3d83e1bac -r 290f552d7e05 toolfactory/README.md
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/toolfactory/README.md Sat Apr 17 22:58:34 2021 +0000
@@ -0,0 +1,380 @@
+## Breaking news! Docker container at https://github.com/fubar2/toolfactory-galaxy-docker recommended as at December 2020
+
+### New demonstration of planemo tool_factory command 
+
+## This is the original ToolFactory suitable for non-docker situations. Please use the docker container if you can because it's integrated with a Toolshed...
+
+# WARNING
+
+Install this tool to a throw-away private Galaxy or Docker container ONLY!
+
+Please NEVER on a public or production instance where a hostile user may
+be able to gain access if they can acquire an administrative account login.
+
+It only runs for server administrators - the ToolFactory tool will refuse to execute for an ordinary user since
+it can install new tools to the Galaxy server it executes on! This is not something you should allow other than
+on a throw away instance that is protected from potentially hostile users.
+
+## Short Story
+
+Galaxy is easily extended to new applications by adding a new tool. Each new scientific computational package added as
+a tool to Galaxy requires an XML document describing how the application interacts with Galaxy.
+This is sometimes termed "wrapping" the package because the instructions tell Galaxy how to run the package
+as a new Galaxy tool. Any tool that has been wrapped is readily available to all the users through a consistent
+and easy to use interface once installed in the local Galaxy server.
+
+Most Galaxy tool wrappers have been manually prepared by skilled programmers, many using Planemo because it
+automates much of the boilerplate and makes the process much easier.
+The ToolFactory (TF) now uses Planemo under the hood for testing, but hides the command
+line complexities. The user will still need appropriate skills in terms of describing the interface between
+Galaxy and the new application, but will be helped by a Galaxy tool form to collect all the needed
+settings, together with automated testing and uploading to a toolshed with optional local installation.
+
+
+## ToolFactory generated tools are ordinary Galaxy tools
+
+A TF generated tool that passes the Planemo test is ready to publish in any Galaxy Toolshed and ready to install in any running Galaxy instance.
+They are fully workflow compatible and work exactly like any hand-written tool. The user can select input files of the specified type(s) from their
+history and edit each of the specified parameters. The tool form will show all the labels and help text supplied when the tool was built. When the tool
+is executed, the dependent binary or script will be passed all the i/o files and parameters as specified, and will write outputs to the specified new
+history datasets - just like any other Galaxy tool.
+
+## Models for tool command line construction
+
+The key to turning any software package into a Galaxy tool is the automated construction of a suitable command line.
+
+The TF can build a new tool that will allow the tool user to select input files from their history, set any parameters and when run will send the
+new output files to the history as specified when the tool builder completed the form and built the new tool.
+
+That tool can contain instructions to run any Conda dependency or a system executable like bash. Whether a bash script you have written or
+a Conda package like bwa, the executable will expect to find settings for input, output and parameters on a command line.
+
+These are often passed as "--name value" (argparse style) or in a fixed order (positional style).
+
+The ToolFactory allows either, or for "filter" applications that process input from STDIN and write processed output to STDOUT.
+
+The simplest tool model wraps a simple script or Conda dependency package requiring only input and output files, with no user supplied settings illustrated by
+the Tacrev demonstration tool found in the Galaxy running in the ToolFactory docker container. It passes a user selected input file from the current history on STDIN
+to a bash script. The bash script runs the unix tac utility (reverse cat) piped to the unix rev (reverse lines in a text file) utility. It's a one liner:
+
+`tac | rev`
+
+The tool building form allows zero or more Conda package name(s) and version(s) and an optional script to be executed by either a system
+executable like ``bash`` or the first of any named Conda dependency package/version. Tacrev uses a tiny bash script shown above and uses the system
+bash. Conda bash can be specified if it is important to use the same version consistently for the tool.
+
+On the tool form, the repeat section allowing zero or more input files was set to be a text file to be selected by the tool user and
+in the repeat section allowing one or more outputs, a new output file with special value `STDOUT` as the positional parameter, causes the TF to
+generate a command to capture STDOUT and send it to the new history file containing the reversed input text.
+
+By reversed, we mean really, truly reversed.
+
+That simple model can be made much more complicated, and can pass inputs and outputs as named or positional parameters,
+to allow more complicated scripts or dependent binaries that require:
+
+1. Any number of input data files selected by the user from existing history data
+2. Any number of output data files written to the user's history
+3. Any number of user supplied parameters. These can be passed as command line arguments to the script or the dependency package. Either
+positional or named (argparse) style command line parameter passing can be used.
+
+More complex models can be seen in the Sedtest, Pyrevpos and Pyrevargparse tools illustrating positional and argparse parameter passing.
+
+The most complex demonstration is the Planemo advanced tool tutorial BWA tool. There is one version using a command-override to implement
+exactly the same command structure in the Planemo tutorial. A second version uses a bash script and positional parameters to achieve the same
+result. Some tool builders may find the bash version more familiar and cleaner but the choice is yours.
+
+## Overview
+
+
+
+
+Steps in building a new Galaxy tool are all conducted through Galaxy running in the docker container:
+
+1. Login to the Galaxy running in the container at http://localhost:8080 using an admin account. They are specified in config/galaxy.yml and
+ in the documentation at
+ and the ToolFactory will error out and refuse to run for non-administrative tool builders as a minimal protection from opportunistic hostile use.
+
+2. Start the TF and fill in the form, providing sample inputs and parameter values to suit the Conda package being wrapped.
+
+3. Execute the tool to create a new XML tool wrapper using the sample inputs and parameter settings for the inbuilt tool test. Planemo runs twice.
+ firstly to generate the test outputs and then to perform a proper test. The completed toolshed archive is written to the history
+ together with the planemo test report. Optionally the new tool archive can be uploaded
+ to the toolshed running in the same container (http://localhost:9009) and then installed inside the Galaxy in the container for further testing.
+
+4. If the test fails, rerun the failed history job and correct errors on the tool form before rerunning until everything works correctly.
+
+
+
+## Planning and building new Galaxy tool wrappers.
+
+It is best to have all the required planning done to wrap any new script or binary before firing up the TF.
+Conda is the only current dependency manager supported. Before starting, at the very least, the tool builder will need
+to know the required software package name in Conda and the version to use, how the command line for
+the package must be constructed, and there must be sample inputs in the working history for each of the required data inputs
+for the package, together with values for every parameter to suit these sample inputs. These are required on the TF form
+for preparing the inbuilt tool test. That test is run using Planemo, as part of the tool generation process.
+
+A new tool is specified by filling in the usual Galaxy tool form.
+
+The form starts with a new tool name. Most tools will need dependency packages and versions
+for the executable. Only Conda is currently supported.
+
+If a script is needed, it can be pasted into a text box and the interpreter named. Available system executables
+can be used such as bash, or an interpreter such as python, perl or R can be nominated as conda dependencies
+to ensure reproducible analyses.
+
+The tool form will be generated from the input data and the tool builder supplied parameters. The command line for the
+executable is built using positional or argparse (named e.g. --input_file /foo/baz) style
+parameters and is completely dependent on the executable. These can include:
+
+1. Any number of input data sets needed by the executable. Each appears to the tool user on the run form and is included
+on the command line for the executable. The tool builder must supply a small representative sample for each one as
+an input for the automated tool test.
+
+2. Any number of output data sets generated by the package can be added to the command line and will appear in
+the user's history at the end of the job
+
+3. Any number of text or numeric parameters. Each will appear to the tool user on the run form and are included
+on the command line to the executable. The tool builder must supply a suitable representative value for each one as
+the value to be used for the automated tool test.
+
+Once the form is completed, executing the TF will build a new XML tool wrapper
+including a functional test based on the sample settings and data.
+
+If the Planemo test passes, the tool can be optionally uploaded to the local Galaxy used in the image for more testing.
+
+A local toolshed runs inside the container to allow an automated installation, although any toolshed and any accessible
+Galaxy can be specified for this process by editing the default URL and API keys to provide appropriate credentials.
+
+## Generated Tool Dependency management
+
+Conda is used for all dependency management although tools that use system utilities like sed, bash or awk
+may be available on job execution nodes. Sed and friends are available as Conda (conda-forge) dependencies if necessary.
+Versioned Conda dependencies are always baked-in to the tool and will be used for reproducible calculation.
+
+## Requirements
+
+These are all managed automagically. The TF relies on galaxyxml to generate tool xml and uses ephemeris and
+bioblend to load tools to the toolshed and to Galaxy. Planemo is used for testing and runs in a biocontainer currently at
+https://quay.io/fubar2/planemo-biocontainer
+
+## Caveats
+
+This docker image requires privileged mode so exposes potential security risks if hostile tool builders gain access.
+Please, do not run it in any situation where that is a problem - never, ever on a public facing Galaxy server.
+On a laptop or workstation should be fine in a non-hostile environment.
+
+
+## Example generated XML
+
+For the bwa-mem example, a supplied bash script is included as a configfile and so has escaped characters.
+```
+
+
+
+
+ Planemo advanced tool building sample bwa mem mapper as a ToolFactory demo
+
+ bwa
+ samtools
+
+
+ tempsam
+samtools view -Sb tempsam > temporary_bam_file.bam
+samtools sort -o "\$BAMOUT" temporary_bam_file.bam
+
+]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ tempsam
+ samtools view -Sb tempsam > temporary_bam_file.bam
+ samtools sort -o "$BAMOUT" temporary_bam_file.bam
+
+]]>
+
+
+```
+
+
+
+## More Explanation
+
+The TF is an unusual Galaxy tool, designed to allow a skilled user to make new Galaxy tools.
+It appears in Galaxy just like any other tool but outputs include new Galaxy tools generated
+using instructions provided by the user and the results of Planemo lint and tool testing using
+small sample inputs provided by the TF user. The small samples become tests built in to the new tool.
+
+It offers a familiar Galaxy form driven way to define how the user of the new tool will
+choose input data from their history, and what parameters the new tool user will be able to adjust.
+The TF user must know, or be able to read, enough about the tool to be able to define the details of
+the new Galaxy interface and the ToolFactory offers little guidance on that other than some examples.
+
+Tools always depend on other things. Most tools in Galaxy depend on third party
+scientific packages, so TF tools usually have one or more dependencies. These can be
+scientific packages such as BWA or scripting languages such as Python and are
+managed by Conda. If the new tool relies on a system utility such as bash or awk
+where the importance of version control on reproducibility is low, these can be used without
+Conda management - but remember the potential risks of unmanaged dependencies on computational
+reproducibility.
+
+The TF user can optionally supply a working script where scripting is
+required and the chosen dependency is a scripting language such as Python or a system
+scripting executable such as bash. Whatever the language, the script must correctly parse the command line
+arguments it receives at tool execution, as they are defined by the TF user. The
+text of that script is "baked in" to the new tool and will be executed each time
+the new tool is run. It is highly recommended that scripts and their command lines be developed
+and tested until proven to work before the TF is invoked. Galaxy as a software development
+environment is actually possible, but not recommended being somewhat clumsy and inefficient.
+
+Tools nearly always take one or more data sets from the user's history as input. TF tools
+allow the TF user to define what Galaxy datatypes the tool end user will be able to choose and what
+names or positions will be used to pass them on a command line to the package or script.
+
+Tools often have various parameter settings. The TF allows the TF user to define how each
+parameter will appear on the tool form to the end user, and what names or positions will be
+used to pass them on the command line to the package. At present, parameters are limited to
+simple text and number fields. Pull requests for other kinds of parameters that galaxyxml
+can handle are welcomed.
+
+Best practice Galaxy tools have one or more automated tests. These should use small sample data sets and
+specific parameter settings so when the tool is tested, the outputs can be compared with their expected
+values. The TF will automatically create a test for the new tool. It will use the sample data sets
+chosen by the TF user when they built the new tool.
+
+The TF works by exposing *unrestricted* and therefore extremely dangerous scripting
+to all designated administrators of the host Galaxy server, allowing them to
+run scripts in R, python, sh and perl. For this reason, a Docker container is
+available to help manage the associated risks.
+
+## Scripting uses
+
+To use a scripting language to create a new tool, you must first prepared and properly test a script. Use small sample
+data sets for testing. When the script is working correctly, upload the small sample datasets
+into a new history, start configuring a new ToolFactory tool, and paste the script into the script text box on the TF form.
+
+### Outputs
+
+The TF will generate the new tool described on the TF form, and test it
+using planemo. Optionally if a local toolshed is running, it can be used to
+install the new tool back into the generating Galaxy.
+
+A toolshed is built in to the Docker container and configured
+so a tool can be tested, sent to that toolshed, then installed in the Galaxy
+where the TF is running using the default toolshed and Galaxy URL and API keys.
+
+Once it's in a ToolShed, it can be installed into any local Galaxy server
+from the server administrative interface.
+
+Once the new tool is installed, local users can run it - each time, the
+package and/or script that was supplied when it was built will be executed with the input chosen
+from the user's history, together with user supplied parameters. In other words, the tools you generate with the
+TF run just like any other Galaxy tool.
+
+TF generated tools work as normal workflow components.
+
+
+## Limitations
+
+The TF is flexible enough to generate wrappers for many common scientific packages
+but the inbuilt automation will not cope with all possible situations. Users can
+supply overrides for two tool XML segments - tests and command and the BWA
+example in the supplied samples workflow illustrates their use. It does not deal with
+repeated elements or conditional parameters such as allowing a user to choose to see "simple"
+or "advanced" parameters (yet) and there will be plenty of packages it just
+won't cover - but it's a quick and efficient tool for the other 90% of cases. Perfect for
+that bash one liner you need to get that workflow functioning correctly for this
+afternoon's demonstration!
+
+## Installation
+
+The Docker container https://github.com/fubar2/toolfactory-galaxy-docker/blob/main/README.md
+is the best way to use the TF because it is preconfigured
+to automate new tool testing and has a built in local toolshed where each new tool
+is uploaded. If you grab the docker container, it should just work after a restart and you
+can run a workflow to generate all the sample tools. Running the samples and rerunning the ToolFactory
+jobs that generated them allows you to add fields and experiment to see how things work.
+
+It can be installed like any other tool from the Toolshed, but you will need to make some
+configuration changes (TODO write a configuration). You can install it most conveniently using the
+administrative "Search and browse tool sheds" link. Find the Galaxy Main
+toolshed at https://toolshed.g2.bx.psu.edu/ and search for the toolfactory
+repository in the Tool Maker section. Open it and review the code and select the option to install it.
+
+If not already there please add:
+
+```
+
+```
+
+to your local config/data_types_conf.xml.
+
+
+## Restricted execution
+
+The tool factory tool itself will ONLY run for admin users -
+people with IDs in config/galaxy.yml "admin_users".
+
+*ONLY admin_users can run this tool*
+
+That doesn't mean it's safe to install on a shared or exposed instance - please don't.
+
+## Generated tool Security
+
+Once you install a generated tool, it's just
+another tool - assuming the script is safe. They just run normally and their
+user cannot do anything unusually insecure but please, practice safe toolshed.
+Read the code before you install any tool. Especially this one - it is really scary.
+
+## Attribution
+
+Creating re-usable tools from scripts: The Galaxy Tool Factory
+Ross Lazarus; Antony Kaspi; Mark Ziemann; The Galaxy Team
+Bioinformatics 2012; doi: 10.1093/bioinformatics/bts573
+
+http://bioinformatics.oxfordjournals.org/cgi/reprint/bts573?ijkey=lczQh1sWrMwdYWJ&keytype=ref
+
diff -r 9fd3d83e1bac -r 290f552d7e05 toolfactory/rgToolFactory2.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/toolfactory/rgToolFactory2.py Sat Apr 17 22:58:34 2021 +0000
@@ -0,0 +1,1181 @@
+# replace with shebang for biocontainer
+# see https://github.com/fubar2/toolfactory
+#
+# copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
+#
+# all rights reserved
+# Licensed under the LGPL
+# suggestions for improvement and bug fixes welcome at
+# https://github.com/fubar2/toolfactory
+#
+# July 2020: BCC was fun and I feel like rip van winkle after 5 years.
+# Decided to
+# 1. Fix the toolfactory so it works - done for simplest case
+# 2. Fix planemo so the toolfactory function works
+# 3. Rewrite bits using galaxyxml functions where that makes sense - done
+
+import argparse
+import copy
+import json
+import logging
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import time
+
+from bioblend import ConnectionError
+from bioblend import toolshed
+
+import galaxyxml.tool as gxt
+import galaxyxml.tool.parameters as gxtp
+
+import lxml
+
+import yaml
+
+myversion = "V2.2 February 2021"
+verbose = True
+debug = True
+toolFactoryURL = "https://github.com/fubar2/toolfactory"
+foo = len(lxml.__version__)
+FAKEEXE = "~~~REMOVE~~~ME~~~"
+# need this until a PR/version bump to fix galaxyxml prepending the exe even
+# with override.
+
+
+def timenow():
+ """return current time as a string"""
+ return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
+
+
+cheetah_escape_table = {"$": "\\$", "#": "\\#"}
+
+
+def cheetah_escape(text):
+ """Produce entities within text."""
+ return "".join([cheetah_escape_table.get(c, c) for c in text])
+
+
+def parse_citations(citations_text):
+ """"""
+ citations = [c for c in citations_text.split("**ENTRY**") if c.strip()]
+ citation_tuples = []
+ for citation in citations:
+ if citation.startswith("doi"):
+ citation_tuples.append(("doi", citation[len("doi") :].strip()))
+ else:
+ citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
+ return citation_tuples
+
+
+class ScriptRunner:
+ """Wrapper for an arbitrary script
+ uses galaxyxml
+
+ """
+
+ def __init__(self, args=None): # noqa
+ """
+ prepare command line cl for running the tool here
+ and prepare elements needed for galaxyxml tool generation
+ """
+ self.ourcwd = os.getcwd()
+ self.collections = []
+ if len(args.collection) > 0:
+ try:
+ self.collections = [
+ json.loads(x) for x in args.collection if len(x.strip()) > 1
+ ]
+ except Exception:
+ print(
+ f"--collections parameter {str(args.collection)} is malformed - should be a dictionary"
+ )
+ try:
+ self.infiles = [
+ json.loads(x) for x in args.input_files if len(x.strip()) > 1
+ ]
+ except Exception:
+ print(
+ f"--input_files parameter {str(args.input_files)} is malformed - should be a dictionary"
+ )
+ try:
+ self.outfiles = [
+ json.loads(x) for x in args.output_files if len(x.strip()) > 1
+ ]
+ except Exception:
+ print(
+ f"--output_files parameter {args.output_files} is malformed - should be a dictionary"
+ )
+ try:
+ self.addpar = [
+ json.loads(x) for x in args.additional_parameters if len(x.strip()) > 1
+ ]
+ except Exception:
+ print(
+ f"--additional_parameters {args.additional_parameters} is malformed - should be a dictionary"
+ )
+ try:
+ self.selpar = [
+ json.loads(x) for x in args.selecttext_parameters if len(x.strip()) > 1
+ ]
+ except Exception:
+ print(
+ f"--selecttext_parameters {args.selecttext_parameters} is malformed - should be a dictionary"
+ )
+ self.args = args
+ self.cleanuppar()
+ self.lastclredirect = None
+ self.lastxclredirect = None
+ self.cl = []
+ self.xmlcl = []
+ self.is_positional = self.args.parampass == "positional"
+ if self.args.sysexe:
+ if ' ' in self.args.sysexe:
+ self.executeme = self.args.sysexe.split(' ')
+ else:
+ self.executeme = [self.args.sysexe, ]
+ else:
+ if self.args.packages:
+ self.executeme = [self.args.packages.split(",")[0].split(":")[0].strip(), ]
+ else:
+ self.executeme = None
+ aCL = self.cl.append
+ aXCL = self.xmlcl.append
+ assert args.parampass in [
+ "0",
+ "argparse",
+ "positional",
+ ], 'args.parampass must be "0","positional" or "argparse"'
+ self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name)
+ self.tool_id = self.tool_name
+ self.newtool = gxt.Tool(
+ self.tool_name,
+ self.tool_id,
+ self.args.tool_version,
+ self.args.tool_desc,
+ FAKEEXE,
+ )
+ self.newtarpath = "%s_toolshed.gz" % self.tool_name
+ self.tooloutdir = "./tfout"
+ self.repdir = "./TF_run_report_tempdir"
+ self.testdir = os.path.join(self.tooloutdir, "test-data")
+ if not os.path.exists(self.tooloutdir):
+ os.mkdir(self.tooloutdir)
+ if not os.path.exists(self.testdir):
+ os.mkdir(self.testdir)
+ if not os.path.exists(self.repdir):
+ os.mkdir(self.repdir)
+ self.tinputs = gxtp.Inputs()
+ self.toutputs = gxtp.Outputs()
+ self.testparam = []
+ if self.args.script_path:
+ self.prepScript()
+ if self.args.command_override:
+ scos = open(self.args.command_override, "r").readlines()
+ self.command_override = [x.rstrip() for x in scos]
+ else:
+ self.command_override = None
+ if self.args.test_override:
+ stos = open(self.args.test_override, "r").readlines()
+ self.test_override = [x.rstrip() for x in stos]
+ else:
+ self.test_override = None
+ if self.args.script_path:
+ for ex in self.executeme:
+ aCL(ex)
+ aXCL(ex)
+ aCL(self.sfile)
+ aXCL("$runme")
+ else:
+ for ex in self.executeme:
+ aCL(ex)
+ aXCL(ex)
+
+ self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name)
+ self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name)
+ if self.args.parampass == "0":
+ self.clsimple()
+ else:
+ if self.args.parampass == "positional":
+ self.prepclpos()
+ self.clpositional()
+ else:
+ self.prepargp()
+ self.clargparse()
+ if self.args.cl_suffix: # DIY CL end
+ clp = shlex.split(self.args.cl_suffix)
+ for c in clp:
+ aCL(c)
+ aXCL(c)
+
+ def clsimple(self):
+ """no parameters or repeats - uses < and > for i/o"""
+ aCL = self.cl.append
+ aXCL = self.xmlcl.append
+ if len(self.infiles) > 0:
+ aCL("<")
+ aCL(self.infiles[0]["infilename"])
+ aXCL("<")
+ aXCL("$%s" % self.infiles[0]["infilename"])
+ if len(self.outfiles) > 0:
+ aCL(">")
+ aCL(self.outfiles[0]["name"])
+ aXCL(">")
+ aXCL("$%s" % self.outfiles[0]["name"])
+
+ def prepargp(self):
+ clsuffix = []
+ xclsuffix = []
+ for i, p in enumerate(self.infiles):
+ nam = p["infilename"]
+ if p["origCL"].strip().upper() == "STDIN":
+ appendme = [
+ nam,
+ nam,
+ "< %s" % nam,
+ ]
+ xappendme = [
+ nam,
+ nam,
+ "< $%s" % nam,
+ ]
+ else:
+ rep = p["repeat"] == "1"
+ over = ""
+ if rep:
+ over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for'
+ appendme = [p["CL"], p["CL"], ""]
+ xappendme = [p["CL"], "$%s" % p["CL"], over]
+ clsuffix.append(appendme)
+ xclsuffix.append(xappendme)
+ for i, p in enumerate(self.outfiles):
+ if p["origCL"].strip().upper() == "STDOUT":
+ self.lastclredirect = [">", p["name"]]
+ self.lastxclredirect = [">", "$%s" % p["name"]]
+ else:
+ clsuffix.append([p["name"], p["name"], ""])
+ xclsuffix.append([p["name"], "$%s" % p["name"], ""])
+ for p in self.addpar:
+ nam = p["name"]
+ rep = p["repeat"] == "1"
+ if rep:
+ over = f'#for $rep in $R_{nam}:\n--{nam} "$rep.{nam}"\n#end for'
+ else:
+ over = p["override"]
+ clsuffix.append([p["CL"], nam, over])
+ xclsuffix.append([p["CL"], nam, over])
+ for p in self.selpar:
+ clsuffix.append([p["CL"], p["name"], p["override"]])
+ xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]])
+ self.xclsuffix = xclsuffix
+ self.clsuffix = clsuffix
+
+ def prepclpos(self):
+ clsuffix = []
+ xclsuffix = []
+ for i, p in enumerate(self.infiles):
+ if p["origCL"].strip().upper() == "STDIN":
+ appendme = [
+ "999",
+ p["infilename"],
+ "< $%s" % p["infilename"],
+ ]
+ xappendme = [
+ "999",
+ p["infilename"],
+ "< $%s" % p["infilename"],
+ ]
+ else:
+ appendme = [p["CL"], p["infilename"], ""]
+ xappendme = [p["CL"], "$%s" % p["infilename"], ""]
+ clsuffix.append(appendme)
+ xclsuffix.append(xappendme)
+ for i, p in enumerate(self.outfiles):
+ if p["origCL"].strip().upper() == "STDOUT":
+ self.lastclredirect = [">", p["name"]]
+ self.lastxclredirect = [">", "$%s" % p["name"]]
+ else:
+ clsuffix.append([p["CL"], p["name"], ""])
+ xclsuffix.append([p["CL"], "$%s" % p["name"], ""])
+ for p in self.addpar:
+ nam = p["name"]
+ rep = p["repeat"] == "1" # repeats make NO sense
+ if rep:
+ print(f'### warning. Repeats for {nam} ignored - not permitted in positional parameter command lines!')
+ over = p["override"]
+ clsuffix.append([p["CL"], nam, over])
+ xclsuffix.append([p["CL"], '"$%s"' % nam, over])
+ for p in self.selpar:
+ clsuffix.append([p["CL"], p["name"], p["override"]])
+ xclsuffix.append([p["CL"], '"$%s"' % p["name"], p["override"]])
+ clsuffix.sort()
+ xclsuffix.sort()
+ self.xclsuffix = xclsuffix
+ self.clsuffix = clsuffix
+
+ def prepScript(self):
+ rx = open(self.args.script_path, "r").readlines()
+ rx = [x.rstrip() for x in rx]
+ rxcheck = [x.strip() for x in rx if x.strip() > ""]
+ assert len(rxcheck) > 0, "Supplied script is empty. Cannot run"
+ self.script = "\n".join(rx)
+ fhandle, self.sfile = tempfile.mkstemp(
+ prefix=self.tool_name, suffix="_%s" % (self.executeme[0])
+ )
+ tscript = open(self.sfile, "w")
+ tscript.write(self.script)
+ tscript.close()
+ self.escapedScript = [cheetah_escape(x) for x in rx]
+ self.spacedScript = [f" {x}" for x in rx if x.strip() > ""]
+ art = "%s.%s" % (self.tool_name, self.executeme[0])
+ artifact = open(art, "wb")
+ artifact.write(bytes("\n".join(self.escapedScript), "utf8"))
+ artifact.close()
+
+ def cleanuppar(self):
+ """ positional parameters are complicated by their numeric ordinal"""
+ if self.args.parampass == "positional":
+ for i, p in enumerate(self.infiles):
+ assert (
+ p["CL"].isdigit() or p["CL"].strip().upper() == "STDIN"
+ ), "Positional parameters must be ordinal integers - got %s for %s" % (
+ p["CL"],
+ p["label"],
+ )
+ for i, p in enumerate(self.outfiles):
+ assert (
+ p["CL"].isdigit() or p["CL"].strip().upper() == "STDOUT"
+ ), "Positional parameters must be ordinal integers - got %s for %s" % (
+ p["CL"],
+ p["name"],
+ )
+ for i, p in enumerate(self.addpar):
+ assert p[
+ "CL"
+ ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
+ p["CL"],
+ p["name"],
+ )
+ for i, p in enumerate(self.infiles):
+ infp = copy.copy(p)
+ infp["origCL"] = infp["CL"]
+ if self.args.parampass in ["positional", "0"]:
+ infp["infilename"] = infp["label"].replace(" ", "_")
+ else:
+ infp["infilename"] = infp["CL"]
+ self.infiles[i] = infp
+ for i, p in enumerate(self.outfiles):
+ p["origCL"] = p["CL"] # keep copy
+ self.outfiles[i] = p
+ for i, p in enumerate(self.addpar):
+ p["origCL"] = p["CL"]
+ self.addpar[i] = p
+
+ def clpositional(self):
+ # inputs in order then params
+ aCL = self.cl.append
+ for (k, v, koverride) in self.clsuffix:
+ if " " in v:
+ aCL("%s" % v)
+ else:
+ aCL(v)
+ aXCL = self.xmlcl.append
+ for (k, v, koverride) in self.xclsuffix:
+ aXCL(v)
+ if self.lastxclredirect:
+ aXCL(self.lastxclredirect[0])
+ aXCL(self.lastxclredirect[1])
+
+ def clargparse(self):
+ """argparse style"""
+ aCL = self.cl.append
+ aXCL = self.xmlcl.append
+ # inputs then params in argparse named form
+
+ for (k, v, koverride) in self.xclsuffix:
+ if koverride > "":
+ k = koverride
+ aXCL(k)
+ else:
+ if len(k.strip()) == 1:
+ k = "-%s" % k
+ else:
+ k = "--%s" % k
+ aXCL(k)
+ aXCL(v)
+ for (k, v, koverride) in self.clsuffix:
+ if koverride > "":
+ k = koverride
+ elif len(k.strip()) == 1:
+ k = "-%s" % k
+ else:
+ k = "--%s" % k
+ aCL(k)
+ aCL(v)
+ if self.lastxclredirect:
+ aXCL(self.lastxclredirect[0])
+ aXCL(self.lastxclredirect[1])
+
+ def getNdash(self, newname):
+ if self.is_positional:
+ ndash = 0
+ else:
+ ndash = 2
+ if len(newname) < 2:
+ ndash = 1
+ return ndash
+
+ def doXMLparam(self):
+ """Add all needed elements to tool""" # noqa
+ for p in self.outfiles:
+ newname = p["name"]
+ newfmt = p["format"]
+ newcl = p["CL"]
+ test = p["test"]
+ oldcl = p["origCL"]
+ test = test.strip()
+ ndash = self.getNdash(newcl)
+ aparm = gxtp.OutputData(
+ name=newname, format=newfmt, num_dashes=ndash, label=newname
+ )
+ aparm.positional = self.is_positional
+ if self.is_positional:
+ if oldcl.upper() == "STDOUT":
+ aparm.positional = 9999999
+ aparm.command_line_override = "> $%s" % newname
+ else:
+ aparm.positional = int(oldcl)
+ aparm.command_line_override = "$%s" % newname
+ self.toutputs.append(aparm)
+ ld = None
+ if test.strip() > "":
+ if test.startswith("diff"):
+ c = "diff"
+ ld = 0
+ if test.split(":")[1].isdigit:
+ ld = int(test.split(":")[1])
+ tp = gxtp.TestOutput(
+ name=newname,
+ value="%s_sample" % newname,
+ compare=c,
+ lines_diff=ld,
+ )
+ elif test.startswith("sim_size"):
+ c = "sim_size"
+ tn = test.split(":")[1].strip()
+ if tn > "":
+ if "." in tn:
+ delta = None
+ delta_frac = min(1.0, float(tn))
+ else:
+ delta = int(tn)
+ delta_frac = None
+ tp = gxtp.TestOutput(
+ name=newname,
+ value="%s_sample" % newname,
+ compare=c,
+ delta=delta,
+ delta_frac=delta_frac,
+ )
+ else:
+ c = test
+ tp = gxtp.TestOutput(
+ name=newname,
+ value="%s_sample" % newname,
+ compare=c,
+ )
+ self.testparam.append(tp)
+ for p in self.infiles:
+ newname = p["infilename"]
+ newfmt = p["format"]
+ ndash = self.getNdash(newname)
+ reps = p.get("repeat", "0") == "1"
+ if not len(p["label"]) > 0:
+ alab = p["CL"]
+ else:
+ alab = p["label"]
+ aninput = gxtp.DataParam(
+ newname,
+ optional=False,
+ label=alab,
+ help=p["help"],
+ format=newfmt,
+ multiple=False,
+ num_dashes=ndash,
+ )
+ aninput.positional = self.is_positional
+ if self.is_positional:
+ if p["origCL"].upper() == "STDIN":
+ aninput.positional = 9999998
+ aninput.command_line_override = "> $%s" % newname
+ else:
+ aninput.positional = int(p["origCL"])
+ aninput.command_line_override = "$%s" % newname
+ if reps:
+ repe = gxtp.Repeat(name=f"R_{newname}", title=f"Add as many {alab} as needed")
+ repe.append(aninput)
+ self.tinputs.append(repe)
+ tparm = gxtp.TestRepeat(name=f"R_{newname}")
+ tparm2 = gxtp.TestParam(newname, value="%s_sample" % newname)
+ tparm.append(tparm2)
+ self.testparam.append(tparm)
+ else:
+ self.tinputs.append(aninput)
+ tparm = gxtp.TestParam(newname, value="%s_sample" % newname)
+ self.testparam.append(tparm)
+ for p in self.addpar:
+ newname = p["name"]
+ newval = p["value"]
+ newlabel = p["label"]
+ newhelp = p["help"]
+ newtype = p["type"]
+ newcl = p["CL"]
+ oldcl = p["origCL"]
+ reps = p["repeat"] == "1"
+ if not len(newlabel) > 0:
+ newlabel = newname
+ ndash = self.getNdash(newname)
+ if newtype == "text":
+ aparm = gxtp.TextParam(
+ newname,
+ label=newlabel,
+ help=newhelp,
+ value=newval,
+ num_dashes=ndash,
+ )
+ elif newtype == "integer":
+ aparm = gxtp.IntegerParam(
+ newname,
+ label=newlabel,
+ help=newhelp,
+ value=newval,
+ num_dashes=ndash,
+ )
+ elif newtype == "float":
+ aparm = gxtp.FloatParam(
+ newname,
+ label=newlabel,
+ help=newhelp,
+ value=newval,
+ num_dashes=ndash,
+ )
+ elif newtype == "boolean":
+ aparm = gxtp.BooleanParam(
+ newname,
+ label=newlabel,
+ help=newhelp,
+ value=newval,
+ num_dashes=ndash,
+ )
+ else:
+ raise ValueError(
+ 'Unrecognised parameter type "%s" for\
+ additional parameter %s in makeXML'
+ % (newtype, newname)
+ )
+ aparm.positional = self.is_positional
+ if self.is_positional:
+ aparm.positional = int(oldcl)
+ if reps:
+ repe = gxtp.Repeat(name=f"R_{newname}", title=f"Add as many {newlabel} as needed")
+ repe.append(aparm)
+ self.tinputs.append(repe)
+ tparm = gxtp.TestRepeat(name=f"R_{newname}")
+ tparm2 = gxtp.TestParam(newname, value=newval)
+ tparm.append(tparm2)
+ self.testparam.append(tparm)
+ else:
+ self.tinputs.append(aparm)
+ tparm = gxtp.TestParam(newname, value=newval)
+ self.testparam.append(tparm)
+ for p in self.selpar:
+ newname = p["name"]
+ newval = p["value"]
+ newlabel = p["label"]
+ newhelp = p["help"]
+ newtype = p["type"]
+ newcl = p["CL"]
+ if not len(newlabel) > 0:
+ newlabel = newname
+ ndash = self.getNdash(newname)
+ if newtype == "selecttext":
+ newtext = p["texts"]
+ aparm = gxtp.SelectParam(
+ newname,
+ label=newlabel,
+ help=newhelp,
+ num_dashes=ndash,
+ )
+ for i in range(len(newval)):
+ anopt = gxtp.SelectOption(
+ value=newval[i],
+ text=newtext[i],
+ )
+ aparm.append(anopt)
+ aparm.positional = self.is_positional
+ if self.is_positional:
+ aparm.positional = int(newcl)
+ self.tinputs.append(aparm)
+ tparm = gxtp.TestParam(newname, value=newval)
+ self.testparam.append(tparm)
+ else:
+ raise ValueError(
+ 'Unrecognised parameter type "%s" for\
+ selecttext parameter %s in makeXML'
+ % (newtype, newname)
+ )
+ for p in self.collections:
+ newkind = p["kind"]
+ newname = p["name"]
+ newlabel = p["label"]
+ newdisc = p["discover"]
+ collect = gxtp.OutputCollection(newname, label=newlabel, type=newkind)
+ disc = gxtp.DiscoverDatasets(
+ pattern=newdisc, directory=f"{newname}", visible="false"
+ )
+ collect.append(disc)
+ self.toutputs.append(collect)
+ try:
+ tparm = gxtp.TestOutputCollection(newname) # broken until PR merged.
+ self.testparam.append(tparm)
+ except Exception:
+ print("#### WARNING: Galaxyxml version does not have the PR merged yet - tests for collections must be over-ridden until then!")
+
+ def doNoXMLparam(self):
+ """filter style package - stdin to stdout"""
+ if len(self.infiles) > 0:
+ alab = self.infiles[0]["label"]
+ if len(alab) == 0:
+ alab = self.infiles[0]["infilename"]
+ max1s = (
+ "Maximum one input if parampass is 0 but multiple input files supplied - %s"
+ % str(self.infiles)
+ )
+ assert len(self.infiles) == 1, max1s
+ newname = self.infiles[0]["infilename"]
+ aninput = gxtp.DataParam(
+ newname,
+ optional=False,
+ label=alab,
+ help=self.infiles[0]["help"],
+ format=self.infiles[0]["format"],
+ multiple=False,
+ num_dashes=0,
+ )
+ aninput.command_line_override = "< $%s" % newname
+ aninput.positional = True
+ self.tinputs.append(aninput)
+ tp = gxtp.TestParam(name=newname, value="%s_sample" % newname)
+ self.testparam.append(tp)
+ if len(self.outfiles) > 0:
+ newname = self.outfiles[0]["name"]
+ newfmt = self.outfiles[0]["format"]
+ anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0)
+ anout.command_line_override = "> $%s" % newname
+ anout.positional = self.is_positional
+ self.toutputs.append(anout)
+ tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname)
+ self.testparam.append(tp)
+
+ def makeXML(self): # noqa
+ """
+ Create a Galaxy xml tool wrapper for the new script
+ Uses galaxyhtml
+ Hmmm. How to get the command line into correct order...
+ """
+ if self.command_override:
+ self.newtool.command_override = self.command_override # config file
+ else:
+ self.newtool.command_override = self.xmlcl
+ cite = gxtp.Citations()
+ acite = gxtp.Citation(type="doi", value="10.1093/bioinformatics/bts573")
+ cite.append(acite)
+ self.newtool.citations = cite
+ safertext = ""
+ if self.args.help_text:
+ helptext = open(self.args.help_text, "r").readlines()
+ safertext = "\n".join([cheetah_escape(x) for x in helptext])
+ if len(safertext.strip()) == 0:
+ safertext = (
+ "Ask the tool author (%s) to rebuild with help text please\n"
+ % (self.args.user_email)
+ )
+ if self.args.script_path:
+ if len(safertext) > 0:
+ safertext = safertext + "\n\n------\n" # transition allowed!
+ scr = [x for x in self.spacedScript if x.strip() > ""]
+ scr.insert(0, "\n\nScript::\n")
+ if len(scr) > 300:
+ scr = (
+ scr[:100]
+ + [" >300 lines - stuff deleted", " ......"]
+ + scr[-100:]
+ )
+ scr.append("\n")
+ safertext = safertext + "\n".join(scr)
+ self.newtool.help = safertext
+ self.newtool.version_command = f'echo "{self.args.tool_version}"'
+ std = gxtp.Stdios()
+ std1 = gxtp.Stdio()
+ std.append(std1)
+ self.newtool.stdios = std
+ requirements = gxtp.Requirements()
+ if self.args.packages:
+ for d in self.args.packages.split(","):
+ ver = ""
+ d = d.replace("==", ":")
+ d = d.replace("=", ":")
+ if ":" in d:
+ packg, ver = d.split(":")
+ else:
+ packg = d
+ requirements.append(
+ gxtp.Requirement("package", packg.strip(), ver.strip())
+ )
+ self.newtool.requirements = requirements
+ if self.args.parampass == "0":
+ self.doNoXMLparam()
+ else:
+ self.doXMLparam()
+ self.newtool.outputs = self.toutputs
+ self.newtool.inputs = self.tinputs
+ if self.args.script_path:
+ configfiles = gxtp.Configfiles()
+ configfiles.append(
+ gxtp.Configfile(name="runme", text="\n".join(self.escapedScript))
+ )
+ self.newtool.configfiles = configfiles
+ tests = gxtp.Tests()
+ test_a = gxtp.Test()
+ for tp in self.testparam:
+ test_a.append(tp)
+ tests.append(test_a)
+ self.newtool.tests = tests
+ self.newtool.add_comment(
+ "Created by %s at %s using the Galaxy Tool Factory."
+ % (self.args.user_email, timenow())
+ )
+ self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL))
+ exml0 = self.newtool.export()
+ exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted
+ if (
+ self.test_override
+ ): # cannot do this inside galaxyxml as it expects lxml objects for tests
+ part1 = exml.split("")[0]
+ part2 = exml.split("")[1]
+ fixed = "%s\n%s\n%s" % (part1, "\n".join(self.test_override), part2)
+ exml = fixed
+ # exml = exml.replace('range="1:"', 'range="1000:"')
+ xf = open("%s.xml" % self.tool_name, "w")
+ xf.write(exml)
+ xf.write("\n")
+ xf.close()
+ # ready for the tarball
+
+ def run(self):
+ """
+ generate test outputs by running a command line
+ won't work if command or test override in play - planemo is the
+ easiest way to generate test outputs for that case so is
+ automagically selected
+ """
+ scl = " ".join(self.cl)
+ err = None
+ if self.args.parampass != "0":
+ if os.path.exists(self.elog):
+ ste = open(self.elog, "a")
+ else:
+ ste = open(self.elog, "w")
+ if self.lastclredirect:
+ sto = open(self.lastclredirect[1], "wb") # is name of an output file
+ else:
+ if os.path.exists(self.tlog):
+ sto = open(self.tlog, "a")
+ else:
+ sto = open(self.tlog, "w")
+ sto.write(
+ "## Executing Toolfactory generated command line = %s\n" % scl
+ )
+ sto.flush()
+ subp = subprocess.run(
+ self.cl, shell=False, stdout=sto, stderr=ste
+ )
+ sto.close()
+ ste.close()
+ retval = subp.returncode
+ else: # work around special case - stdin and write to stdout
+ if len(self.infiles) > 0:
+ sti = open(self.infiles[0]["name"], "rb")
+ else:
+ sti = sys.stdin
+ if len(self.outfiles) > 0:
+ sto = open(self.outfiles[0]["name"], "wb")
+ else:
+ sto = sys.stdout
+ subp = subprocess.run(
+ self.cl, shell=False, stdout=sto, stdin=sti
+ )
+ sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
+ retval = subp.returncode
+ sto.close()
+ sti.close()
+ if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
+ os.unlink(self.tlog)
+ if os.path.isfile(self.elog) and os.stat(self.elog).st_size == 0:
+ os.unlink(self.elog)
+ if retval != 0 and err: # problem
+ sys.stderr.write(err)
+ logging.debug("run done")
+ return retval
+
+ def shedLoad(self):
+ """
+ use bioblend to create new repository
+ or update existing
+
+ """
+ if os.path.exists(self.tlog):
+ sto = open(self.tlog, "a")
+ else:
+ sto = open(self.tlog, "w")
+
+ ts = toolshed.ToolShedInstance(
+ url=self.args.toolshed_url,
+ key=self.args.toolshed_api_key,
+ verify=False,
+ )
+ repos = ts.repositories.get_repositories()
+ rnames = [x.get("name", "?") for x in repos]
+ rids = [x.get("id", "?") for x in repos]
+ tfcat = "ToolFactory generated tools"
+ if self.tool_name not in rnames:
+ tscat = ts.categories.get_categories()
+ cnames = [x.get("name", "?").strip() for x in tscat]
+ cids = [x.get("id", "?") for x in tscat]
+ catID = None
+ if tfcat.strip() in cnames:
+ ci = cnames.index(tfcat)
+ catID = cids[ci]
+ res = ts.repositories.create_repository(
+ name=self.args.tool_name,
+ synopsis="Synopsis:%s" % self.args.tool_desc,
+ description=self.args.tool_desc,
+ type="unrestricted",
+ remote_repository_url=self.args.toolshed_url,
+ homepage_url=None,
+ category_ids=catID,
+ )
+ tid = res.get("id", None)
+ sto.write(f"#create_repository {self.args.tool_name} tid={tid} res={res}\n")
+ else:
+ i = rnames.index(self.tool_name)
+ tid = rids[i]
+ try:
+ res = ts.repositories.update_repository(
+ id=tid, tar_ball_path=self.newtarpath, commit_message=None
+ )
+ sto.write(f"#update res id {id} ={res}\n")
+ except ConnectionError:
+ sto.write(
+ "####### Is the toolshed running and the API key correct? Bioblend shed upload failed\n"
+ )
+ sto.close()
+
+ def eph_galaxy_load(self):
+ """
+ use ephemeris to load the new tool from the local toolshed after planemo uploads it
+ """
+ if os.path.exists(self.tlog):
+ tout = open(self.tlog, "a")
+ else:
+ tout = open(self.tlog, "w")
+ cll = [
+ "shed-tools",
+ "install",
+ "-g",
+ self.args.galaxy_url,
+ "--latest",
+ "-a",
+ self.args.galaxy_api_key,
+ "--name",
+ self.tool_name,
+ "--owner",
+ "fubar",
+ "--toolshed",
+ self.args.toolshed_url,
+ "--section_label",
+ "ToolFactory",
+ ]
+ tout.write("running\n%s\n" % " ".join(cll))
+ subp = subprocess.run(
+ cll,
+ cwd=self.ourcwd,
+ shell=False,
+ stderr=tout,
+ stdout=tout,
+ )
+ tout.write(
+ "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
+ )
+ tout.close()
+ return subp.returncode
+
+ def writeShedyml(self):
+ """for planemo"""
+ yuser = self.args.user_email.split("@")[0]
+ yfname = os.path.join(self.tooloutdir, ".shed.yml")
+ yamlf = open(yfname, "w")
+ odict = {
+ "name": self.tool_name,
+ "owner": yuser,
+ "type": "unrestricted",
+ "description": self.args.tool_desc,
+ "synopsis": self.args.tool_desc,
+ "category": "TF Generated Tools",
+ }
+ yaml.dump(odict, yamlf, allow_unicode=True)
+ yamlf.close()
+
+ def makeTool(self):
+ """write xmls and input samples into place"""
+ if self.args.parampass == 0:
+ self.doNoXMLparam()
+ else:
+ self.makeXML()
+ if self.args.script_path:
+ stname = os.path.join(self.tooloutdir, self.sfile)
+ if not os.path.exists(stname):
+ shutil.copyfile(self.sfile, stname)
+ xreal = "%s.xml" % self.tool_name
+ xout = os.path.join(self.tooloutdir, xreal)
+ shutil.copyfile(xreal, xout)
+ for p in self.infiles:
+ pth = p["name"]
+ dest = os.path.join(self.testdir, "%s_sample" % p["infilename"])
+ shutil.copyfile(pth, dest)
+ dest = os.path.join(self.repdir, "%s_sample" % p["infilename"])
+ shutil.copyfile(pth, dest)
+
+ def makeToolTar(self, report_fail=False):
+ """move outputs into test-data and prepare the tarball"""
+ excludeme = "_planemo_test_report.html"
+
+ def exclude_function(tarinfo):
+ filename = tarinfo.name
+ return None if filename.endswith(excludeme) else tarinfo
+
+ if os.path.exists(self.tlog):
+ tout = open(self.tlog, "a")
+ else:
+ tout = open(self.tlog, "w")
+ for p in self.outfiles:
+ oname = p["name"]
+ tdest = os.path.join(self.testdir, "%s_sample" % oname)
+ src = os.path.join(self.testdir, oname)
+ if not os.path.isfile(tdest):
+ if os.path.isfile(src):
+ shutil.copyfile(src, tdest)
+ dest = os.path.join(self.repdir, "%s.sample" % (oname))
+ shutil.copyfile(src, dest)
+ else:
+ if report_fail:
+ tout.write(
+ "###Tool may have failed - output file %s not found in testdir after planemo run %s."
+ % (tdest, self.testdir)
+ )
+ tf = tarfile.open(self.newtarpath, "w:gz")
+ tf.add(
+ name=self.tooloutdir,
+ arcname=self.tool_name,
+ filter=exclude_function,
+ )
+ tf.close()
+ shutil.copyfile(self.newtarpath, self.args.new_tool)
+
+ def moveRunOutputs(self):
+ """need to move planemo or run outputs into toolfactory collection"""
+ with os.scandir(self.tooloutdir) as outs:
+ for entry in outs:
+ if not entry.is_file():
+ continue
+ if "." in entry.name:
+ _, ext = os.path.splitext(entry.name)
+ if ext in [".tgz", ".json"]:
+ continue
+ if ext in [".yml", ".xml", ".yaml"]:
+ newname = f"{entry.name.replace('.','_')}.txt"
+ else:
+ newname = entry.name
+ else:
+ newname = f"{entry.name}.txt"
+ dest = os.path.join(self.repdir, newname)
+ src = os.path.join(self.tooloutdir, entry.name)
+ shutil.copyfile(src, dest)
+ if self.args.include_tests:
+ with os.scandir(self.testdir) as outs:
+ for entry in outs:
+ if (not entry.is_file()) or entry.name.endswith(
+ "_planemo_test_report.html"
+ ):
+ continue
+ if "." in entry.name:
+ _, ext = os.path.splitext(entry.name)
+ if ext in [".tgz", ".json"]:
+ continue
+ if ext in [".yml", ".xml", ".yaml"]:
+ newname = f"{entry.name.replace('.','_')}.txt"
+ else:
+ newname = entry.name
+ else:
+ newname = f"{entry.name}.txt"
+ dest = os.path.join(self.repdir, newname)
+ src = os.path.join(self.testdir, entry.name)
+ shutil.copyfile(src, dest)
+
+ def planemo_test_once(self):
+ """planemo is a requirement so is available for testing but needs a
+ different call if in the biocontainer - see above
+ and for generating test outputs if command or test overrides are
+ supplied test outputs are sent to repdir for display
+ """
+ xreal = "%s.xml" % self.tool_name
+ tool_test_path = os.path.join(
+ self.repdir, f"{self.tool_name}_planemo_test_report.html"
+ )
+ if os.path.exists(self.tlog):
+ tout = open(self.tlog, "a")
+ else:
+ tout = open(self.tlog, "w")
+ cll = [
+ "planemo",
+ "test",
+ "--galaxy_python_version",
+ self.args.python_version,
+ "--conda_auto_init",
+ "--test_data",
+ os.path.abspath(self.testdir),
+ "--test_output",
+ os.path.abspath(tool_test_path),
+ "--galaxy_root",
+ self.args.galaxy_root,
+ "--update_test_data",
+ os.path.abspath(xreal),
+ ]
+ p = subprocess.run(
+ cll,
+ shell=False,
+ cwd=self.tooloutdir,
+ stderr=tout,
+ stdout=tout,
+ )
+ tout.close()
+ return p.returncode
+
+ def set_planemo_galaxy_root(self, galaxyroot='/galaxy-central', config_path=".planemo.yml"):
+ # bug in planemo - bogus '--dev-wheels' passed to run_tests.sh as at april 2021 - need a fiddled copy so it is ignored until fixed
+ CONFIG_TEMPLATE = """## Planemo Global Configuration File.
+## Everything in this file is completely optional - these values can all be
+## configured via command line options for the corresponding commands.
+
+## Specify a default galaxy_root for test and server commands here.
+galaxy_root: %s
+## Username used with toolshed(s).
+#shed_username: ""
+sheds:
+ # For each tool shed you wish to target, uncomment key or both email and
+ # password.
+ toolshed:
+ #key: ""
+ #email: ""
+ #password: ""
+ testtoolshed:
+ #key: ""
+ #email: ""
+ #password: ""
+ local:
+ #key: ""
+ #email: ""
+ #password: ""
+"""
+ if not os.path.exists(config_path):
+ with open(config_path, "w") as f:
+ f.write(CONFIG_TEMPLATE % galaxyroot)
+
+
+def main():
+ """
+ This is a Galaxy wrapper.
+ It expects to be called by a special purpose tool.xml
+
+ """
+ parser = argparse.ArgumentParser()
+ a = parser.add_argument
+ a("--script_path", default=None)
+ a("--history_test", default=None)
+ a("--cl_suffix", default=None)
+ a("--sysexe", default=None)
+ a("--packages", default=None)
+ a("--tool_name", default="newtool")
+ a("--tool_dir", default=None)
+ a("--input_files", default=[], action="append")
+ a("--output_files", default=[], action="append")
+ a("--user_email", default="Unknown")
+ a("--bad_user", default=None)
+ a("--make_Tool", default="runonly")
+ a("--help_text", default=None)
+ a("--tool_desc", default=None)
+ a("--tool_version", default=None)
+ a("--citations", default=None)
+ a("--command_override", default=None)
+ a("--test_override", default=None)
+ a("--additional_parameters", action="append", default=[])
+ a("--selecttext_parameters", action="append", default=[])
+ a("--edit_additional_parameters", action="store_true", default=False)
+ a("--parampass", default="positional")
+ a("--tfout", default="./tfout")
+ a("--new_tool", default="new_tool")
+ a("--galaxy_url", default="http://localhost:8080")
+ a("--toolshed_url", default="http://localhost:9009")
+ # make sure this is identical to tool_sheds_conf.xml
+ # localhost != 127.0.0.1 so validation fails
+ a("--toolshed_api_key", default="fakekey")
+ a("--galaxy_api_key", default="fakekey")
+ a("--galaxy_root", default="/galaxy-central")
+ a("--galaxy_venv", default="/galaxy_venv")
+ a("--collection", action="append", default=[])
+ a("--include_tests", default=False, action="store_true")
+ a("--python_version", default="3.9")
+ args = parser.parse_args()
+ assert not args.bad_user, (
+ 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \
+admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file'
+ % (args.bad_user, args.bad_user)
+ )
+ assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
+ assert (
+ args.sysexe or args.packages
+ ), "## Tool Factory wrapper expects an interpreter \
+or an executable package in --sysexe or --packages"
+ r = ScriptRunner(args)
+ r.writeShedyml()
+ r.makeTool()
+ if args.make_Tool == "generate":
+ r.run()
+ r.moveRunOutputs()
+ r.makeToolTar()
+ else:
+ r.planemo_test_once()
+ r.moveRunOutputs()
+ r.makeToolTar(report_fail=True)
+ if args.make_Tool == "gentestinstall":
+ r.shedLoad()
+ r.eph_galaxy_load()
+
+
+if __name__ == "__main__":
+ main()