# HG changeset patch # User fubar # Date 1598584261 14400 # Node ID 35a912ce0c83b80d2a2cf73a94cc30eb6e460ce3 # Parent 5a7a5b06bce0159ccc3f6c94a138321054893fde Can now make the bwa example from planemo :) diff -r 5a7a5b06bce0 -r 35a912ce0c83 toolfactory/README.md --- a/toolfactory/README.md Sun Aug 23 21:03:48 2020 -0400 +++ b/toolfactory/README.md Thu Aug 27 23:11:01 2020 -0400 @@ -1,137 +1,164 @@ -*WARNING before you start* +**Breaking news! Docker container is recommended as at August 2020** + +A Docker container can be built - see the docker directory. +It is highly recommended for isolation. It also has an integrated toolshed to allow installation of new tools back +into the Galaxy being used to generate them. + +Built from quay.io/bgruening/galaxy:20.05 but updates the +Galaxy code to the dev branch - it seems to work fine with updated bioblend>=0.14 +with planemo and the right version of gxformat2 needed by the ToolFactory (TF). - Install this tool on a private Galaxy ONLY - Please NEVER on a public or production instance - -Updated august 2014 by John Chilton adding citation support +The runclean.sh script run from the docker subdirectory of your local clone of this repository +should create a container (eventually) and serve it at localhost:8080 with a toolshed at +localhost:9009. -Updated august 8 2014 to fix bugs reported by Marius van den Beek +Once it's up, please restart Galaxy in the container with +```docker exec [container name] supervisorctl restart galaxy: ``` +Jobs just do not seem to run properly otherwise and the next steps won't work! -Please cite the resource at -http://bioinformatics.oxfordjournals.org/cgi/reprint/bts573?ijkey=lczQh1sWrMwdYWJ&keytype=ref -if you use this tool in your published work. +The generated container includes a workflow and 2 sample data sets for the workflow -**Short Story** +Load the workflow. Adjust the inputs for each as labelled. The perl example counts GC in phiX.fasta. +The python scripts use the rgToolFactory.py as their input - any text file will work but I like the +recursion. The BWA example has some mitochondrial reads and reference. Run the workflow and watch. +This should fill the history with some sample tools you can rerun and play with. +Note that each new tool will have been tested using Planemo. In the workflow, in Galaxy. +Extremely cool to watch. + +*WARNING* + + Install this tool on a throw-away private Galaxy or Docker container ONLY + Please NEVER on a public or production instance -This is an unusual Galaxy tool capable of generating new Galaxy tools. -It works by exposing *unrestricted* and therefore extremely dangerous scripting -to all designated administrators of the host Galaxy server, allowing them to -run scripts in R, python, sh and perl over multiple selected input data sets, -writing a single new data set as output. +*Short Story* + +Galaxy is easily extended to new applications by adding a new tool. Each new scientific computational package added as +a tool to Galaxy requires some special instructions to be written. This is sometimes termed "wrapping" the package +because the instructions tell Galaxy how to run the package as a new Galaxy tool. Any tool in a Galaxy is +readily available to all the users through a consistent and easy to use interface. -*You have a working r/python/perl/bash script or any executable with positional or argparse style parameters* +Most Galaxy tool wrappers have been manually prepared by skilled programmers, many using Planemo because it +automates much of the basic boilerplate and makes the process much easier. The ToolFactory (TF) +uses Planemo under the hood for many functions, but hides the command +line complexities from the TF user. -It can be turned into an ordinary Galaxy tool in minutes, using a Galaxy tool. - +*More Explanation* -**Automated generation of new Galaxy tools for installation into any Galaxy** +The TF is an unusual Galaxy tool, designed to allow a skilled user to make new Galaxy tools. +It appears in Galaxy just like any other tool but outputs include new Galaxy tools generated +using instructions provided by the user and the results of Planemo lint and tool testing using +small sample inputs provided by the TF user. The small samples become tests built in to the new tool. + +It offers a familiar Galaxy form driven way to define how the user of the new tool will +choose input data from their history, and what parameters the new tool user will be able to adjust. +The TF user must know, or be able to read, enough about the tool to be able to define the details of +the new Galaxy interface and the ToolFactory offers little guidance on that other than some examples. -A test is generated using small sample test data inputs and parameter settings you supply. -Once the test case outputs have been produced, they can be used to build a -new Galaxy tool. The supplied script or executable is baked as a requirement -into a new, ordinary Galaxy tool, fully workflow compatible out of the box. -Generated tools are installed via a tool shed by an administrator -and work exactly like all other Galaxy tools for your users. +Tools always depend on other things. Most tools in Galaxy depend on third party +scientific packages, so TF tools usually have one or more dependencies. These can be +scientific packages such as BWA or scripting languages such as Python and are +usually managed by Conda. If the new tool relies on a system utility such as bash or awk +where the importance of version control on reproducibility is low, these can be used without +Conda management - but remember the potential risks of unmanaged dependencies on computational +reproducibility. -**More Detail** +The TF user can optionally supply a working script where scripting is +required and the chosen dependency is a scripting language such as Python or a system +scripting executable such as bash. Whatever the language, the script must correctly parse the command line +arguments it receives at tool execution, as they are defined by the TF user. The +text of that script is "baked in" to the new tool and will be executed each time +the new tool is run. It is highly recommended that scripts and their command lines be developed +and tested until proven to work before the TF is invoked. Galaxy as a software development +environment is actually possible, but not recommended being somewhat clumsy and inefficient. -To use the ToolFactory, you should have prepared a script to paste into a -text box, or have a package in mind and a small test input example ready to select from your history -to test your new script. +Tools nearly always take one or more data sets from the user's history as input. TF tools +allow the TF user to define what Galaxy datatypes the tool end user will be able to choose and what +names or positions will be used to pass them on a command line to the package or script. -```planemo test rgToolFactory2.xml --galaxy_root ~/galaxy --test_data ~/galaxy/tools/tool_makers/toolfactory/test-data``` works for me +Tools often have various parameter settings. The TF allows the TF user to define how each +parameter will appear on the tool form to the end user, and what names or positions will be +used to pass them on the command line to the package. At present, parameters are limited to +simple text and number fields. Pull requests for other kinds of parameters that galaxyxml +can handle are welcomed. -There is an example in each scripting language on the Tool Factory form. You -can just cut and paste these to try it out - remember to select the right -interpreter please. You'll also need to create a small test data set using -the Galaxy history add new data tool. +Best practice Galaxy tools have one or more automated tests. These should use small sample data sets and +specific parameter settings so when the tool is tested, the outputs can be compared with their expected +values. The TF will automatically create a test for the new tool. It will use the sample data sets +chosen by the TF user when they built the new tool. -If the script fails somehow, use the "redo" button on the tool output in -your history to recreate the form complete with broken script. Fix the bug -and execute again. Rinse, wash, repeat. +The TF works by exposing *unrestricted* and therefore extremely dangerous scripting +to all designated administrators of the host Galaxy server, allowing them to +run scripts in R, python, sh and perl. For this reason, a Docker container is +available to help manage the associated risks. + +*Scripting uses* + +To use a scripting language to create a new tool, you must first prepared and properly test a script. Use small sample +data sets for testing. When the script is working correctly, upload the small sample datasets +into a new history, start configuring a new ToolFactory tool, and paste the script into the script text box on the TF form. + +*Outputs* Once the script runs sucessfully, a new Galaxy tool that runs your script can be generated. Select the "generate" option and supply some help text and names. The new tool will be generated in the form of a new Galaxy datatype -*toolshed.gz* - as the name suggests, it's an archive ready to upload to a +*tgz* - as the name suggests, it's an archive ready to upload to a Galaxy ToolShed as a new tool repository. +It is also possible to run a tool to generate test outputs, then test it +using planemo. A toolshed is built in to the Docker container and configured +so a tool can be tested, sent to that toolshed, then installed in the Galaxy +where the TF is running. + +If the tool requires a command or test XML override, then planemo is +needed to generate test outputs to make a complete tool, rerun to test +and if required upload to the local toolshed and install in the Galaxy +where the TF is running. + Once it's in a ToolShed, it can be installed into any local Galaxy server from the server administrative interface. -Once the new tool is installed, local users can run it - each time, the script -that was supplied when it was built will be executed with the input chosen -from the user's history. In other words, the tools you generate with the -ToolFactory run just like any other Galaxy tool,but run your script every time. +Once the new tool is installed, local users can run it - each time, the +package and/or script that was supplied when it was built will be executed with the input chosen +from the user's history, together with user supplied parameters. In other words, the tools you generate with the +ToolFactory run just like any other Galaxy tool. -Tool factory tools are perfect for workflow components. One input, one output, -no variables. +TF generated tools work as normal workflow components. + + +*Limitations* -*To fully and safely exploit the awesome power* of this tool, -Galaxy and the ToolShed, you should be a developer installing this -tool on a private/personal/scratch local instance where you are an -admin_user. Then, if you break it, you get to keep all the pieces see -https://bitbucket.org/fubar/galaxytoolfactory/wiki/Home +The TF is flexible enough to generate wrappers for many common scientific packages +but the inbuilt automation will not cope with all possible situations. Users can +supply overrides for two tool XML segments - tests and command and the BWA +example in the supplied samples workflow illustrates their use. -**Installation** -This is a Galaxy tool. You can install it most conveniently using the +*Installation* + +The Docker container is the best way to use the TF because it is preconfigured +to automate new tool testing and has a built in local toolshed where each new tool +is uploaded. It is easy to install without Docker, but you will need to make some +configuration changes (TODO write a configuration). You can install it most conveniently using the administrative "Search and browse tool sheds" link. Find the Galaxy Main toolshed at https://toolshed.g2.bx.psu.edu/ and search for the toolfactory -repository. Open it and review the code and select the option to install it. - -If you can't get the tool that way, the xml and py files here need to be -copied into a new tools -subdirectory such as tools/toolfactory Your tool_conf.xml needs a new entry -pointing to the xml -file - something like:: +repository in the Tool Maker section. Open it and review the code and select the option to install it. -
- -
- -If not already there, +Otherwise, if not already there pending an accepted PR, please add: - to your local data_types_conf.xml. -**Restricted execution** +*Restricted execution* The tool factory tool itself will then be usable ONLY by admin users - -people with IDs in admin_users in universe_wsgi.ini **Yes, that's right. ONLY +people with IDs in admin_users. **Yes, that's right. ONLY admin_users can run this tool** Think about it for a moment. If allowed to run any arbitrary script on your Galaxy server, the only thing that would impede a miscreant bent on destroying all your Galaxy data would probably be lack of appropriate technical skills. -**What it does** - -This is a tool factory for simple scripts in python, R and -perl currently. Functional tests are automatically generated. How cool is that. - -LIMITED to simple scripts that read one input from the history. Optionally can -write one new history dataset, and optionally collect any number of outputs -into links on an autogenerated HTML index page for the user to navigate - -useful if the script writes images and output files - pdf outputs are shown -as thumbnails and R's bloated pdf's are shrunk with ghostscript so that and -imagemagik need to be available. - -Generated tools can be edited and enhanced like any Galaxy tool, so start -small and build up since a generated script gets you a serious leg up to a -more complex one. - -**What you do** - -You paste and run your script, you fix the syntax errors and -eventually it runs. You can use the redo button and edit the script before -trying to rerun it as you debug - it works pretty well. - -Once the script works on some test data, you can generate a toolshed compatible -gzip file containing your script ready to run as an ordinary Galaxy tool in -a repository on your local toolshed. That means safe and largely automated -installation in any production Galaxy configured to use your toolshed. - **Generated tool Security** Once you install a generated tool, it's just @@ -141,7 +168,7 @@ **Send Code** -Patches and suggestions welcome as bitbucket issues please? +Pull requests and suggestions welcome as git issues please? **Attribution** @@ -160,7 +187,3 @@ Licensed under the LGPL -**Obligatory screenshot** - -http://bitbucket.org/fubar/galaxytoolmaker/src/fda8032fe989/images/dynamicScriptTool.png - diff -r 5a7a5b06bce0 -r 35a912ce0c83 toolfactory/rgToolFactory2.py --- a/toolfactory/rgToolFactory2.py Sun Aug 23 21:03:48 2020 -0400 +++ b/toolfactory/rgToolFactory2.py Thu Aug 27 23:11:01 2020 -0400 @@ -6,7 +6,8 @@ # # all rights reserved # Licensed under the LGPL -# suggestions for improvement and bug fixes welcome at https://github.com/fubar2/toolfactory +# suggestions for improvement and bug fixes welcome at +# https://github.com/fubar2/toolfactory # # July 2020: BCC was fun and I feel like rip van winkle after 5 years. # Decided to @@ -46,6 +47,10 @@ debug = True toolFactoryURL = "https://github.com/fubar2/toolfactory" ourdelim = "~~~" +ALOT = 10000000 # srsly. command or test overrides use read() so just in case +STDIOXML = """ + +""" # --input_files="$input_files~~~$CL~~~$input_formats~~~$input_label # ~~~$input_help" @@ -56,11 +61,13 @@ IHELPOS = 4 IOCLPOS = 5 -# --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL +# --output_files "$otab.history_name~~~$otab.history_format~~~$otab.CL~~~otab.history_test ONAMEPOS = 0 OFMTPOS = 1 OCLPOS = 2 -OOCLPOS = 3 +OTESTPOS = 3 +OOCLPOS = 4 + # --additional_parameters="$i.param_name~~~$i.param_value~~~ # $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride" @@ -76,6 +83,10 @@ foo = len(lxml.__version__) # fug you, flake8. Say my name! +FAKEEXE = "~~~REMOVE~~~ME~~~" +# need this until a PR/version bump to fix galaxyxml prepending the exe even +# with override. + def timenow(): """return current time as a string @@ -120,9 +131,7 @@ if citation.startswith("doi"): citation_tuples.append(("doi", citation[len("doi") :].strip())) else: - citation_tuples.append( - ("bibtex", citation[len("bibtex") :].strip()) - ) + citation_tuples.append(("bibtex", citation[len("bibtex") :].strip())) return citation_tuples @@ -137,7 +146,6 @@ prepare command line cl for running the tool here and prepare elements needed for galaxyxml tool generation """ - self.infiles = [x.split(ourdelim) for x in args.input_files] self.outfiles = [x.split(ourdelim) for x in args.output_files] self.addpar = [x.split(ourdelim) for x in args.additional_parameters] @@ -148,28 +156,31 @@ self.cl = [] self.xmlcl = [] self.is_positional = self.args.parampass == "positional" + if self.args.packages: + self.executeme = self.args.packages.split(",")[0].split(":")[0] + else: + self.executeme = self.args.sysexe + assert ( + self.executeme is not None + ), "No system or managed executable passed in. Cannot build" aCL = self.cl.append + aXCL = self.xmlcl.append assert args.parampass in [ "0", "argparse", "positional", - ], 'Parameter passing in args.parampass must be "0","positional" or "argparse"' + ], 'args.parampass must be "0","positional" or "argparse"' self.tool_name = re.sub("[^a-zA-Z0-9_]+", "", args.tool_name) self.tool_id = self.tool_name - if self.args.interpreter_name: - exe = "$runMe" - else: - exe = self.args.exe_package - assert ( - exe is not None - ), "No interpeter or executable passed in - nothing to run so cannot build" self.tool = gxt.Tool( self.args.tool_name, self.tool_id, self.args.tool_version, self.args.tool_desc, - exe, + FAKEEXE, ) + if self.args.script_path: + self.tool.interpreter = self.executeme self.tooloutdir = "tfout" self.repdir = "TF_run_report_tempdir" self.testdir = os.path.join(self.tooloutdir, "test-data") @@ -182,21 +193,42 @@ self.tinputs = gxtp.Inputs() self.toutputs = gxtp.Outputs() self.testparam = [] - if ( - self.args.runmode == "Executable" or self.args.runmode == "system" - ): - if len(self.args.cl_override) > 0: - for x in self.args.cl_override.split(' '): - aCL(x) + if self.args.script_path: + self.prepScript() + if self.args.command_override: + scos = open(self.args.command_override, "r").readlines() + self.command_override = [x.rstrip() for x in scos] + else: + self.command_override = None + if self.args.test_override: + stos = open(self.args.test_override, "r").readlines() + self.test_override = [x.rstrip() for x in stos] + else: + self.test_override = None + if self.args.cl_prefix: # DIY CL start + clp = self.args.cl_prefix.split(" ") + for c in clp: + aCL(c) + aXCL(c) + else: + if self.args.runmode == "Executable": + if self.args.script_path: + aCL(self.executeme) + aCL(self.sfile) + aXCL("$runme") + else: + aCL(self.executeme) # this little CL will just run + aXCL(self.executeme) else: - aCL(self.args.exe_package) # this little CL will just run - else: - self.prepScript() - aCL(self.args.interpreter_name) - aCL(self.sfile) - - self.elog = "%s_error_log.txt" % self.tool_name - self.tlog = "%s_runner_log.txt" % self.tool_name + if self.args.script_path: + aCL(self.executeme) + aCL(self.sfile) + aXCL("$runme") + else: + aCL(self.executeme) # this little CL will just run + aXCL(self.executeme) + self.elog = os.path.join(self.repdir,"%s_error_log.txt" % self.tool_name) + self.tlog = os.path.join(self.repdir,"%s_runner_log.txt" % self.tool_name) if self.args.parampass == "0": self.clsimple() @@ -222,20 +254,15 @@ xappendme = [p[IOCLPOS], p[ICLPOS], "$%s" % p[ICLPOS], ""] clsuffix.append(appendme) xclsuffix.append(xappendme) - # print('##infile i=%d, appendme=%s' % (i,appendme)) for i, p in enumerate(self.outfiles): if p[OOCLPOS] == "STDOUT": self.lastclredirect = [">", p[ONAMEPOS]] self.lastxclredirect = [">", "$%s" % p[OCLPOS]] else: clsuffix.append([p[OOCLPOS], p[OCLPOS], p[ONAMEPOS], ""]) - xclsuffix.append( - [p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""] - ) + xclsuffix.append([p[OOCLPOS], p[OCLPOS], "$%s" % p[ONAMEPOS], ""]) for p in self.addpar: - clsuffix.append( - [p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]] - ) + clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]]) xclsuffix.append( [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]] ) @@ -255,21 +282,15 @@ assert len(rxcheck) > 0, "Supplied script is empty. Cannot run" self.script = "\n".join(rx) fhandle, self.sfile = tempfile.mkstemp( - prefix=self.tool_name, suffix="_%s" % (self.args.interpreter_name) + prefix=self.tool_name, suffix="_%s" % (self.executeme) ) tscript = open(self.sfile, "w") tscript.write(self.script) tscript.close() - self.indentedScript = " %s" % "\n".join( - [" %s" % html_escape(x) for x in rx] - ) - self.escapedScript = "%s" % "\n".join( - [" %s" % html_escape(x) for x in rx] - ) - art = "%s.%s" % (self.tool_name, self.args.interpreter_name) + self.indentedScript = " %s" % "\n".join([" %s" % html_escape(x) for x in rx]) + self.escapedScript = "%s" % "\n".join([" %s" % html_escape(x) for x in rx]) + art = "%s.%s" % (self.tool_name, self.executeme) artifact = open(art, "wb") - if self.args.interpreter_name == "python": - artifact.write(bytes("#!/usr/bin/env python\n", "utf8")) artifact.write(bytes(self.script, "utf8")) artifact.close() @@ -377,7 +398,7 @@ def doXMLparam(self): """flake8 made me do this...""" for p in self.outfiles: - newname, newfmt, newcl, oldcl = p + newname, newfmt, newcl, test, oldcl = p ndash = self.getNdash(newcl) aparm = gxtp.OutputData(newcl, format=newfmt, num_dashes=ndash) aparm.positional = self.is_positional @@ -389,9 +410,17 @@ aparm.positional = int(oldcl) aparm.command_line_override = "$%s" % newcl self.toutputs.append(aparm) - tp = gxtp.TestOutput( - name=newcl, value="%s_sample" % newcl, format=newfmt - ) + usetest = None + ld = None + if test > '': + if test.startswith('diff'): + usetest = 'diff' + if test.split(':')[1].isdigit: + ld = int(test.split(':')[1]) + else: + usetest = test + tp = gxtp.TestOutput(name=newcl, value="%s_sample" % newcl, format=newfmt, + compare=usetest, lines_diff=ld, delta=None,) self.testparam.append(tp) for p in self.infiles: newname = p[ICLPOS] @@ -457,11 +486,12 @@ self.testparam.append(tparm) def doNoXMLparam(self): + """filter style package - stdin to stdout""" alab = self.infiles[0][ILABPOS] if len(alab) == 0: alab = self.infiles[0][ICLPOS] max1s = ( - "Maximum one input if parampass is 0 - more than one input files supplied - %s" + "Maximum one input if parampass is 0 but multiple input files supplied - %s" % str(self.infiles) ) assert len(self.infiles) == 1, max1s @@ -486,9 +516,7 @@ anout.command_line_override = "> $%s" % newname anout.positional = self.is_positional self.toutputs.append(anout) - tp = gxtp.TestOutput( - name=newname, value="%s_sample" % newname, format=newfmt - ) + tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname, format=newfmt) self.testparam.append(tp) def makeXML(self): @@ -497,12 +525,12 @@ Uses galaxyhtml Hmmm. How to get the command line into correct order... """ - if self.args.cl_override: - self.tool.command_line_override = self.args.cl_override.split(' ') + self.xmlcl + if self.command_override: + self.tool.command_line_override = self.command_override # config file else: self.tool.command_line_override = self.xmlcl - if self.args.interpreter_name: - self.tool.interpreter = self.args.interpreter_name + # if self.args.interpreter_name: + # self.tool.interpreter = self.args.interpreter_name if self.args.help_text: helptext = open(self.args.help_text, "r").readlines() helptext = [html_escape(x) for x in helptext] @@ -515,38 +543,14 @@ ) self.tool.version_command = None # do not want requirements = gxtp.Requirements() - - if self.args.interpreter_name: - if self.args.dependencies: - for d in self.args.dependencies.split(','): - requirements.append( - gxtp.Requirement( - "package", d, "" - ) - ) - if self.args.interpreter_name == "python": - requirements.append( - gxtp.Requirement( - "package", "python", self.args.interpreter_version - ) - ) - elif self.args.interpreter_name not in ["bash", "sh"]: - requirements.append( - gxtp.Requirement( - "package", - self.args.interpreter_name, - self.args.interpreter_version, - ) - ) - else: - if self.args.exe_package and self.args.parampass != "system": - requirements.append( - gxtp.Requirement( - "package", - self.args.exe_package, - self.args.exe_package_version, - ) - ) + if self.args.packages: + for d in self.args.packages.split(","): + if ":" in d: + packg, ver = d.split(":") + else: + packg = d + ver = "" + requirements.append(gxtp.Requirement("package", packg.strip(), ver.strip())) self.tool.requirements = requirements if self.args.parampass == "0": self.doNoXMLparam() @@ -554,9 +558,11 @@ self.doXMLparam() self.tool.outputs = self.toutputs self.tool.inputs = self.tinputs - if self.args.runmode not in ["Executable", "system"]: + if ( + self.args.script_path + ): configfiles = gxtp.Configfiles() - configfiles.append(gxtp.Configfile(name="runMe", text=self.script)) + configfiles.append(gxtp.Configfile(name="runme", text=self.script)) self.tool.configfiles = configfiles tests = gxtp.Tests() test_a = gxtp.Test() @@ -573,75 +579,69 @@ "Cite: Creating re-usable tools from scripts doi: \ 10.1093/bioinformatics/bts573" ) - exml = self.tool.export() - xf = open('%s.xml' % self.tool_name, "w") + exml0 = self.tool.export() + exml = exml0.replace(FAKEEXE, "") # temporary work around until PR accepted + if self.test_override: # cannot do this inside galaxyxml as it expects lxml objects for tests + part1 = exml.split('')[0] + part2 = exml.split('')[1] + fixed = '%s\n%s\n%s' % (part1,self.test_override,part2) + exml = fixed + xf = open("%s.xml" % self.tool_name, "w") xf.write(exml) xf.write("\n") xf.close() # ready for the tarball - def makeTooltar(self): - """ - a tool is a gz tarball with eg - /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ... - NOTE names for test inputs and outputs are munged here so must - correspond to actual input and output names used on the generated cl + def makeTool(self): + """write xmls and samples into place """ - retval = self.run() - if retval: - sys.stderr.write( - "## Run failed. Cannot build yet. Please fix and retry" - ) - sys.exit(1) self.makeXML() + if self.args.script_path: + stname = os.path.join(self.tooloutdir, "%s" % (self.sfile)) + if not os.path.exists(stname): + shutil.copyfile(self.sfile, stname) + xreal = "%s.xml" % self.tool_name + xout = os.path.join(self.tooloutdir, xreal) + shutil.copyfile(xreal, xout) for p in self.infiles: pth = p[IPATHPOS] dest = os.path.join(self.testdir, "%s_sample" % p[ICLPOS]) shutil.copyfile(pth, dest) - for p in self.outfiles: - pth = p[OCLPOS] - if p[OOCLPOS] == "STDOUT" or self.args.parampass == "0": - pth = p[ONAMEPOS] - dest = os.path.join(self.testdir, "%s_sample" % p[ONAMEPOS]) - shutil.copyfile(pth, dest) - dest = os.path.join(self.tooloutdir, p[ONAMEPOS]) - shutil.copyfile(pth, dest) - else: - pth = p[OCLPOS] - dest = os.path.join(self.testdir, "%s_sample" % p[OCLPOS]) - shutil.copyfile(pth, dest) - dest = os.path.join(self.tooloutdir, p[OCLPOS]) - shutil.copyfile(pth, dest) - if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0: - shutil.copyfile(self.tlog, os.path.join(self.tooloutdir, "test1_log_outfiletxt")) - if self.args.runmode not in ["Executable", "system"]: - stname = os.path.join(self.tooloutdir, "%s" % (self.sfile)) - if not os.path.exists(stname): - shutil.copyfile(self.sfile, stname) - xreal = '%s.xml' % self.tool_name - xout = os.path.join(self.tooloutdir,xreal) - shutil.copyfile(xreal, xout) + + def makeToolTar(self): self.newtarpath = "toolfactory_%s.tgz" % self.tool_name tf = tarfile.open(self.newtarpath, "w:gz") tf.add(name=self.tooloutdir, arcname=self.tool_name) tf.close() shutil.copyfile(self.newtarpath, self.args.new_tool) - shutil.copyfile(xreal,"tool_xml.txt") - repoutnames = [x[OCLPOS] for x in self.outfiles] - with os.scandir('.') as outs: + if os.path.exists(self.tlog) and os.stat(self.tlog).st_size > 0: + shutil.copyfile( + self.tlog, os.path.join(self.tooloutdir, "test1_log_outfiletxt") + ) + + def moveRunOutputs(self): + """need to move files into toolfactory collection after any run - planemo or not + """ + for p in self.outfiles: + naym = p[ONAMEPOS] + src = os.path.join(self.tooloutdir,naym) + if os.path.isfile(src): + dest = os.path.join(self.testdir, "%s_sample" % naym) + shutil.copyfile(naym, dest) + else: + print('### problem - output file %s not found in tooloutdir %s' % (src,self.tooloutdir)) + with os.scandir(self.tooloutdir) as outs: for entry in outs: - if entry.name.endswith('.tgz') or not entry.is_file(): + if not entry.is_file() or entry.name.startswith('.'): continue - if entry.name in repoutnames: - if '.' in entry.name: - ofne = os.path.splitext(entry.name)[1] - else: - ofne = '.txt' - ofn = '%s%s' % (entry.name.replace('.','_'),ofne) - shutil.copyfile(entry.name,os.path.join(self.repdir,ofn)) - elif entry.name == "%s.xml" % self.tool_name: - shutil.copyfile(entry.name,os.path.join(self.repdir,"new_tool_xml.xml")) - return retval + if "." in entry.name: + nayme,ext = os.path.splitext(entry.name) + else: + ext = ".txt" + ofn = "%s%s" % (entry.name.replace(".", "_"), ext) + dest = os.path.join(self.repdir, ofn) + src = os.path.join(self.tooloutdir,entry.name) + shutil.copyfile(src, dest) def run(self): """ @@ -655,15 +655,12 @@ if self.args.parampass != "0": ste = open(self.elog, "wb") if self.lastclredirect: - sto = open( - self.lastclredirect[1], "wb" - ) # is name of an output file + sto = open(self.lastclredirect[1], "wb") # is name of an output file else: sto = open(self.tlog, "wb") sto.write( bytes( - "## Executing Toolfactory generated command line = %s\n" - % scl, + "## Executing Toolfactory generated command line = %s\n" % scl, "utf8", ) ) @@ -683,7 +680,7 @@ pass tmp_stderr.close() retval = p.returncode - else: # work around special case of simple scripts that take stdin and write to stdout + else: # work around special case - stdin and write to stdout sti = open(self.infiles[0][IPATHPOS], "rb") sto = open(self.outfiles[0][ONAMEPOS], "wb") # must use shell to redirect @@ -700,142 +697,222 @@ logging.debug("run done") return retval - def install_load(self): - testres = self.planemo_test() - if testres == 0: - self.planemo_shedload() - self,eph_galaxy_load() - else: - stderr.write('Planemo test failed - tool %s was not installed' % self.args.tool_name) - - def planemo_shedload(self): """ planemo shed_create --shed_target testtoolshed planemo shed_update --check_diff --shed_target testtoolshed """ - cll = ['planemo', 'shed_create', '--shed_target','local'] - p = subprocess.run(cll, shell=False,cwd=self.tooloutdir ) + if os.path.exists(self.tlog): + tout = open(self.tlog,'a') + else: + tout = open(self.tlog,'w') + cll = ["planemo", "shed_create", "--shed_target", "local"] + try: + p = subprocess.run(cll, shell=False, cwd=self.tooloutdir, stdout=tout, stderr = tout) + except: + pass if p.returncode != 0: - print('Repository %s exists' % self.args.tool_name) + print("Repository %s exists" % self.args.tool_name) else: - print('initiated %s' % self.args.tool_name) - cll = ['planemo', 'shed_upload', '--shed_target','local','-r','--owner','fubar', - '--name',self.args.tool_name,'--shed_key',self.args.toolshed_api_key,'--tar',self.newtarpath] - p = subprocess.run(cll, shell=False,cwd=self.tooloutdir) - print('Ran',' '.join(cll),'got',p.returncode) + print("initiated %s" % self.args.tool_name) + cll = [ + "planemo", + "shed_upload", + "--shed_target", + "local", + "--owner", + "fubar", + "--name", + self.args.tool_name, + "--shed_key", + self.args.toolshed_api_key, + "--tar", + self.newtarpath, + ] + print("Run", " ".join(cll)) + p = subprocess.run(cll, shell=False) + print("Ran", " ".join(cll), "got", p.returncode) + tout.close() return p.returncode - - def planemo_test(self): + def planemo_test(self, genoutputs=True): """planemo is a requirement so is available """ - cll = ['planemo', 'test', '--galaxy_root', self.args.galaxy_root, - self.args.tool_name] - p = subprocess.run(cll, shell=False) - ols = os.listdir('.') - for fn in ols: - if '.' in fn: - ofne = os.path.splitext(fn)[1] - else: - ofne = '.txt' - ofn = '%s%s' % (fn.replace('.','_'),ofne) - shutil.copyfile(fn,os.path.join(self.repdir,ofn)) + xreal = "%s.xml" % self.tool_name + if os.path.exists(self.tlog): + tout = open(self.tlog,'a') + else: + tout = open(self.tlog,'w') + if genoutputs: + cll = [ + "planemo", + "test", + "--galaxy_root", + self.args.galaxy_root, + "--update_test_data", + xreal, + ] + else: + cll = ["planemo", "test", "--galaxy_root", self.args.galaxy_root, xreal] + try: + p = subprocess.run(cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout) + except: + pass + tout.close() return p.returncode - def eph_galaxy_load(self): """ """ - cll = ['shed-tools', 'install', '-g', self.args.galaxy_url, '--latest', - '-a', self.args.galaxy_api_key , '--name', self.args.tool_name, '--owner','fubar', - '--toolshed', self.args.toolshed_url, - '--section_label','Generated Tools','--install_tool_dependencies',] - print('running\n',' '.join(cll)) - p = subprocess.run(cll, shell=False) + if os.path.exists(self.tlog): + tout = open(self.tlog,'a') + else: + tout = open(self.tlog,'w') + cll = [ + "shed-tools", + "install", + "-g", + self.args.galaxy_url, + "--latest", + "-a", + self.args.galaxy_api_key, + "--name", + self.args.tool_name, + "--owner", + "fubar", + "--toolshed", + self.args.toolshed_url, + "--section_label", + "Generated Tools", + "--install_tool_dependencies", + ] + print("running\n", " ".join(cll)) + p = subprocess.run(cll, shell=False, stderr=tout, stdout=tout) if p.returncode != 0: - print('Repository %s installation returned %d' % (self.args.tool_name,p.retcode)) + print( + "Repository %s installation returned %d" + % (self.args.tool_name, p.returncode) + ) else: - print('installed %s' % self.args.tool_name) + print("installed %s" % self.args.tool_name) + tout.close() return p.returncode def writeShedyml(self): - yuser = self.args.user_email.split('@')[0] - yfname = os.path.join(self.tooloutdir,'.shed.yml') - yamlf = open(yfname, 'w') - odict = {'name':self.tool_name,'owner':yuser,'type':'unrestricted','description':self.args.tool_desc} + yuser = self.args.user_email.split("@")[0] + yfname = os.path.join(self.tooloutdir, ".shed.yml") + yamlf = open(yfname, "w") + odict = { + "name": self.tool_name, + "owner": yuser, + "type": "unrestricted", + "description": self.args.tool_desc, + } yaml.dump(odict, yamlf, allow_unicode=True) yamlf.close() + + def install_load(self): + _ = self.planemo_test(genoutputs=True) + testres = self.planemo_test(genoutputs=False) + if testres == 0: + if self.args.make_Tool == "install": + self.planemo_shedload() + self.eph_galaxy_load() + else: + os.stderr.write( + "Planemo test failed - tool %s was not installed" % self.args.tool_name + ) + def main(): """ This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: - rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" + rgBaseScriptWrapper.py --script_path "$scriptPath" + --tool_name "foo" --interpreter "Rscript" """ parser = argparse.ArgumentParser() a = parser.add_argument - a("--script_path", default="") - a("--dependencies", default="") - a("--cl_override", default="") + a("--script_path", default=None) + a("--history_test", default=None) + a("--cl_prefix", default=None) + a("--sysexe", default=None) + a("--packages", default=None) a("--tool_name", default=None) - a("--interpreter_name", default=None) - a("--interpreter_version", default=None) - a("--exe_package", default=None) - a("--exe_package_version", default=None) a("--input_files", default=[], action="append") a("--output_files", default=[], action="append") a("--user_email", default="Unknown") a("--bad_user", default=None) - a("--make_Tool", default=None) + a("--make_Tool", default="runonly") a("--help_text", default=None) a("--tool_desc", default=None) a("--tool_version", default=None) a("--citations", default=None) + a("--command_override", default=None) + a("--test_override", default=None) a("--additional_parameters", action="append", default=[]) a("--edit_additional_parameters", action="store_true", default=False) a("--parampass", default="positional") a("--tfout", default="./tfout") a("--new_tool", default="new_tool") a("--runmode", default=None) - a("--galaxy_url", default="http://localhost") - a("--galaxy_api_key", default='fakekey') + a("--galaxy_url", default="http://localhost:8080") + a("--galaxy_api_key", default="fbdd3c2eecd191e88939fffc02eeeaf8") a("--toolshed_url", default="http://localhost:9009") - a("--toolshed_api_key", default=None) - a("--planemo_test", default="yes") + a("--toolshed_api_key", default="d46e5ed0e242ed52c6e1f506b5d7f9f7") + a("--galaxy_root", default="/home/ross/galaxy") args = parser.parse_args() assert not args.bad_user, ( 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' % (args.bad_user, args.bad_user) ) + assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq" assert ( - args.tool_name - ), "## Tool Factory expects a tool name - eg --tool_name=DESeq" - assert ( - args.interpreter_name or args.exe_package + args.sysexe or args.packages ), "## Tool Factory wrapper expects an interpreter or an executable package" - assert args.exe_package or ( - len(args.script_path) > 0 and os.path.isfile(args.script_path) - ), "## Tool Factory wrapper expects a script path - eg --script_path=foo.R if no executable" - args.input_files = [ - x.replace('"', "").replace("'", "") for x in args.input_files - ] + args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files] # remove quotes we need to deal with spaces in CL params for i, x in enumerate(args.additional_parameters): - args.additional_parameters[i] = args.additional_parameters[i].replace( - '"', "" - ) + args.additional_parameters[i] = args.additional_parameters[i].replace('"', "") r = ScriptRunner(args) - if args.make_Tool: - r.writeShedyml() - retcode = r.makeTooltar() - if retcode == 0: - if args.planemo_test == "yes": - r.install_load() + r.writeShedyml() + r.makeTool() + if args.make_Tool == "runonly": + retcode = r.run() + if retcode: + sys.stderr.write( + "## Run failed with return code %d. Cannot build yet. Please fix and retry" + % retcode + ) + sys.exit(1) + else: + r.moveRunOutputs() + elif args.make_Tool in ["gentestinstall", "generate", "gentest"]: + retcode = r.run() + if retcode: + sys.stderr.write( + "## Run failed with return code %d. Cannot build yet. Please fix and retry" + % retcode + ) + sys.exit(1) + r.moveRunOutputs() + r.makeToolTar() + if args.make_Tool in ["gentestinstall","gentest"]: + r.planemo_test(genoutputs=False) + r.moveRunOutputs() + r.planemo_shedload() + r.eph_galaxy_load() else: - retcode = r.run() - # if retcode: + retcode = r.planemo_test(genoutputs=True) # this fails :( + r.moveRunOutputs() + r.makeToolTar() + retcode = r.planemo_test(genoutputs=False) + r.moveRunOutputs() + if args.make_Tool == "planemotestinstall": + r.planemo_shedload() + r.eph_galaxy_load() + # if retcode: # sys.exit(retcode) # indicate failure to job runner diff -r 5a7a5b06bce0 -r 35a912ce0c83 toolfactory/rgToolFactory2.xml --- a/toolfactory/rgToolFactory2.xml Sun Aug 23 21:03:48 2020 -0400 +++ b/toolfactory/rgToolFactory2.xml Thu Aug 27 23:11:01 2020 -0400 @@ -1,9 +1,46 @@ Scripts into tools + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + help="USE SMALL SAMPLES for the new tool's test. Prompts will form a history item selector as input for users of this new tool"> + @@ -84,47 +123,17 @@ - - + help="Some targets like Planemo expect an unadorned action like 'test' before --galaxy_root." value="" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python + perl + r-base galaxyxml planemo yaml @@ -137,25 +146,27 @@ $__tool_directory__/rgToolFactory2.py --bad_user $__user_email__ #else: $__tool_directory__/rgToolFactory2.py ---dependencies "$interexe.dependencies" ---runmode "$interexe.interpreter" - #if len(str($interexe.cl_override)) > 0: ---cl_override "$interexe.cl_override" - #end if - #if str($interexe.interpreter)=="Executable" or str($interexe.interpreter)=="system": ---exe_package="$interexe.exe_package" - #if $interexe.exe_package_version: ---exe_package_version="$interexe.exe_package_version" + #if len(str($cl_prefix)) > 3: +--cl_prefix "$cl_prefix" + #end if + #if $cover.commover == "yes": + #if len(str($cover.command_override)) > 10: +--command_override "$commandoverride" + #end if + #if len(str($cover.test_override)) > 10: +--test_override "$testoverride" + #end if #end if - #else: ---interpreter_name="$interexe.interpreter" --script_path "$runme" - #if str($interexe.interpreter_version) != 'None': ---interpreter_version="$interexe.interpreter_version" - #end if + #if $interexe.dependency_model == "managed": +--packages "$interexe.packages" + #else: +--sysexe "$interexe.sysexe" + #end if +--tool_name="$tool_name" --user_email="$__user_email__" --citations="$citeme" --parampass="$ppass.parampass" + #if $usescript.choosescript == "yes": +--script_path "$runme" #end if ---tool_name="$tool_name" --user_email="$__user_email__" --citations="$citeme" --parampass="$ppass.parampass" - #if str($makeMode.make_Tool)=="yes": ---planemo_test "$makeMode.planemo_test" + #if str($makeMode.make_Tool)!="runonly": --make_Tool="$makeMode.make_Tool" --tool_desc="$makeMode.tool_desc" --tool_version="$makeMode.tool_version" @@ -174,28 +185,35 @@ --input_files="$intab.input_files~~~$intab.input_CL~~~$intab.input_formats~~~$intab.input_label~~~$intab.input_help" #end for #for $otab in $ppass.history_outputs: ---output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL" +--output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL~~~$otab.history_test" #end for #end if ]]> - -#if $interexe.interpreter != "Executable" and $interexe.interpreter != "system" : -${interexe.dynScript} -#else: -$tool_name +#if $usescript.choosescript == 'yes': +$usescript.dynScript #end if + +#if $cover.commover == "yes" and len(str($cover.command_override).strip()) > 1: +$cover.command_override +#end if + + +#if $cover.commover == "yes" and len(str($cover.test_override).strip()) > 1: +$cover.test_override +#end if + - #if $makeMode.make_Tool == "yes": + #if $makeMode.make_Tool != "runonly": ${makeMode.help_text} #else $tool_name help goes here #end if -#if $makeMode.make_Tool == "yes": +#if $makeMode.make_Tool != "runonly": #for $citation in $makeMode.citations: #if $citation.citation_type.type == "bibtex": **ENTRY**bibtex @@ -209,131 +227,135 @@ - + help="Toolshed repository name. Choose thoughtfully to avoid namespace clashes with other tool writers. Lower case, digits and underscores only"> + - - - - - - - - + + + - - - - - + + + - - - - - - - - - - - - - - - - - - - - + + + - - + + + + + + + + + + + + + + + + + + + - - - + + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - makeMode['make_Tool'] == "yes" + makeMode['make_Tool'] != "runonly" @@ -351,7 +373,7 @@ - + diff -r 5a7a5b06bce0 -r 35a912ce0c83 toolfactory/testtf.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/testtf.sh Thu Aug 27 23:11:01 2020 -0400 @@ -0,0 +1,2 @@ +planemo test --no_cleanup --no_dependency_resolution --skip_venv --galaxy_root ~/galaxy ~/galaxy/tools/tool_makers/toolfactory &>foo +