# HG changeset patch # User fubar # Date 1603680714 0 # Node ID 881bd8c81ac946bbc11e1ca4c153b8bc7ffe273c # Parent dc923f16e6ffe37ab326659fe944553396934859 Uploaded diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/rgToolFactory2.py --- a/toolfactory/rgToolFactory2.py Thu Oct 22 03:39:09 2020 +0000 +++ b/toolfactory/rgToolFactory2.py Mon Oct 26 02:51:54 2020 +0000 @@ -1,4 +1,3 @@ -#!/usr/bin/env python # rgToolFactory.py # see https://github.com/fubar2/toolfactory # @@ -325,15 +324,18 @@ """ no parameters - uses < and > for i/o """ aCL = self.cl.append - aCL("<") - aCL(self.infiles[0][IPATHPOS]) - aCL(">") - aCL(self.outfiles[0][OCLPOS]) aXCL = self.xmlcl.append - aXCL("<") - aXCL("$%s" % self.infiles[0][ICLPOS]) - aXCL(">") - aXCL("$%s" % self.outfiles[0][ONAMEPOS]) + + if len(self.infiles) > 0: + aCL("<") + aCL(self.infiles[0][IPATHPOS]) + aXCL("<") + aXCL("$%s" % self.infiles[0][ICLPOS]) + if len(self.outfiles) > 0: + aCL(">") + aCL(self.outfiles[0][OCLPOS]) + aXCL(">") + aXCL("$%s" % self.outfiles[0][ONAMEPOS]) def clpositional(self): # inputs in order then params @@ -482,37 +484,39 @@ def doNoXMLparam(self): """filter style package - stdin to stdout""" - alab = self.infiles[0][ILABPOS] - if len(alab) == 0: - alab = self.infiles[0][ICLPOS] - max1s = ( - "Maximum one input if parampass is 0 but multiple input files supplied - %s" - % str(self.infiles) - ) - assert len(self.infiles) == 1, max1s - newname = self.infiles[0][ICLPOS] - aninput = gxtp.DataParam( - newname, - optional=False, - label=alab, - help=self.infiles[0][IHELPOS], - format=self.infiles[0][IFMTPOS], - multiple=False, - num_dashes=0, - ) - aninput.command_line_override = "< $%s" % newname - aninput.positional = self.is_positional - self.tinputs.append(aninput) - tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) - self.testparam.append(tp) - newname = self.outfiles[0][OCLPOS] - newfmt = self.outfiles[0][OFMTPOS] - anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) - anout.command_line_override = "> $%s" % newname - anout.positional = self.is_positional - self.toutputs.append(anout) - tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname, format=newfmt) - self.testparam.append(tp) + if len(self.infiles) > 0: + alab = self.infiles[0][ILABPOS] + if len(alab) == 0: + alab = self.infiles[0][ICLPOS] + max1s = ( + "Maximum one input if parampass is 0 but multiple input files supplied - %s" + % str(self.infiles) + ) + assert len(self.infiles) == 1, max1s + newname = self.infiles[0][ICLPOS] + aninput = gxtp.DataParam( + newname, + optional=False, + label=alab, + help=self.infiles[0][IHELPOS], + format=self.infiles[0][IFMTPOS], + multiple=False, + num_dashes=0, + ) + aninput.command_line_override = "< $%s" % newname + aninput.positional = self.is_positional + self.tinputs.append(aninput) + tp = gxtp.TestParam(name=newname, value="%s_sample" % newname) + self.testparam.append(tp) + if len(self.outfiles > 0): + newname = self.outfiles[0][OCLPOS] + newfmt = self.outfiles[0][OFMTPOS] + anout = gxtp.OutputData(newname, format=newfmt, num_dashes=0) + anout.command_line_override = "> $%s" % newname + anout.positional = self.is_positional + self.toutputs.append(anout) + tp = gxtp.TestOutput(name=newname, value="%s_sample" % newname, format=newfmt) + self.testparam.append(tp) def makeXML(self): """ @@ -529,13 +533,13 @@ safertext = [html_escape(x) for x in helptext] if self.args.script_path: scrpt = self.script.split('\n') - scrpt.append("```\n") + scrpt.insert(0,"```\n") if len(scrpt) > 300: safertext = safertext + scrpt[:100] + ['>500 lines - stuff deleted','......'] + scrpt[-100:] else: safertext = safertext + scrpt safertext.append("\n```") - self.newtool.help = "".join([x for x in safertext]) + self.newtool.help = "\n".join([x for x in safertext]) else: self.newtool.help = ( "Please ask the tool author (%s) for help \ @@ -627,8 +631,14 @@ ste.close() retval = p.returncode else: # work around special case - stdin and write to stdout - sti = open(self.infiles[0][IPATHPOS], "rb") - sto = open(self.outfiles[0][ONAMEPOS], "wb") + if len(self.infiles) > 0: + sti = open(self.infiles[0][IPATHPOS], "rb") + else: + sti = sys.stdin + if len(self.outfiles) > 0: + sto = open(self.outfiles[0][ONAMEPOS], "wb") + else: + sto = sys.stdout p = subprocess.run(self.cl, shell=False, stdout=sto, stdin=sti) retval = p.returncode sto.close() @@ -695,7 +705,6 @@ cll = [ "planemo", "test", - "--skip_venv", "--galaxy_root", self.args.galaxy_root, "--update_test_data", @@ -703,7 +712,7 @@ ] else: cll = ["planemo", "test", "--galaxy_root", - self.args.galaxy_root, "--skip_venv", + self.args.galaxy_root, xreal,] p = subprocess.run( cll, shell=False, cwd=self.tooloutdir, stderr=tout, stdout=tout diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/rgToolFactory2.xml --- a/toolfactory/rgToolFactory2.xml Thu Oct 22 03:39:09 2020 +0000 +++ b/toolfactory/rgToolFactory2.xml Mon Oct 26 02:51:54 2020 +0000 @@ -1,5 +1,5 @@ - - Scripts into tools v2.01 + + Scripts into tools v2.0 - tfdeps + planemo + galaxyxml - + diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/test-data/input1_sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/test-data/input1_sample Mon Oct 26 02:51:54 2020 +0000 @@ -0,0 +1,166 @@ +*WARNING before you start* + + Install this tool on a private Galaxy ONLY + Please NEVER on a public or production instance + +Updated august 2014 by John Chilton adding citation support + +Updated august 8 2014 to fix bugs reported by Marius van den Beek + +Please cite the resource at +http://bioinformatics.oxfordjournals.org/cgi/reprint/bts573?ijkey=lczQh1sWrMwdYWJ&keytype=ref +if you use this tool in your published work. + +**Short Story** + +This is an unusual Galaxy tool capable of generating new Galaxy tools. +It works by exposing *unrestricted* and therefore extremely dangerous scripting +to all designated administrators of the host Galaxy server, allowing them to +run scripts in R, python, sh and perl over multiple selected input data sets, +writing a single new data set as output. + +*You have a working r/python/perl/bash script or any executable with positional or argparse style parameters* + +It can be turned into an ordinary Galaxy tool in minutes, using a Galaxy tool. + + +**Automated generation of new Galaxy tools for installation into any Galaxy** + +A test is generated using small sample test data inputs and parameter settings you supply. +Once the test case outputs have been produced, they can be used to build a +new Galaxy tool. The supplied script or executable is baked as a requirement +into a new, ordinary Galaxy tool, fully workflow compatible out of the box. +Generated tools are installed via a tool shed by an administrator +and work exactly like all other Galaxy tools for your users. + +**More Detail** + +To use the ToolFactory, you should have prepared a script to paste into a +text box, or have a package in mind and a small test input example ready to select from your history +to test your new script. + +```planemo test rgToolFactory2.xml --galaxy_root ~/galaxy --test_data ~/galaxy/tools/tool_makers/toolfactory/test-data``` works for me + +There is an example in each scripting language on the Tool Factory form. You +can just cut and paste these to try it out - remember to select the right +interpreter please. You'll also need to create a small test data set using +the Galaxy history add new data tool. + +If the script fails somehow, use the "redo" button on the tool output in +your history to recreate the form complete with broken script. Fix the bug +and execute again. Rinse, wash, repeat. + +Once the script runs sucessfully, a new Galaxy tool that runs your script +can be generated. Select the "generate" option and supply some help text and +names. The new tool will be generated in the form of a new Galaxy datatype +*toolshed.gz* - as the name suggests, it's an archive ready to upload to a +Galaxy ToolShed as a new tool repository. + +Once it's in a ToolShed, it can be installed into any local Galaxy server +from the server administrative interface. + +Once the new tool is installed, local users can run it - each time, the script +that was supplied when it was built will be executed with the input chosen +from the user's history. In other words, the tools you generate with the +ToolFactory run just like any other Galaxy tool,but run your script every time. + +Tool factory tools are perfect for workflow components. One input, one output, +no variables. + +*To fully and safely exploit the awesome power* of this tool, +Galaxy and the ToolShed, you should be a developer installing this +tool on a private/personal/scratch local instance where you are an +admin_user. Then, if you break it, you get to keep all the pieces see +https://bitbucket.org/fubar/galaxytoolfactory/wiki/Home + +**Installation** +This is a Galaxy tool. You can install it most conveniently using the +administrative "Search and browse tool sheds" link. Find the Galaxy Main +toolshed at https://toolshed.g2.bx.psu.edu/ and search for the toolfactory +repository. Open it and review the code and select the option to install it. + +If you can't get the tool that way, the xml and py files here need to be +copied into a new tools +subdirectory such as tools/toolfactory Your tool_conf.xml needs a new entry +pointing to the xml +file - something like:: + +
+ +
+ +If not already there, +please add: + +to your local data_types_conf.xml. + + +**Restricted execution** + +The tool factory tool itself will then be usable ONLY by admin users - +people with IDs in admin_users in universe_wsgi.ini **Yes, that's right. ONLY +admin_users can run this tool** Think about it for a moment. If allowed to +run any arbitrary script on your Galaxy server, the only thing that would +impede a miscreant bent on destroying all your Galaxy data would probably +be lack of appropriate technical skills. + +**What it does** + +This is a tool factory for simple scripts in python, R and +perl currently. Functional tests are automatically generated. How cool is that. + +LIMITED to simple scripts that read one input from the history. Optionally can +write one new history dataset, and optionally collect any number of outputs +into links on an autogenerated HTML index page for the user to navigate - +useful if the script writes images and output files - pdf outputs are shown +as thumbnails and R's bloated pdf's are shrunk with ghostscript so that and +imagemagik need to be available. + +Generated tools can be edited and enhanced like any Galaxy tool, so start +small and build up since a generated script gets you a serious leg up to a +more complex one. + +**What you do** + +You paste and run your script, you fix the syntax errors and +eventually it runs. You can use the redo button and edit the script before +trying to rerun it as you debug - it works pretty well. + +Once the script works on some test data, you can generate a toolshed compatible +gzip file containing your script ready to run as an ordinary Galaxy tool in +a repository on your local toolshed. That means safe and largely automated +installation in any production Galaxy configured to use your toolshed. + +**Generated tool Security** + +Once you install a generated tool, it's just +another tool - assuming the script is safe. They just run normally and their +user cannot do anything unusually insecure but please, practice safe toolshed. +Read the code before you install any tool. Especially this one - it is really scary. + +**Send Code** + +Patches and suggestions welcome as bitbucket issues please? + +**Attribution** + +Creating re-usable tools from scripts: The Galaxy Tool Factory +Ross Lazarus; Antony Kaspi; Mark Ziemann; The Galaxy Team +Bioinformatics 2012; doi: 10.1093/bioinformatics/bts573 + +http://bioinformatics.oxfordjournals.org/cgi/reprint/bts573?ijkey=lczQh1sWrMwdYWJ&keytype=ref + +**Licensing** + +Copyright Ross Lazarus 2010 +ross lazarus at g mail period com + +All rights reserved. + +Licensed under the LGPL + +**Obligatory screenshot** + +http://bitbucket.org/fubar/galaxytoolmaker/src/fda8032fe989/images/dynamicScriptTool.png + diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/test-data/output2_sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/test-data/output2_sample Mon Oct 26 02:51:54 2020 +0000 @@ -0,0 +1,165 @@ +*trats uoy erofeb GNINRAW* + +YLNO yxalaG etavirp a no loot siht llatsnI +ecnatsni noitcudorp ro cilbup a no REVEN esaelP + +troppus noitatic gnidda notlihC nhoJ yb 4102 tsugua detadpU + +keeB ned nav suiraM yb detroper sgub xif ot 4102 8 tsugua detadpU + +ta ecruoser eht etic esaelP +fer=epytyek&JWYdwMrWs1hQzcl=yekji?375stb/tnirper/igc/gro.slanruojdrofxo.scitamrofnioib//:ptth +.krow dehsilbup ruoy ni loot siht esu uoy fi + +**yrotS trohS** + +.sloot yxalaG wen gnitareneg fo elbapac loot yxalaG lausunu na si sihT +gnitpircs suoregnad ylemertxe erofereht dna *detcirtsernu* gnisopxe yb skrow tI +ot meht gniwolla ,revres yxalaG tsoh eht fo srotartsinimda detangised lla ot +,stes atad tupni detceles elpitlum revo lrep dna hs ,nohtyp ,R ni stpircs nur +.tuptuo sa tes atad wen elgnis a gnitirw + +*sretemarap elyts esrapgra ro lanoitisop htiw elbatucexe yna ro tpircs hsab/lrep/nohtyp/r gnikrow a evah uoY* + +.loot yxalaG a gnisu ,setunim ni loot yxalaG yranidro na otni denrut eb nac tI + + +**yxalaG yna otni noitallatsni rof sloot yxalaG wen fo noitareneg detamotuA** + +.ylppus uoy sgnittes retemarap dna stupni atad tset elpmas llams gnisu detareneg si tset A +a dliub ot desu eb nac yeht ,decudorp neeb evah stuptuo esac tset eht ecnO +tnemeriuqer a sa dekab si elbatucexe ro tpircs deilppus ehT .loot yxalaG wen +.xob eht fo tuo elbitapmoc wolfkrow ylluf ,loot yxalaG yranidro ,wen a otni +rotartsinimda na yb dehs loot a aiv dellatsni era sloot detareneG +.sresu ruoy rof sloot yxalaG rehto lla ekil yltcaxe krow dna + +**liateD eroM** + +a otni etsap ot tpircs a deraperp evah dluohs uoy ,yrotcaFlooT eht esu oT +yrotsih ruoy morf tceles ot ydaer elpmaxe tupni tset llams a dna dnim ni egakcap a evah ro ,xob txet +.tpircs wen ruoy tset ot + +em rof skrow ```atad-tset/yrotcafloot/srekam_loot/sloot/yxalag/~ atad_tset-- yxalag/~ toor_yxalag-- lmx.2yrotcaFlooTgr tset omenalp``` + +uoY .mrof yrotcaF looT eht no egaugnal gnitpircs hcae ni elpmaxe na si erehT +thgir eht tceles ot rebmemer - tuo ti yrt ot eseht etsap dna tuc tsuj nac +gnisu tes atad tset llams a etaerc ot deen osla ll'uoY .esaelp reterpretni +.loot atad wen dda yrotsih yxalaG eht + +ni tuptuo loot eht no nottub "oder" eht esu ,wohemos sliaf tpircs eht fI +gub eht xiF .tpircs nekorb htiw etelpmoc mrof eht etaercer ot yrotsih ruoy +.taeper ,hsaw ,esniR .niaga etucexe dna + +tpircs ruoy snur taht loot yxalaG wen a ,yllufssecus snur tpircs eht ecnO +dna txet pleh emos ylppus dna noitpo "etareneg" eht tceleS .detareneg eb nac +epytatad yxalaG wen a fo mrof eht ni detareneg eb lliw loot wen ehT .seman +a ot daolpu ot ydaer evihcra na s'ti ,stseggus eman eht sa - *zg.dehsloot* +.yrotisoper loot wen a sa dehSlooT yxalaG + +revres yxalaG lacol yna otni dellatsni eb nac ti ,dehSlooT a ni s'ti ecnO +.ecafretni evitartsinimda revres eht morf + +tpircs eht ,emit hcae - ti nur nac sresu lacol ,dellatsni si loot wen eht ecnO +nesohc tupni eht htiw detucexe eb lliw tliub saw ti nehw deilppus saw taht +eht htiw etareneg uoy sloot eht ,sdrow rehto nI .yrotsih s'resu eht morf +.emit yreve tpircs ruoy nur tub,loot yxalaG rehto yna ekil tsuj nur yrotcaFlooT + +,tuptuo eno ,tupni enO .stnenopmoc wolfkrow rof tcefrep era sloot yrotcaf looT +.selbairav on + +,loot siht fo *rewop emosewa eht tiolpxe ylefas dna ylluf oT* +siht gnillatsni repoleved a eb dluohs uoy ,dehSlooT eht dna yxalaG +na era uoy erehw ecnatsni lacol hctarcs/lanosrep/etavirp a no loot +ees seceip eht lla peek ot teg uoy ,ti kaerb uoy fi ,nehT .resu_nimda +emoH/ikiw/yrotcaflootyxalag/rabuf/gro.tekcubtib//:sptth + +**noitallatsnI** +eht gnisu yltneinevnoc tsom ti llatsni nac uoY .loot yxalaG a si sihT +niaM yxalaG eht dniF .knil "sdehs loot esworb dna hcraeS" evitartsinimda +yrotcafloot eht rof hcraes dna /ude.usp.xb.2g.dehsloot//:sptth ta dehsloot +.ti llatsni ot noitpo eht tceles dna edoc eht weiver dna ti nepO .yrotisoper + +eb ot deen ereh selif yp dna lmx eht ,yaw taht loot eht teg t'nac uoy fI +sloot wen a otni deipoc +yrtne wen a sdeen lmx.fnoc_loot ruoY yrotcafloot/sloot sa hcus yrotceridbus +lmx eht ot gnitniop +::ekil gnihtemos - elif + +>"sredliubloot"=di "sloot gnidliub looT"=eman noitces< +>/"lmx.yrotcaFlooTgr/yrotcafloot"=elif loot< +>noitces/< + +,ereht ydaerla ton fI +:dda esaelp +"yraniB:yranib.sepytatad.yxalag"=epyt "zg.dehsloot"=noisnetxe epytatad< +>/ "eurT"=ssalcbus "pizg-x/trapitlum"=epytemim +.lmx.fnoc_sepyt_atad lacol ruoy ot + + +**noitucexe detcirtseR** + +- sresu nimda yb YLNO elbasu eb neht lliw flesti loot yrotcaf loot ehT +YLNO .thgir s'taht ,seY** ini.igsw_esrevinu ni sresu_nimda ni sDI htiw elpoep +ot dewolla fI .tnemom a rof ti tuoba knihT **loot siht nur nac sresu_nimda +dluow taht gniht ylno eht ,revres yxalaG ruoy no tpircs yrartibra yna nur +ylbaborp dluow atad yxalaG ruoy lla gniyortsed no tneb tnaercsim a edepmi +.slliks lacinhcet etairporppa fo kcal eb + +**seod ti tahW** + +dna R ,nohtyp ni stpircs elpmis rof yrotcaf loot a si sihT +.taht si looc woH .detareneg yllacitamotua era stset lanoitcnuF .yltnerruc lrep + +nac yllanoitpO .yrotsih eht morf tupni eno daer taht stpircs elpmis ot DETIMIL +stuptuo fo rebmun yna tcelloc yllanoitpo dna ,tesatad yrotsih wen eno etirw +- etagivan ot resu eht rof egap xedni LMTH detarenegotua na no sknil otni +nwohs era stuptuo fdp - selif tuptuo dna segami setirw tpircs eht fi lufesu +dna taht os tpircstsohg htiw knurhs era s'fdp detaolb s'R dna slianbmuht sa +.elbaliava eb ot deen kigamegami + +trats os ,loot yxalaG yna ekil decnahne dna detide eb nac sloot detareneG +a ot pu gel suoires a uoy steg tpircs detareneg a ecnis pu dliub dna llams +.eno xelpmoc erom + +**od uoy tahW** + +dna srorre xatnys eht xif uoy ,tpircs ruoy nur dna etsap uoY +erofeb tpircs eht tide dna nottub oder eht esu nac uoY .snur ti yllautneve +.llew ytterp skrow ti - gubed uoy sa ti nurer ot gniyrt + +elbitapmoc dehsloot a etareneg nac uoy ,atad tset emos no skrow tpircs eht ecnO +ni loot yxalaG yranidro na sa nur ot ydaer tpircs ruoy gniniatnoc elif pizg +detamotua ylegral dna efas snaem tahT .dehsloot lacol ruoy no yrotisoper a +.dehsloot ruoy esu ot derugifnoc yxalaG noitcudorp yna ni noitallatsni + +**ytiruceS loot detareneG** + +tsuj s'ti ,loot detareneg a llatsni uoy ecnO +rieht dna yllamron nur tsuj yehT .efas si tpircs eht gnimussa - loot rehtona +.dehsloot efas ecitcarp ,esaelp tub erucesni yllausunu gnihtyna od tonnac resu +.yracs yllaer si ti - eno siht yllaicepsE .loot yna llatsni uoy erofeb edoc eht daeR + +**edoC dneS** + +?esaelp seussi tekcubtib sa emoclew snoitseggus dna sehctaP + +**noitubirttA** + +yrotcaF looT yxalaG ehT :stpircs morf sloot elbasu-er gnitaerC +maeT yxalaG ehT ;nnameiZ kraM ;ipsaK ynotnA ;surazaL ssoR +375stb/scitamrofnioib/3901.01 :iod ;2102 scitamrofnioiB + +fer=epytyek&JWYdwMrWs1hQzcl=yekji?375stb/tnirper/igc/gro.slanruojdrofxo.scitamrofnioib//:ptth + +**gnisneciL** + +0102 surazaL ssoR thgirypoC +moc doirep liam g ta surazal ssor + +.devreser sthgir llA + +LPGL eht rednu desneciL + +**tohsneercs yrotagilbO** + +gnp.looTtpircScimanyd/segami/989ef2308adf/crs/rekamlootyxalag/rabuf/gro.tekcubtib//:ptth diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/test-data/pyrevpos.python --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/test-data/pyrevpos.python Mon Oct 26 02:51:54 2020 +0000 @@ -0,0 +1,13 @@ +# reverse order of text by row +import sys +inp = sys.argv[1] +outp = sys.argv[2] +i = open(inp,'r').readlines() +o = open(outp,'w') +for row in i: + rs = row.rstrip() + rs = list(rs) + rs.reverse() + o.write(''.join(rs)) +o.close() + diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/test-data/test1_log.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/toolfactory/test-data/test1_log.txt Mon Oct 26 02:51:54 2020 +0000 @@ -0,0 +1,1 @@ +## Executing Toolfactory generated command line = python /tmp/pyrevposq5dmcdy1.python /tmp/tmpqrksf8sd/files/5/b/9/dataset_5b952a86-87df-44ad-a415-ea549f3f0cee.dat output2 diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/test-data/toolfactory_pyrevpos_tgz_sample Binary file toolfactory/test-data/toolfactory_pyrevpos_tgz_sample has changed diff -r dc923f16e6ff -r 881bd8c81ac9 toolfactory/tool_dependencies.xml --- a/toolfactory/tool_dependencies.xml Thu Oct 22 03:39:09 2020 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file