Mercurial > repos > fubar > tool_factory_2
changeset 113:6873c211b250 draft
Uploaded
| author | fubar |
|---|---|
| date | Tue, 01 Dec 2020 09:03:34 +0000 |
| parents | 5509dc4c1cf2 |
| children | e0cf315853cf |
| files | toolfactory/rgToolFactory2.py toolfactory/rgToolFactory2.xml |
| diffstat | 2 files changed, 30 insertions(+), 40 deletions(-) [+] |
line wrap: on
line diff
--- a/toolfactory/rgToolFactory2.py Sun Nov 29 10:34:48 2020 +0000 +++ b/toolfactory/rgToolFactory2.py Tue Dec 01 09:03:34 2020 +0000 @@ -194,7 +194,7 @@ if not os.path.exists(self.tooloutdir): os.mkdir(self.tooloutdir) if not os.path.exists(self.testdir): - os.mkdir(self.testdir) # make tests directory + os.mkdir(self.testdir) if not os.path.exists(self.repdir): os.mkdir(self.repdir) self.tinputs = gxtp.Inputs() @@ -224,7 +224,7 @@ aXCL(self.executeme) aXCL("$runme") else: - aCL(self.executeme) # this little CL will just run + aCL(self.executeme) aXCL(self.executeme) self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name) self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name) @@ -257,7 +257,7 @@ if p[OOCLPOS] == "STDOUT": self.lastclredirect = [">", p[ONAMEPOS]] self.lastxclredirect = [">", "$%s" % p[OCLPOS]] - else: # for (o_v, k, v, koverride) in self.xclsuffix: + else: clsuffix.append([p[ONAMEPOS], p[ONAMEPOS], p[ONAMEPOS], ""]) xclsuffix.append([p[ONAMEPOS], p[ONAMEPOS], "$%s" % p[ONAMEPOS], ""]) for p in self.addpar: @@ -313,7 +313,7 @@ for i, p in enumerate( self.outfiles ): - if self.args.parampass == "positional" and p[OCLPOS] != "STDOUT": + if self.args.parampass == "positional" and p[OCLPOS].upper() != "STDOUT": assert p[ OCLPOS ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % ( @@ -321,7 +321,7 @@ p[ONAMEPOS], ) p.append(p[OCLPOS]) # keep copy - if p[OOCLPOS].isdigit() or p[OOCLPOS] == "STDOUT": + if p[OOCLPOS].isdigit() or p[OOCLPOS].upper() == "STDOUT": scl = p[ONAMEPOS] p[OCLPOS] = scl self.outfiles[i] = p @@ -369,12 +369,6 @@ if self.lastxclredirect: aXCL(self.lastxclredirect[0]) aXCL(self.lastxclredirect[1]) - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - tout.write(f" #### clpositional: self.clsuffix = {self.clsuffix} and self.xclsuffix = {self.xclsuffix}") - tout.close() def clargparse(self): @@ -401,12 +395,6 @@ k = "--%s" % k aCL(k) aCL(v) - if os.path.exists(self.tlog): - tout = open(self.tlog, "a") - else: - tout = open(self.tlog, "w") - tout.write(f" #### clargparse: self.clsuffix = {self.clsuffix} and self.xclsuffix = {self.xclsuffix}") - tout.close() def getNdash(self, newname): @@ -427,7 +415,7 @@ aparm = gxtp.OutputData(name=newname, format=newfmt, num_dashes=ndash, label=newcl) aparm.positional = self.is_positional if self.is_positional: - if oldcl == "STDOUT": + if oldcl.upper() == "STDOUT": aparm.positional = 9999999 aparm.command_line_override = "> $%s" % newname else: @@ -603,11 +591,13 @@ requirements = gxtp.Requirements() if self.args.packages: for d in self.args.packages.split(","): + ver = '' + d = d.replace('==',':') + d = d.replace('=',':') if ":" in d: packg, ver = d.split(":") else: packg = d - ver = "" requirements.append( gxtp.Requirement("package", packg.strip(), ver.strip()) ) @@ -751,6 +741,7 @@ slogl = str(rlog).split("\\n") slog = "\n".join(slogl) tout.write(f"## got rlog {slog} from {cl}\n") + if os.path.exists(self.tlog): tout = open(self.tlog, "a") else: @@ -765,16 +756,18 @@ tvolname = tvol.name destdir = "/toolfactory/ptest" imrep = os.path.join(destdir, repname) - # need to keep the container running so sleep a while - we stop and destroy it when we are done + # need to keep the container running so keep it open with tail -f. + # will stop and destroy it when we are done container = client.containers.run( planemoimage, - "sleep 30m", + "tail -f /galaxy-central/CITATION", detach=True, user="biodocker", volumes={f"{tvolname}": {"bind": "/toolfactory", "mode": "rw"}}, ) cl = f"mkdir -p {destdir}" prun(container, tout, cl, user="root") + # that's how hard it is to get root on a biodocker container :( cl = f"rm -rf {destdir}/*" prun(container, tout, cl, user="root") ptestpath = os.path.join(destdir, "tfout", xreal) @@ -785,10 +778,10 @@ ptestcl = f"planemo test --update_test_data --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}" try: rlog = container.exec_run(ptestcl) + # fails because test outputs missing but updates the test-data directory except: e = sys.exc_info()[0] tout.write(f"#### error: {e} from {ptestcl}\n") - # fails - used to generate test outputs cl = f"planemo test --test_output {imrep} --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}" try: prun(container, tout, cl) @@ -808,17 +801,12 @@ container.stop() container.remove() tvol.remove() - # shutil.rmtree(testouts) + # shutil.rmtree(testouts) leave for debugging def shedLoad(self): """ - {'deleted': False, - 'description': 'Tools for manipulating data', - 'id': '175812cd7caaf439', - 'model_class': 'Category', - 'name': 'Text Manipulation', - 'url': '/api/categories/175812cd7caaf439'}] - + use bioblend to create new repository + or update existing """ if os.path.exists(self.tlog): @@ -851,7 +839,7 @@ category_ids=catID, ) tid = res.get("id", None) - sto.write(f"#####create_repository {self.args.tool_name} tid={tid} res={res}\n") + sto.write(f"#create_repository {self.args.tool_name} tid={tid} res={res}\n") else: i = rnames.index(self.tool_name) tid = rids[i] @@ -859,15 +847,17 @@ res = ts.repositories.update_repository( id=tid, tar_ball_path=self.newtarpath, commit_message=None ) - sto.write(f"#####update res id {id} ={res}\n") + sto.write(f"#update res id {id} ={res}\n") except ConnectionError: sto.write( - "Is the toolshed running and the API key correct? Bioblend shed upload failed\n" + "####### Is the toolshed running and the API key correct? Bioblend shed upload failed\n" ) sto.close() def eph_galaxy_load(self): - """load the new tool from the local toolshed after planemo uploads it""" + """ + use ephemeris to load the new tool from the local toolshed after planemo uploads it + """ if os.path.exists(self.tlog): tout = open(self.tlog, "a") else: @@ -1034,7 +1024,7 @@ a("--galaxy_venv", default="/galaxy_venv") args = parser.parse_args() assert not args.bad_user, ( - 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the Galaxy configuration file' + 'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file' % (args.bad_user, args.bad_user) ) assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
--- a/toolfactory/rgToolFactory2.xml Sun Nov 29 10:34:48 2020 +0000 +++ b/toolfactory/rgToolFactory2.xml Tue Dec 01 09:03:34 2020 +0000 @@ -39,7 +39,7 @@ </repeat> </xml> <xml name="io"> - <repeat name="history_inputs" title="Add a data file from your history to pass in to the script. Use the '+' button as needed" + <repeat name="history_inputs" title="zero or more input data file parameters to appear on the tool form to be passed to the executable" help="USE SMALL SAMPLES for the new tool's test. Prompts will form a history item selector as input for users of this new tool"> <param name="input_files" type="data" format="data" label="Select an input file from your history" optional="true" multiple="false" help=""/> @@ -67,7 +67,7 @@ help="If you will pass positional parameters, enter the integer ordinal for this parameter. If Argparse style, '--' will be prepended or '-' if single character" value=""> </param> </repeat> - <repeat name="history_outputs" title="Add a tool run output file to the user's history from your tool - Use the '+' button to add as many as needed" + <repeat name="history_outputs" title="one or more output file parameters generated by the tool to be added to the user history" help="The name will become a history item for users of the new tool you are making containing one of it's outputs"> <param name="history_name" type="text" label="Name for this output to appear in new history" optional="false" help="Argparse will also use this name as --[name]"/> <param name="history_format" type="select" multiple="false" label="Select the datatype for this output" @@ -83,13 +83,13 @@ </repeat> </xml> <xml name="additparam"> - <param name="edit_params" type="select" display="radio" label="Add any additional parameters to the generated tool form so they are user editable?" + <param name="edit_params" type="select" display="radio" label="Make parameters on the generated tool form user editable?" help="If no (default), users will NOT be able to alter any additional parameters. If yes, these will appear on the tool form as text fields with no validation or sanitizing"> <option value="yes" selected="true">Yes, allow user to edit all additional parameters on the generated tool form</option> <option value="no">No - use the fixed values for all additional parameters - no user editing</option> </param> - <repeat name="additional_parameters" title="Pass additional parameters to the script" + <repeat name="additional_parameters" title="zero or more user supplied parameter fields to the tool form for the executable" help="See examples below to see how these can be parsed by scripts in the various languages"> <param name="param_name" type="text" value="" label="Choose the name for this parameter - MUST not be blank!"> <sanitizer invalid_char=""> @@ -238,7 +238,7 @@ </sanitizer> </param> <param name="packages" type="text" value="" label="Conda dependencies as package name[:version, name:version...]. These will always be available when this tool executes" - optional="false" help="Use :[ver] for specific version - 'bwa:0.17.0'. Default is latest. Will be used every time the tool is (re)run. Only Conda is currently supported" /> + optional="false" help="Use =[ver] or :[ver] for specific version - 'bwa=0.17.0'. Default is latest. Will be used every time the tool is (re)run. Only Conda is currently supported" /> <conditional name="usescript"> <param name="choosescript" type="select" display="radio" label="Supply a script for a dependency (e.g. python/R/bash) or a system executable such as Bash"