comparison toolfactory/rgToolFactory2.py @ 119:8ea1133b9d9a draft

Uploaded

118:e43c43396a70

# July 2020: BCC was fun and I feel like rip van winkle after 5 years.
# Decided to
# 1. Fix the toolfactory so it works - done for simplest case
# 2. Fix planemo so the toolfactory function works
# 3. Rewrite bits using galaxyxml functions where that makes sense - done
#
# uses planemo in a biodocker sort of image as a requirement
# otherwise planemo seems to leak dependencies back into the
# calling venv. Hilarity ensues.



import argparse
import copy
import datetime
import grp
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time


from bioblend import ConnectionError
from bioblend import toolshed

import docker

import galaxyxml.tool as gxt
import galaxyxml.tool.parameters as gxtp

import lxml

verbose = True
debug = True
toolFactoryURL = "https://github.com/fubar2/toolfactory"
ourdelim = "~~~"

# --input_files="$intab.input_files~~~$intab.input_CL~~~$intab.input_formats\
#~~~$intab.input_label~~~$intab.input_help"
IPATHPOS = 0
ICLPOS = 1
IFMTPOS = 2
ILABPOS = 3
IHELPOS = 4
IOCLPOS = 5

# --output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL~~~$otab.history_test"
ONAMEPOS = 0
OFMTPOS = 1
OCLPOS = 2
OTESTPOS = 3
OOCLPOS = 4


# --additional_parameters="$i.param_name~~~$i.param_value~~~
# $i.param_label~~~$i.param_help~~~$i.param_type~~~$i.CL~~~i$.param_CLoverride"
ANAMEPOS = 0
AVALPOS = 1
ALABPOS = 2
AHELPPOS = 3
ATYPEPOS = 4

        return s
    except ValueError:
        return '"%s"' % s


html_escape_table = {"&": "&amp;", ">": "&gt;", "<": "&lt;", "$": r"\$","#":"&#35;", "$":"&#36;"}
cheetah_escape_table = {"$": "\$","#":"\#"}

def html_escape(text):
    """Produce entities within text."""
    return "".join([html_escape_table.get(c, c) for c in text])

def cheetah_escape(text):
    """Produce entities within text."""
    return "".join([cheetah_escape_table.get(c, c) for c in text])

    """Revert entities within text. Multiple character targets so use replace"""
    t = text.replace("&", "&")
    t = t.replace(">", ">")
    t = t.replace("&lt;", "<")
    t = t.replace("\\$", "$")
    t = t.replace("&#36;","$")
    t = t.replace("&#35;","#")
    return t


def parse_citations(citations_text):
    """"""

    citation_tuples = []
    for citation in citations:
        if citation.startswith("doi"):
            citation_tuples.append(("doi", citation[len("doi") :].strip()))
        else:
            citation_tuples.append(("bibtex", citation[len("bibtex") :].strip()))
    return citation_tuples


class ScriptRunner:
    """Wrapper for an arbitrary script

        self.is_positional = self.args.parampass == "positional"
        if self.args.sysexe:
            self.executeme = self.args.sysexe
        else:
            if self.args.packages:
                self.executeme = self.args.packages.split(",")[0].split(":")[0]
            else:
                self.executeme = None
        aCL = self.cl.append
        aXCL = self.xmlcl.append
        assert args.parampass in [

                aXCL(self.executeme)
                aXCL("$runme")
            else:
                aCL(self.executeme)
                aXCL(self.executeme)
        self.elog = os.path.join(self.repdir, "%s_error_log.txt" % self.tool_name)
        self.tlog = os.path.join(self.repdir, "%s_runner_log.txt" % self.tool_name)

        if self.args.parampass == "0":
            self.clsimple()
        else:
            clsuffix = []
            xclsuffix = []
            for i, p in enumerate(self.infiles):
                if p[IOCLPOS] == "STDIN":
                    appendme = [
                        p[IOCLPOS],
                        p[ICLPOS],
                        p[IPATHPOS],
                        "< %s" % p[IPATHPOS],
                    ]
                    xappendme = [
                        p[IOCLPOS],
                        p[ICLPOS],
                        p[IPATHPOS],
                        "< $%s" % p[ICLPOS],
                    ]
                else:

            for i, p in enumerate(self.outfiles):
                if p[OOCLPOS] == "STDOUT":
                    self.lastclredirect = [">", p[ONAMEPOS]]
                    self.lastxclredirect = [">", "$%s" % p[OCLPOS]]
                else:
                    clsuffix.append([p[ONAMEPOS], p[ONAMEPOS], p[ONAMEPOS], ""])
                    xclsuffix.append([p[ONAMEPOS], p[ONAMEPOS], "$%s" % p[ONAMEPOS], ""])
            for p in self.addpar:
                clsuffix.append([p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]])
                xclsuffix.append(
                    [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]]
                )
            clsuffix.sort()
            xclsuffix.sort()

        tscript.close()
        self.escapedScript = [cheetah_escape(x) for x in rx]
        self.spacedScript = [f"    {x}" for x in rx if x.strip() > ""]
        art = "%s.%s" % (self.tool_name, self.executeme)
        artifact = open(art, "wb")
        artifact.write(bytes('\n'.join(self.escapedScript),'utf8'))
        artifact.close()

    def cleanuppar(self):
        """ positional parameters are complicated by their numeric ordinal"""
        for i, p in enumerate(self.infiles):
            infp = copy.copy(p)
            if self.args.parampass == "positional":
                assert infp[
                    ICLPOS
                ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
                    infp[ICLPOS],
                    infp[ILABPOS],
                )
            icl = infp[ICLPOS]
            infp.append(icl)
            if infp[ICLPOS].isdigit() or self.args.parampass == "0":
                scl = "input%d" % (i + 1)
                infp[ICLPOS] = scl
            self.infiles[i] = infp
        for i, p in enumerate(
            self.outfiles
        ):
            if self.args.parampass == "positional" and p[OCLPOS].upper() != "STDOUT":
                assert p[
                    OCLPOS
                ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
                    p[OCLPOS],
                    p[ONAMEPOS],
                )
            p.append(p[OCLPOS]) # keep copy
            if p[OOCLPOS].isdigit() or p[OOCLPOS].upper() == "STDOUT":
                scl = p[ONAMEPOS]
                p[OCLPOS] = scl
            self.outfiles[i] = p
        for i, p in enumerate(self.addpar):
            if self.args.parampass == "positional":
                assert p[
                    ACLPOS
                ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
                    p[ACLPOS],
                    p[ANAMEPOS],
                )
            p.append(p[ACLPOS])
            if p[ACLPOS].isdigit():
                scl = "input%s" % p[ACLPOS]
                p[ACLPOS] = scl
            self.addpar[i] = p

    def clsimple(self):
        """no parameters - uses < and > for i/o"""

            aXCL(v)
        if self.lastxclredirect:
            aXCL(self.lastxclredirect[0])
            aXCL(self.lastxclredirect[1])


    def clargparse(self):
        """argparse style"""
        aCL = self.cl.append
        aXCL = self.xmlcl.append
        # inputs then params in argparse named form

            else:
                k = "--%s" % k
            aCL(k)
            aCL(v)


    def getNdash(self, newname):
        if self.is_positional:
            ndash = 0
        else:
            ndash = 2

                ndash = 1
        return ndash

    def doXMLparam(self):
        """flake8 made me do this..."""
        for p in self.outfiles: # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL~~~$otab.history_test"
            newname, newfmt, newcl, test, oldcl = p
            test = test.strip()
            ndash = self.getNdash(newcl)
            aparm = gxtp.OutputData(name=newname, format=newfmt, num_dashes=ndash, label=newcl)
            aparm.positional = self.is_positional
            if self.is_positional:
                if oldcl.upper() == "STDOUT":
                    aparm.positional = 9999999
                    aparm.command_line_override = "> $%s" % newname

                    c = "diff"
                    ld = 0
                    if test.split(":")[1].isdigit:
                        ld = int(test.split(":")[1])
                    tp = gxtp.TestOutput(
                                    name=newcl,
                                    value="%s_sample" % newcl,
                                    format=newfmt,
                                    compare= c,
                                    lines_diff=ld,
                                )
                elif test.startswith("sim_size"):
                    c = "sim_size"
                    tn = test.split(":")[1].strip()
                    if tn > '':
                        if '.' in tn:
                            delta = None
                            delta_frac = min(1.0,float(tn))
                        else:
                            delta = int(tn)
                            delta_frac = None
                    tp = gxtp.TestOutput(
                                    name=newcl,
                                    value="%s_sample" % newcl,
                                    format=newfmt,
                                    compare= c,
                                    delta = delta,
                                    delta_frac = delta_frac
                                )
                self.testparam.append(tp)
        for p in self.infiles:
            newname = p[ICLPOS]
            newfmt = p[IFMTPOS]
            ndash = self.getNdash(newname)

            aninput.positional = self.is_positional
            self.tinputs.append(aninput)
            tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname)
            self.testparam.append(tparm)
        for p in self.addpar:
            newname, newval, newlabel, newhelp, newtype, newcl, override, oldcl = p
            if not len(newlabel) > 0:
                newlabel = newname
            ndash = self.getNdash(newname)
            if newtype == "text":
                aparm = gxtp.TextParam(

        Create a Galaxy xml tool wrapper for the new script
        Uses galaxyhtml
        Hmmm. How to get the command line into correct order...
        """
        if self.command_override:
            self.newtool.command_override = self.command_override  # config file
        else:
            self.newtool.command_override = self.xmlcl
        if self.args.help_text:
            helptext = open(self.args.help_text, "r").readlines()
            safertext = "\n".join([cheetah_escape(x) for x in helptext])
            if self.args.script_path:
                scr = [x for x in self.spacedScript if x.strip() > ""]
                scr.insert(0,'\n------\n\n\nScript::\n')
                if len(scr) > 300:
                    scr = (
                        scr[:100]
                        + ["    >300 lines - stuff deleted", "    ......"]
                        + scr[-100:]
                    )
                scr.append('\n')
                safertext = safertext + "\n".join(scr)
            self.newtool.help = safertext
        else:
            self.newtool.help = (
                "Please ask the tool author (%s) for help \

            )
        self.newtool.version_command = None  # do not want
        requirements = gxtp.Requirements()
        if self.args.packages:
            for d in self.args.packages.split(","):
                ver = ''
                d = d.replace('==',':')
                d = d.replace('=',':')
                if ":" in d:
                    packg, ver = d.split(":")
                else:
                    packg = d
                requirements.append(

            self.doXMLparam()
        self.newtool.outputs = self.toutputs
        self.newtool.inputs = self.tinputs
        if self.args.script_path:
            configfiles = gxtp.Configfiles()
            configfiles.append(gxtp.Configfile(name="runme", text="\n".join(self.escapedScript)))
            self.newtool.configfiles = configfiles
        tests = gxtp.Tests()
        test_a = gxtp.Test()
        for tp in self.testparam:
            test_a.append(tp)

        self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL))
        self.newtool.add_comment(
            "Cite: Creating re-usable tools from scripts doi:10.1093/bioinformatics/bts573"
        )
        exml0 = self.newtool.export()
        exml = exml0.replace(FAKEEXE, "")  # temporary work around until PR accepted
        if (
            self.test_override
        ):  # cannot do this inside galaxyxml as it expects lxml objects for tests
            part1 = exml.split("<tests>")[0]
            part2 = exml.split("</tests>")[1]
            fixed = "%s\n%s\n%s" % (part1, self.test_override, part2)
            exml = fixed
        #exml = exml.replace('range="1:"', 'range="1000:"')
        xf = open("%s.xml" % self.tool_name, "w")
        xf.write(exml)
        xf.write("\n")
        xf.close()
        # ready for the tarball

            if os.path.exists(self.elog):
                ste = open(self.elog, "a")
            else:
                ste = open(self.elog, "w")
            if self.lastclredirect:
                sto = open(self.lastclredirect[1], "wb")  # is name of an output file
            else:
                if os.path.exists(self.tlog):
                    sto = open(self.tlog, "a")
                else:
                    sto = open(self.tlog, "w")
                sto.write(
                    "## Executing Toolfactory generated command line = %s\n" % scl
                )
            sto.flush()
            subp = subprocess.run(
                self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste
            )

            else:
                sto = sys.stdout
            subp = subprocess.run(
                self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti
            )
            sto.write("## Executing Toolfactory generated command line = %s\n" % scl)
            retval = subp.returncode
            sto.close()
            sti.close()
        if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
            os.unlink(self.tlog)

        if retval != 0 and err:  # problem
            sys.stderr.write(err)
        logging.debug("run done")
        return retval

    def copy_to_container(self, src, dest, container):
        """Recreate the src directory tree at dest - full path included"""
        idir = os.getcwd()
        workdir = os.path.dirname(src)
        os.chdir(workdir)
        _, tfname = tempfile.mkstemp(suffix=".tar")
        tar = tarfile.open(tfname, mode="w")
        srcb = os.path.basename(src)
        tar.add(srcb)
        tar.close()
        data = open(tfname, "rb").read()
        container.put_archive(dest, data)
        os.unlink(tfname)
        os.chdir(idir)

    def copy_from_container(self, src, dest, container):
        """recreate the src directory tree at dest using docker sdk"""
        os.makedirs(dest, exist_ok=True)
        _, tfname = tempfile.mkstemp(suffix=".tar")
        tf = open(tfname, "wb")
        bits, stat = container.get_archive(src)
        for chunk in bits:
            tf.write(chunk)
        tf.close()
        tar = tarfile.open(tfname, "r")
        tar.extractall(dest)
        tar.close()
        os.unlink(tfname)

    def planemo_biodocker_test(self):
        """planemo currently leaks dependencies if used in the same container and gets unhappy after a
        first successful run. https://github.com/galaxyproject/planemo/issues/1078#issuecomment-731476930

        Docker biocontainer has planemo with caches filled to save repeated downloads


        """

        def prun(container, tout, cl, user="biodocker"):
            rlog = container.exec_run(cl, user=user)
            slogl = str(rlog).split("\\n")
            slog = "\n".join(slogl)
            tout.write(f"## got rlog {slog} from {cl}\n")

        if os.path.exists(self.tlog):
            tout = open(self.tlog, "a")
        else:
            tout = open(self.tlog, "w")
        planemoimage = "quay.io/fubar2/planemo-biocontainer"
        xreal = "%s.xml" % self.tool_name
        repname = f"{self.tool_name}_planemo_test_report.html"
        ptestrep_path = os.path.join(self.repdir, repname)
        tool_name = self.tool_name
        client = docker.from_env()
        tvol = client.volumes.create()
        tvolname = tvol.name
        destdir = "/toolfactory/ptest"
        imrep = os.path.join(destdir, repname)
        # need to keep the container running so keep it open with sleep
        # will stop and destroy it when we are done
        container = client.containers.run(
            planemoimage,
            "sleep 120m",
            detach=True,
            user="biodocker",
            volumes={f"{tvolname}": {"bind": "/toolfactory", "mode": "rw"}},
        )
        cl = f"mkdir -p {destdir}"
        prun(container, tout, cl, user="root")
        # that's how hard it is to get root on a biodocker container :(
        cl = f"rm -rf {destdir}/*"
        prun(container, tout, cl, user="root")
        ptestpath = os.path.join(destdir, "tfout", xreal)
        self.copy_to_container(self.tooloutdir, destdir, container)
        cl = "chown -R biodocker /toolfactory"
        prun(container, tout, cl, user="root")
        rlog = container.exec_run(f"ls -la {destdir}")
        ptestcl = f"planemo test  --update_test_data  --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
        try:
            rlog = container.exec_run(ptestcl)
            # fails because test outputs missing but updates the test-data directory
        except:
            e = sys.exc_info()[0]
            tout.write(f"#### error: {e} from {ptestcl}\n")
        cl = f"planemo test  --test_output {imrep} --no_cleanup --test_data {destdir}/tfout/test-data --galaxy_root /home/biodocker/galaxy-central {ptestpath}"
        try:
            prun(container, tout, cl)
        except:
            e = sys.exc_info()[0]
            tout.write(f"#### error: {e} from {ptestcl}\n")
        testouts = tempfile.mkdtemp(suffix=None, prefix="tftemp", dir=".")
        self.copy_from_container(destdir, testouts, container)
        src = os.path.join(testouts, "ptest")
        if os.path.isdir(src):
            shutil.copytree(src, ".", dirs_exist_ok=True)
            src = repname
            if os.path.isfile(repname):
                shutil.copyfile(src, ptestrep_path)
        else:
            tout.write(f"No output from run to shutil.copytree in {src}\n")
        tout.close()
        container.stop()
        container.remove()
        tvol.remove()
        shutil.rmtree(testouts) # leave for debugging

    def shedLoad(self):
        """
        use bioblend to create new repository
        or update existing

            sto = open(self.tlog, "a")
        else:
            sto = open(self.tlog, "w")

        ts = toolshed.ToolShedInstance(
            url=self.args.toolshed_url, key=self.args.toolshed_api_key, verify=False
        )
        repos = ts.repositories.get_repositories()
        rnames = [x.get("name", "?") for x in repos]
        rids = [x.get("id", "?") for x in repos]
        tfcat = "ToolFactory generated tools"

                remote_repository_url=self.args.toolshed_url,
                homepage_url=None,
                category_ids=catID,
            )
            tid = res.get("id", None)
            sto.write(f"#create_repository {self.args.tool_name} tid={tid} res={res}\n")
        else:
            i = rnames.index(self.tool_name)
            tid = rids[i]
        try:
            res = ts.repositories.update_repository(

            "--section_label",
            "ToolFactory",
        ]
        tout.write("running\n%s\n" % " ".join(cll))
        subp = subprocess.run(
            cll, env=self.ourenv, cwd=self.ourcwd, shell=False, stderr=tout, stdout=tout
        )
        tout.write(
            "installed %s - got retcode %d\n" % (self.tool_name, subp.returncode)
        )
        tout.close()
        return subp.returncode



    def writeShedyml(self):
        """for planemo"""
        yuser = self.args.user_email.split("@")[0]
        yfname = os.path.join(self.tooloutdir, ".shed.yml")

                    tout.write(
                        "###Output file %s not found in testdir %s. This is normal during the first Planemo run that generates test outputs"
                        % (tdest, self.testdir)
                    )
        tf = tarfile.open(self.newtarpath, "w:gz")
        tf.add(name=self.tooloutdir, arcname=self.tool_name, filter=exclude_function)
        tf.close()
        shutil.copyfile(self.newtarpath, self.args.new_tool)

    def moveRunOutputs(self):
        """need to move planemo or run outputs into toolfactory collection"""

                shutil.copyfile(src, dest)


def main():
    """
    This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml

    """
    parser = argparse.ArgumentParser()
    a = parser.add_argument
    a("--script_path", default=None)

    a("--parampass", default="positional")
    a("--tfout", default="./tfout")
    a("--new_tool", default="new_tool")
    a("--galaxy_url", default="http://localhost:8080")
    a("--toolshed_url", default="http://localhost:9009")
    # make sure this is identical to tool_sheds_conf.xml  localhost != 127.0.0.1 so validation fails
    a("--toolshed_api_key", default="fakekey")
    a("--galaxy_api_key", default="fakekey")
    a("--galaxy_root", default="/galaxy-central")
    a("--galaxy_venv", default="/galaxy_venv")
    args = parser.parse_args()
    assert not args.bad_user, (
        'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file'
        % (args.bad_user, args.bad_user)
    )
    assert args.tool_name, "## Tool Factory expects a tool name - eg --tool_name=DESeq"
    assert (
        args.sysexe or args.packages
    ), "## Tool Factory wrapper expects an interpreter or an executable package"
    args.input_files = [x.replace('"', "").replace("'", "") for x in args.input_files]
    # remove quotes we need to deal with spaces in CL params
    for i, x in enumerate(args.additional_parameters):
        args.additional_parameters[i] = args.additional_parameters[i].replace('"', "")
    r = ScriptRunner(args)
    r.writeShedyml()
    r.makeTool()
    if args.make_Tool == "generate":
        retcode = r.run()  # for testing toolfactory itself
        r.moveRunOutputs()
        r.makeToolTar()
    else:
        r.planemo_biodocker_test()  # test to make outputs and then test
        r.moveRunOutputs()
        r.makeToolTar()
        if args.make_Tool == "gentestinstall":
            r.shedLoad()
            r.eph_galaxy_load()

119:8ea1133b9d9a

# July 2020: BCC was fun and I feel like rip van winkle after 5 years.
# Decided to
# 1. Fix the toolfactory so it works - done for simplest case
# 2. Fix planemo so the toolfactory function works
# 3. Rewrite bits using galaxyxml functions where that makes sense - done

import argparse
import copy
import logging
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time

from bioblend import ConnectionError
from bioblend import toolshed

import galaxyxml.tool as gxt
import galaxyxml.tool.parameters as gxtp

import lxml

verbose = True
debug = True
toolFactoryURL = "https://github.com/fubar2/toolfactory"
ourdelim = "~~~"

# --input_files="$intab.input_files~~~$intab.input_CL~~~
# $intab.input_formats# ~~~$intab.input_label
# ~~~$intab.input_help"
IPATHPOS = 0
ICLPOS = 1
IFMTPOS = 2
ILABPOS = 3
IHELPOS = 4
IOCLPOS = 5

# --output_files "$otab.history_name~~~$otab.history_format~~~
# $otab.history_CL~~~$otab.history_test"
ONAMEPOS = 0
OFMTPOS = 1
OCLPOS = 2
OTESTPOS = 3
OOCLPOS = 4


# --additional_parameters="$i.param_name~~~$i.param_value~~~
# $i.param_label~~~$i.param_help~~~$i.param_type
# ~~~$i.CL~~~i$.param_CLoverride"
ANAMEPOS = 0
AVALPOS = 1
ALABPOS = 2
AHELPPOS = 3
ATYPEPOS = 4

        return s
    except ValueError:
        return '"%s"' % s


html_escape_table = {
    "&": "&",
    ">": ">",
    "<": "&lt;",
    "#": "&#35;",
    "$": "&#36;",
}
cheetah_escape_table = {"$": "\\$", "#": "\\#"}


def html_escape(text):
    """Produce entities within text."""
    return "".join([html_escape_table.get(c, c) for c in text])


def cheetah_escape(text):
    """Produce entities within text."""
    return "".join([cheetah_escape_table.get(c, c) for c in text])

    """Revert entities within text. Multiple character targets so use replace"""
    t = text.replace("&", "&")
    t = t.replace(">", ">")
    t = t.replace("&lt;", "<")
    t = t.replace("\\$", "$")
    t = t.replace("&#36;", "$")
    t = t.replace("&#35;", "#")
    return t


def parse_citations(citations_text):
    """"""

    citation_tuples = []
    for citation in citations:
        if citation.startswith("doi"):
            citation_tuples.append(("doi", citation[len("doi") :].strip()))
        else:
            citation_tuples.append(
                ("bibtex", citation[len("bibtex") :].strip())
            )
    return citation_tuples


class ScriptRunner:
    """Wrapper for an arbitrary script

        self.is_positional = self.args.parampass == "positional"
        if self.args.sysexe:
            self.executeme = self.args.sysexe
        else:
            if self.args.packages:
                self.executeme = (
                    self.args.packages.split(",")[0].split(":")[0].strip()
                )
            else:
                self.executeme = None
        aCL = self.cl.append
        aXCL = self.xmlcl.append
        assert args.parampass in [

                aXCL(self.executeme)
                aXCL("$runme")
            else:
                aCL(self.executeme)
                aXCL(self.executeme)
        self.elog = os.path.join(
            self.repdir, "%s_error_log.txt" % self.tool_name
        )
        self.tlog = os.path.join(
            self.repdir, "%s_runner_log.txt" % self.tool_name
        )

        if self.args.parampass == "0":
            self.clsimple()
        else:
            clsuffix = []
            xclsuffix = []
            for i, p in enumerate(self.infiles):
                if p[IOCLPOS].upper() == "STDIN":
                    appendme = [
                        p[ICLPOS],
                        p[ICLPOS],
                        p[IPATHPOS],
                        "< %s" % p[IPATHPOS],
                    ]
                    xappendme = [
                        p[ICLPOS],
                        p[ICLPOS],
                        p[IPATHPOS],
                        "< $%s" % p[ICLPOS],
                    ]
                else:

            for i, p in enumerate(self.outfiles):
                if p[OOCLPOS] == "STDOUT":
                    self.lastclredirect = [">", p[ONAMEPOS]]
                    self.lastxclredirect = [">", "$%s" % p[OCLPOS]]
                else:
                    clsuffix.append([p[OCLPOS], p[ONAMEPOS], p[ONAMEPOS], ""])
                    xclsuffix.append(
                        [p[OCLPOS], p[ONAMEPOS], "$%s" % p[ONAMEPOS], ""]
                    )
            for p in self.addpar:
                clsuffix.append(
                    [p[AOCLPOS], p[ACLPOS], p[AVALPOS], p[AOVERPOS]]
                )
                xclsuffix.append(
                    [p[AOCLPOS], p[ACLPOS], '"$%s"' % p[ANAMEPOS], p[AOVERPOS]]
                )
            clsuffix.sort()
            xclsuffix.sort()

        tscript.close()
        self.escapedScript = [cheetah_escape(x) for x in rx]
        self.spacedScript = [f"    {x}" for x in rx if x.strip() > ""]
        art = "%s.%s" % (self.tool_name, self.executeme)
        artifact = open(art, "wb")
        artifact.write(bytes("\n".join(self.escapedScript), "utf8"))
        artifact.close()

    def cleanuppar(self):
        """ positional parameters are complicated by their numeric ordinal"""
        if self.args.parampass == "positional":
            for i, p in enumerate(self.infiles):
                assert (
                    p[ICLPOS].isdigit() or p[ICLPOS].strip().upper() == "STDIN"
                ), "Positional parameters must be ordinal integers - got %s for %s" % (
                    p[ICLPOS],
                    p[ILABPOS],
                )
            for i, p in enumerate(self.outfiles):
                assert (
                    p[OCLPOS].isdigit()
                    or p[OCLPOS].strip().upper() == "STDOUT"
                ), "Positional parameters must be ordinal integers - got %s for %s" % (
                    p[OCLPOS],
                    p[ONAMEPOS],
                )
            for i, p in enumerate(self.addpar):
                assert p[
                    ACLPOS
                ].isdigit(), "Positional parameters must be ordinal integers - got %s for %s" % (
                    p[ACLPOS],
                    p[ANAMEPOS],
                )
        for i, p in enumerate(self.infiles):
            infp = copy.copy(p)
            icl = infp[ICLPOS]
            infp.append(icl)
            if (
                infp[ICLPOS].isdigit()
                or self.args.parampass == "0"
                or infp[ICLPOS].strip().upper() == "STDOUT"
            ):
                scl = "input%d" % (i + 1)
                infp[ICLPOS] = scl
            self.infiles[i] = infp
        for i, p in enumerate(self.outfiles):
            p.append(p[OCLPOS])  # keep copy
            if (
                p[OOCLPOS].isdigit() and self.args.parampass != "positional"
            ) or p[OOCLPOS].strip().upper() == "STDOUT":
                scl = p[ONAMEPOS]
                p[OCLPOS] = scl
            self.outfiles[i] = p
        for i, p in enumerate(self.addpar):
            p.append(p[ACLPOS])
            if p[ACLPOS].isdigit():
                scl = "param%s" % p[ACLPOS]
                p[ACLPOS] = scl
            self.addpar[i] = p

    def clsimple(self):
        """no parameters - uses < and > for i/o"""

            aXCL(v)
        if self.lastxclredirect:
            aXCL(self.lastxclredirect[0])
            aXCL(self.lastxclredirect[1])

    def clargparse(self):
        """argparse style"""
        aCL = self.cl.append
        aXCL = self.xmlcl.append
        # inputs then params in argparse named form

            else:
                k = "--%s" % k
            aCL(k)
            aCL(v)

    def getNdash(self, newname):
        if self.is_positional:
            ndash = 0
        else:
            ndash = 2

                ndash = 1
        return ndash

    def doXMLparam(self):
        """flake8 made me do this..."""
        for (
            p
        ) in (
            self.outfiles
        ):  # --output_files "$otab.history_name~~~$otab.history_format~~~$otab.history_CL~~~$otab.history_test"
            newname, newfmt, newcl, test, oldcl = p
            test = test.strip()
            ndash = self.getNdash(newcl)
            aparm = gxtp.OutputData(
                name=newname, format=newfmt, num_dashes=ndash, label=newcl
            )
            aparm.positional = self.is_positional
            if self.is_positional:
                if oldcl.upper() == "STDOUT":
                    aparm.positional = 9999999
                    aparm.command_line_override = "> $%s" % newname

                    c = "diff"
                    ld = 0
                    if test.split(":")[1].isdigit:
                        ld = int(test.split(":")[1])
                    tp = gxtp.TestOutput(
                        name=newname,
                        value="%s_sample" % newname,
                        format=newfmt,
                        compare=c,
                        lines_diff=ld,
                    )
                elif test.startswith("sim_size"):
                    c = "sim_size"
                    tn = test.split(":")[1].strip()
                    if tn > "":
                        if "." in tn:
                            delta = None
                            delta_frac = min(1.0, float(tn))
                        else:
                            delta = int(tn)
                            delta_frac = None
                    tp = gxtp.TestOutput(
                        name=newname,
                        value="%s_sample" % newname,
                        format=newfmt,
                        compare=c,
                        delta=delta,
                        delta_frac=delta_frac,
                    )
                self.testparam.append(tp)
        for p in self.infiles:
            newname = p[ICLPOS]
            newfmt = p[IFMTPOS]
            ndash = self.getNdash(newname)

            aninput.positional = self.is_positional
            self.tinputs.append(aninput)
            tparm = gxtp.TestParam(name=newname, value="%s_sample" % newname)
            self.testparam.append(tparm)
        for p in self.addpar:
            (
                newname,
                newval,
                newlabel,
                newhelp,
                newtype,
                newcl,
                override,
                oldcl,
            ) = p
            if not len(newlabel) > 0:
                newlabel = newname
            ndash = self.getNdash(newname)
            if newtype == "text":
                aparm = gxtp.TextParam(

        Create a Galaxy xml tool wrapper for the new script
        Uses galaxyhtml
        Hmmm. How to get the command line into correct order...
        """
        if self.command_override:
            self.newtool.command_override = (
                self.command_override
            )  # config file
        else:
            self.newtool.command_override = self.xmlcl
        if self.args.help_text:
            helptext = open(self.args.help_text, "r").readlines()
            safertext = "\n".join([cheetah_escape(x) for x in helptext])
            if self.args.script_path:
                scr = [x for x in self.spacedScript if x.strip() > ""]
                scr.insert(0, "\n------\n\n\nScript::\n")
                if len(scr) > 300:
                    scr = (
                        scr[:100]
                        + ["    >300 lines - stuff deleted", "    ......"]
                        + scr[-100:]
                    )
                scr.append("\n")
                safertext = safertext + "\n".join(scr)
            self.newtool.help = safertext
        else:
            self.newtool.help = (
                "Please ask the tool author (%s) for help \

            )
        self.newtool.version_command = None  # do not want
        requirements = gxtp.Requirements()
        if self.args.packages:
            for d in self.args.packages.split(","):
                ver = ""
                d = d.replace("==", ":")
                d = d.replace("=", ":")
                if ":" in d:
                    packg, ver = d.split(":")
                else:
                    packg = d
                requirements.append(

            self.doXMLparam()
        self.newtool.outputs = self.toutputs
        self.newtool.inputs = self.tinputs
        if self.args.script_path:
            configfiles = gxtp.Configfiles()
            configfiles.append(
                gxtp.Configfile(
                    name="runme", text="\n".join(self.escapedScript)
                )
            )
            self.newtool.configfiles = configfiles
        tests = gxtp.Tests()
        test_a = gxtp.Test()
        for tp in self.testparam:
            test_a.append(tp)

        self.newtool.add_comment("Source in git at: %s" % (toolFactoryURL))
        self.newtool.add_comment(
            "Cite: Creating re-usable tools from scripts doi:10.1093/bioinformatics/bts573"
        )
        exml0 = self.newtool.export()
        exml = exml0.replace(
            FAKEEXE, ""
        )  # temporary work around until PR accepted
        if (
            self.test_override
        ):  # cannot do this inside galaxyxml as it expects lxml objects for tests
            part1 = exml.split("<tests>")[0]
            part2 = exml.split("</tests>")[1]
            fixed = "%s\n%s\n%s" % (part1, self.test_override, part2)
            exml = fixed
        # exml = exml.replace('range="1:"', 'range="1000:"')
        xf = open("%s.xml" % self.tool_name, "w")
        xf.write(exml)
        xf.write("\n")
        xf.close()
        # ready for the tarball

            if os.path.exists(self.elog):
                ste = open(self.elog, "a")
            else:
                ste = open(self.elog, "w")
            if self.lastclredirect:
                sto = open(
                    self.lastclredirect[1], "wb"
                )  # is name of an output file
            else:
                if os.path.exists(self.tlog):
                    sto = open(self.tlog, "a")
                else:
                    sto = open(self.tlog, "w")
                sto.write(
                    "## Executing Toolfactory generated command line = %s\n"
                    % scl
                )
            sto.flush()
            subp = subprocess.run(
                self.cl, env=self.ourenv, shell=False, stdout=sto, stderr=ste
            )

            else:
                sto = sys.stdout
            subp = subprocess.run(
                self.cl, env=self.ourenv, shell=False, stdout=sto, stdin=sti
            )
            sto.write(
                "## Executing Toolfactory generated command line = %s\n" % scl
            )
            retval = subp.returncode
            sto.close()
            sti.close()
        if os.path.isfile(self.tlog) and os.stat(self.tlog).st_size == 0:
            os.unlink(self.tlog)

        if retval != 0 and err:  # problem
            sys.stderr.write(err)
        logging.debug("run done")
        return retval

    def shedLoad(self):
        """
        use bioblend to create new repository
        or update existing

            sto = open(self.tlog, "a")
        else:
            sto = open(self.tlog, "w")

        ts = toolshed.ToolShedInstance(
            url=self.args.toolshed_url,
            key=self.args.toolshed_api_key,
            verify=False,
        )
        repos = ts.repositories.get_repositories()
        rnames = [x.get("name", "?") for x in repos]
        rids = [x.get("id", "?") for x in repos]
        tfcat = "ToolFactory generated tools"

                remote_repository_url=self.args.toolshed_url,
                homepage_url=None,
                category_ids=catID,
            )
            tid = res.get("id", None)
            sto.write(
                f"#create_repository {self.args.tool_name} tid={tid} res={res}\n"
            )
        else:
            i = rnames.index(self.tool_name)
            tid = rids[i]
        try:
            res = ts.repositories.update_repository(

            "--section_label",
            "ToolFactory",
        ]
        tout.write("running\n%s\n" % " ".join(cll))
        subp = subprocess.run(
            cll,
            env=self.ourenv,
            cwd=self.ourcwd,
            shell=False,
            stderr=tout,
            stdout=tout,
        )
        tout.write(
            "installed %s - got retcode %d\n"
            % (self.tool_name, subp.returncode)
        )
        tout.close()
        return subp.returncode

    def writeShedyml(self):
        """for planemo"""
        yuser = self.args.user_email.split("@")[0]
        yfname = os.path.join(self.tooloutdir, ".shed.yml")

                    tout.write(
                        "###Output file %s not found in testdir %s. This is normal during the first Planemo run that generates test outputs"
                        % (tdest, self.testdir)
                    )
        tf = tarfile.open(self.newtarpath, "w:gz")
        tf.add(
            name=self.tooloutdir,
            arcname=self.tool_name,
            filter=exclude_function,
        )
        tf.close()
        shutil.copyfile(self.newtarpath, self.args.new_tool)

    def moveRunOutputs(self):
        """need to move planemo or run outputs into toolfactory collection"""

                shutil.copyfile(src, dest)


def main():
    """
    This is a Galaxy wrapper.
    It expects to be called by a special purpose tool.xml

    """
    parser = argparse.ArgumentParser()
    a = parser.add_argument
    a("--script_path", default=None)

    a("--parampass", default="positional")
    a("--tfout", default="./tfout")
    a("--new_tool", default="new_tool")
    a("--galaxy_url", default="http://localhost:8080")
    a("--toolshed_url", default="http://localhost:9009")
    # make sure this is identical to tool_sheds_conf.xml
    # localhost != 127.0.0.1 so validation fails
    a("--toolshed_api_key", default="fakekey")
    a("--galaxy_api_key", default="fakekey")
    a("--galaxy_root", default="/galaxy-central")
    a("--galaxy_venv", default="/galaxy_venv")
    args = parser.parse_args()
    assert not args.bad_user, (
        'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy \
admin adds %s to "admin_users" in the galaxy.yml Galaxy configuration file'
        % (args.bad_user, args.bad_user)
    )
    assert (
        args.tool_name
    ), "## Tool Factory expects a tool name - eg --tool_name=DESeq"
    assert (
        args.sysexe or args.packages
    ), "## Tool Factory wrapper expects an interpreter \
or an executable package in --sysexe or --packages"
    args.input_files = [
        x.replace('"', "").replace("'", "") for x in args.input_files
    ]
    # remove quotes we need to deal with spaces in CL params
    for i, x in enumerate(args.additional_parameters):
        args.additional_parameters[i] = args.additional_parameters[i].replace(
            '"', ""
        )
    r = ScriptRunner(args)
    r.writeShedyml()
    r.makeTool()
    if args.make_Tool == "generate":
        retcode = r.run()
        r.moveRunOutputs()
        r.makeToolTar()
    else:
        retcode = r.planemo_test(genoutputs=True)  # this fails :( - see PR
        r.moveRunOutputs()
        r.makeToolTar()
        retcode = r.planemo_test(genoutputs=False)
        r.moveRunOutputs()
        r.makeToolTar()
        print(f"second planemo_test returned {retcode}")
        if args.make_Tool == "gentestinstall":
            r.shedLoad()
            r.eph_galaxy_load()