comparison jbrowse2.py @ 7:b04fd993b31e draft

planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/jbrowse2 commit 53a108d8153c955044ae7eb8cb06bdcfd0036717

6:79f7265f90bd

import xml.etree.ElementTree as ET
from collections import defaultdict

logging.basicConfig(level=logging.INFO)
log = logging.getLogger("jbrowse")
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
GALAXY_INFRASTRUCTURE_URL = None
JB2REL = "v2.10.0"
# version pinned for cloning

    return metadata


class JbrowseConnector(object):
    def __init__(self, outdir, genomes):
        self.debug = False
        self.usejson = True
        self.giURL = GALAXY_INFRASTRUCTURE_URL
        self.outdir = outdir
        os.makedirs(self.outdir, exist_ok=True)
        self.genome_paths = genomes

        self.config_json_file = os.path.join(outdir, "config.json")
        self.clone_jbrowse()

    def subprocess_check_call(self, command, output=None):
        if output:
            if self.debug:
                log.debug("cd %s && %s >  %s", self.outdir, " ".join(command), output)
            subprocess.check_call(command, cwd=self.outdir, stdout=output)
        else:
            log.debug("cd %s && %s", self.outdir, " ".join(command))
            subprocess.check_call(command, cwd=self.outdir)

    def subprocess_popen(self, command):
        if self.debug:
            log.debug(command)
        p = subprocess.Popen(
            command,
            cwd=self.outdir,
            shell=True,
            stdin=subprocess.PIPE,

            log.error(output)
            log.error(err)
            raise RuntimeError("Command failed with exit code %s" % (retcode))

    def subprocess_check_output(self, command):
        if self.debug:
            log.debug(" ".join(command))
        return subprocess.check_output(command, cwd=self.outdir)

    def symlink_or_copy(self, src, dest):
        if "GALAXY_JBROWSE_SYMLINKS" in os.environ and bool(
            os.environ["GALAXY_JBROWSE_SYMLINKS"]

                self.config_json["assemblies"] += assemblies
            else:
                self.config_json["assemblies"] = assemblies

    def make_assembly(self, fapath, gname):
        faname = gname + ".fa.gz"
        fadest = os.path.join(self.outdir, faname)
        # fadest = os.path.realpath(os.path.join(self.outdir, faname))
        cmd = "bgzip -i -c %s -I %s.gzi > %s && samtools faidx %s" % (
            fapath,
            fadest,
            fadest,
            fadest,

        """
        tId = trackData["label"]
        # can be served - if public.
        # dsId = trackData["metadata"]["dataset_id"]
        # url = "%s/api/datasets/%s/display?to_ext=hic " % (self.giURL, dsId)
        hname = trackData["name"]
        dest = os.path.join(self.outdir, hname)
        cmd = ["cp", data, dest]
        # these can be very big.
        self.subprocess_check_call(cmd)
        floc = {

            ]
        }
        tId = trackData["label"]
        fname = "%s.bed" % tId
        dest = "%s/%s" % (self.outdir, fname)
        # self.symlink_or_copy(data, dest)
        # Process MAF to bed-like. Need build to munge chromosomes
        gname = self.genome_name
        cmd = [
            "bash",
            os.path.join(INSTALLED_TO, "convertMAF.sh"),
            data,

        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

        os.unlink(gff3)

    def add_bigwig(self, data, trackData):
        url = "%s.bw" % trackData["name"]
        dest = os.path.join(self.outdir, url)
        cmd = ["cp", data, dest]
        self.subprocess_check_call(cmd)
        bwloc = {"uri": url}
        tId = trackData["label"]
        trackDict = {
            "type": "QuantitativeTrack",
            "trackId": tId,
            "name": url,
            "assemblyNames": [
                self.genome_name,
            ],
            "adapter": {
                "type": "BigWigAdapter",

        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_bam(self, data, trackData, bamOpts, bam_index=None, **kwargs):
        tId = trackData["label"]
        fname = "%s.bam" % trackData["label"]
        dest = "%s/%s" % (self.outdir, fname)

        else:
            self.config_json["assemblies"] = [
                asstrack,
            ]

        style_json = self._prepare_track_style(trackData)
        url = "%s.paf" % (trackData["label"])
        dest = "%s/%s" % (self.outdir, url)
        self.symlink_or_copy(os.path.realpath(data), dest)

        if self.usejson:
            trackDict = {
                "type": "SyntenyTrack",
                "trackId": tId,
                "assemblyNames": [self.genome_name, pgname],
                "name": tname,
                "adapter": {
                    "type": "PAFAdapter",
                    "pafLocation": {"uri": url},
                    "assemblyNames": [self.genome_name, pgname],
                },
                "config": style_json,
            }
            self.tracksToAdd.append(trackDict)
            self.trackIdlist.append(tId)
        else:
            self._add_track(
                trackData["label"],
                trackData["key"],
                trackData["category"],
                dest,
                assemblies=[self.genome_name, pgname],
                config=style_json,
            )

    def add_hicab(self, data, trackData, hicOpts, **kwargs):
        rel_dest = os.path.join("data", trackData["label"] + ".hic")
        dest = os.path.join(self.outdir, rel_dest)

        self.symlink_or_copy(os.path.realpath(data), dest)

        style_json = self._prepare_track_style(trackData)

        self._add_track(
            trackData["label"],
            trackData["key"],
            trackData["category"],
            rel_dest,
            config=style_json,
        )

    def add_sparql(self, url, query, query_refnames, trackData):

        json_track_data = {

                "category": category,
                "style": {},
            }

            outputTrackConfig["key"] = track_human_label
            if self.debug:
                log.info(
                    "Processing category = %s, track_human_label = %s",
                    category,
                    track_human_label,
                )
            # We add extra data to hash for the case of REST + SPARQL.
            if (
                "conf" in track
                and "options" in track["conf"]
                and "url" in track["conf"]["options"]
            ):
                rest_url = track["conf"]["options"]["url"]
            else:
                rest_url = ""

            # I chose to use track['category'] instead of 'category' here. This
            # is intentional. This way re-running the tool on a different date
            # will not generate different hashes and make comparison of outputs
            # much simpler.
            hashData = [

        with open(self.config_json_file, "r") as config_file:
            config_json = json.load(config_file)

        for track_conf in self.tracksToAdd:
            track_types[track_conf["trackId"]] = track_conf["type"]

        for on_track in data["visibility"]["default_on"]:
            style_data = {"type": "LinearBasicDisplay", "height": 100}
            if on_track in data["style"]:
                if "display" in data["style"][on_track]:
                    style_data["type"] = data["style"][on_track]["display"]
                    del data["style"][on_track]["display"]
                style_data.update(data["style"][on_track])
            if on_track in data["style_labels"]:
                # TODO fix this: it should probably go in a renderer block (SvgFeatureRenderer) but still does not work
                # TODO move this to per track displays?
                style_data["labels"] = data["style_labels"][on_track]

            tracks_data.append(
                {
                    "type": track_types[on_track],
                    "configuration": on_track,
                    "displays": [style_data],
                }
            )

        # The view for the assembly we're adding
        view_json = {"type": "LinearGenomeView", "tracks": tracks_data}

        refName = None

                start = int(loc_match.group(2))
                end = int(loc_match.group(3))
        elif self.genome_name is not None:
            refName = self.genome_name
            start = 0
            end = 100000  # Booh, hard coded! waiting for https://github.com/GMOD/jbrowse-components/issues/2708

        if refName is not None:
            # TODO displayedRegions is not just zooming to the region, it hides the rest of the chromosome
            view_json["displayedRegions"] = [
                {

        with open(config_path, "w") as config_file:
            json.dump(config_json, config_file, indent=2)

    def clone_jbrowse(self):
        """Clone a JBrowse directory into a destination directory."""
        dest = self.outdir
        cmd = ["jbrowse", "create", "-t", JB2REL, "-f", dest]
        self.subprocess_check_call(cmd)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",
            "robots.txt",

            "version.txt",
            "test_data",
        ]:
            cmd = ["rm", "-rf", os.path.join(self.outdir, fn)]
            self.subprocess_check_call(cmd)
        cmd = ["cp", os.path.join(INSTALLED_TO, "webserver.py"), self.outdir]
        self.subprocess_check_call(cmd)


def parse_style_conf(item):
    if "type" in item.attrib and item.attrib["type"] in ["boolean", "integer"]:

                    "bigwig_multiple",
                    "MultiBigWig",  # Giving an hardcoded name for now
                    {},  # No metadata for multiple bigwig
                )
            )

        track_conf["category"] = track.attrib["cat"]
        track_conf["format"] = track.attrib["format"]
        track_conf["style"] = {
            item.tag: parse_style_conf(item) for item in track.find("options/style")
        }

        track_conf["style"] = {
            item.tag: parse_style_conf(item) for item in track.find("options/style")
        }

        track_conf["style_labels"] = {
            item.tag: parse_style_conf(item)
            for item in track.find("options/style_labels")
        }

        track_conf["conf"] = etree_to_dict(track.find("options"))
        keys = jc.process_annotations(track_conf)

        if keys:
            for key in keys:
                default_session_data["visibility"][
                    track.attrib.get("visibility", "default_off")
                ].append(key)
                default_session_data["style"][key] = track_conf[
                    "style"
                ]  # TODO do we need this anymore?
                default_session_data["style_labels"][key] = track_conf["style_labels"]

        default_session_data["defaultLocation"] = root.find(
            "metadata/general/defaultLocation"
        ).text
        default_session_data["session_name"] = root.find(

        jc.add_general_configuration(general_data)
    x = open(args.xml, "r").read()
    jc.config_json["tracks"] = jc.tracksToAdd
    if jc.usejson:
        jc.write_config()
    # jc.add_default_view()
    jc.add_default_session(default_session_data)

    # jc.text_index() not sure what broke here.

7:b04fd993b31e

import xml.etree.ElementTree as ET
from collections import defaultdict

logging.basicConfig(level=logging.INFO)
log = logging.getLogger("jbrowse")

JB2VER = "v2.10.0"
# version pinned for cloning

TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
GALAXY_INFRASTRUCTURE_URL = None
JB2REL = "v2.10.0"
# version pinned for cloning

    return metadata


class JbrowseConnector(object):
    def __init__(self, outdir, genomes):
        self.usejson = True
        self.giURL = GALAXY_INFRASTRUCTURE_URL
        self.outdir = outdir
        os.makedirs(self.outdir, exist_ok=True)
        self.genome_paths = genomes

        self.config_json_file = os.path.join(outdir, "config.json")
        self.clone_jbrowse()

    def subprocess_check_call(self, command, output=None):
        if output:
            log.debug("cd %s && %s >  %s", self.outdir, " ".join(command), output)
            subprocess.check_call(command, cwd=self.outdir, stdout=output)
        else:
            log.debug("cd %s && %s", self.outdir, " ".join(command))
            subprocess.check_call(command, cwd=self.outdir)

    def subprocess_popen(self, command):
        log.debug(command)
        p = subprocess.Popen(
            command,
            cwd=self.outdir,
            shell=True,
            stdin=subprocess.PIPE,

            log.error(output)
            log.error(err)
            raise RuntimeError("Command failed with exit code %s" % (retcode))

    def subprocess_check_output(self, command):
        log.debug(" ".join(command))
        return subprocess.check_output(command, cwd=self.outdir)

    def symlink_or_copy(self, src, dest):
        if "GALAXY_JBROWSE_SYMLINKS" in os.environ and bool(
            os.environ["GALAXY_JBROWSE_SYMLINKS"]

                self.config_json["assemblies"] += assemblies
            else:
                self.config_json["assemblies"] = assemblies

    def make_assembly(self, fapath, gname):
        hashData = [
            fapath,
            gname,
        ]
        hashData = "|".join(hashData).encode("utf-8")
        ghash = hashlib.md5(hashData).hexdigest()
        faname = ghash + ".fa.gz"
        fadest = os.path.join(self.outdir, faname)
        cmd = "bgzip -i -c %s -I %s.gzi > %s && samtools faidx %s" % (
            fapath,
            fadest,
            fadest,
            fadest,

        """
        tId = trackData["label"]
        # can be served - if public.
        # dsId = trackData["metadata"]["dataset_id"]
        # url = "%s/api/datasets/%s/display?to_ext=hic " % (self.giURL, dsId)
        hname = trackData["label"]
        dest = os.path.join(self.outdir, hname)
        cmd = ["cp", data, dest]
        # these can be very big.
        self.subprocess_check_call(cmd)
        floc = {

            ]
        }
        tId = trackData["label"]
        fname = "%s.bed" % tId
        dest = "%s/%s" % (self.outdir, fname)
        gname = self.genome_name
        cmd = [
            "bash",
            os.path.join(INSTALLED_TO, "convertMAF.sh"),
            data,

        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)
        os.unlink(gff3)

    def add_bigwig(self, data, trackData):
        url = "%s.bigwig" % trackData["label"]
        # slashes in names cause path trouble
        dest = os.path.join(self.outdir, url)
        cmd = ["cp", data, dest]
        self.subprocess_check_call(cmd)
        bwloc = {"uri": url}
        tId = trackData["label"]
        trackDict = {
            "type": "QuantitativeTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [
                self.genome_name,
            ],
            "adapter": {
                "type": "BigWigAdapter",

        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)
        logging.debug("#### wig trackData=%s" % str(trackData))

    def add_bam(self, data, trackData, bamOpts, bam_index=None, **kwargs):
        tId = trackData["label"]
        fname = "%s.bam" % trackData["label"]
        dest = "%s/%s" % (self.outdir, fname)

        else:
            self.config_json["assemblies"] = [
                asstrack,
            ]

        url = "%s.paf" % (trackData["label"])
        dest = "%s/%s" % (self.outdir, url)
        self.symlink_or_copy(os.path.realpath(data), dest)
        trackDict = {
            "type": "SyntenyTrack",
            "trackId": tId,
            "assemblyNames": [self.genome_name, pgname],
            "name": tname,
            "adapter": {
                "type": "PAFAdapter",
                "pafLocation": {"uri": url},
                "assemblyNames": [self.genome_name, pgname],
            },
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_hicab(self, data, trackData, hicOpts, **kwargs):
        rel_dest = os.path.join("data", trackData["label"] + ".hic")
        dest = os.path.join(self.outdir, rel_dest)

        self.symlink_or_copy(os.path.realpath(data), dest)

        self._add_track(
            trackData["label"],
            trackData["key"],
            trackData["category"],
            rel_dest,
            config={},
        )

    def add_sparql(self, url, query, query_refnames, trackData):

        json_track_data = {

                "category": category,
                "style": {},
            }

            outputTrackConfig["key"] = track_human_label

            # We add extra data to hash for the case of REST + SPARQL.
            if (
                "conf" in track
                and "options" in track["conf"]
                and "url" in track["conf"]["options"]
            ):
                rest_url = track["conf"]["options"]["url"]
            else:
                rest_url = ""
            outputTrackConfig["trackset"] = track.get("trackset", {})
            # I chose to use track['category'] instead of 'category' here. This
            # is intentional. This way re-running the tool on a different date
            # will not generate different hashes and make comparison of outputs
            # much simpler.
            hashData = [

        with open(self.config_json_file, "r") as config_file:
            config_json = json.load(config_file)

        for track_conf in self.tracksToAdd:
            track_types[track_conf["trackId"]] = track_conf["type"]
            tId = track_conf["trackId"]
            if tId in data["visibility"]["default_on"]:
                style_data = {"type": "LinearBasicDisplay"}
                if "displays" in track_conf:
                    style_data["type"] = track_conf["displays"][0]["type"]
                if track_conf.get("style_labels", None):
                    # TODO fix this: it should probably go in a renderer block (SvgFeatureRenderer) but still does not work
                    # TODO move this to per track displays?
                    style_data["labels"] = track_conf["style_labels"]
                tracks_data.append(
                    {
                        "type": track_types[tId],
                        "configuration": tId,
                        "displays": [style_data],
                    }
                )

        # The view for the assembly we're adding
        view_json = {"type": "LinearGenomeView", "tracks": tracks_data}

        refName = None

                start = int(loc_match.group(2))
                end = int(loc_match.group(3))
        elif self.genome_name is not None:
            refName = self.genome_name
            start = 0
            end = 10000  # Booh, hard coded! waiting for https://github.com/GMOD/jbrowse-components/issues/2708

        if refName is not None:
            # TODO displayedRegions is not just zooming to the region, it hides the rest of the chromosome
            view_json["displayedRegions"] = [
                {

        with open(config_path, "w") as config_file:
            json.dump(config_json, config_file, indent=2)

    def clone_jbrowse(self):
        """Clone a JBrowse directory into a destination directory."""
        # dest = os.path.realpath(self.outdir)
        dest = self.outdir
        cmd = ["rm", "-rf", dest + "/*"]
        self.subprocess_check_call(cmd)
        cmd = ["jbrowse", "create", dest, "-t", JB2VER, "-f"]
        self.subprocess_check_call(cmd)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",
            "robots.txt",

            "version.txt",
            "test_data",
        ]:
            cmd = ["rm", "-rf", os.path.join(self.outdir, fn)]
            self.subprocess_check_call(cmd)
        cmd = ["cp", os.path.join(INSTALLED_TO, "jb2_webserver.py"), self.outdir]
        self.subprocess_check_call(cmd)


def parse_style_conf(item):
    if "type" in item.attrib and item.attrib["type"] in ["boolean", "integer"]:

                    "bigwig_multiple",
                    "MultiBigWig",  # Giving an hardcoded name for now
                    {},  # No metadata for multiple bigwig
                )
            )
        track_conf["category"] = track.attrib["cat"]
        track_conf["format"] = track.attrib["format"]
        if track.find("options/style"):
            track_conf["style"] = {
                item.tag: parse_style_conf(item) for item in track.find("options/style")
            }
        if track.find("options/style_labels"):
            track_conf["style_labels"] = {
                item.tag: parse_style_conf(item)
                for item in track.find("options/style_labels")
            }

        track_conf["conf"] = etree_to_dict(track.find("options"))
        keys = jc.process_annotations(track_conf)

        if keys:
            for key in keys:
                default_session_data["visibility"][
                    track.attrib.get("visibility", "default_off")
                ].append(key)
                if track_conf.get("style", None):
                    default_session_data["style"][key] = track_conf[
                        "style"
                    ]  # TODO do we need this anymore?
                if track_conf.get("style_lables", None):
                    default_session_data["style_labels"][key] = track_conf.get(
                        "style_labels", None
                    )

        default_session_data["defaultLocation"] = root.find(
            "metadata/general/defaultLocation"
        ).text
        default_session_data["session_name"] = root.find(

        jc.add_general_configuration(general_data)
    x = open(args.xml, "r").read()
    jc.config_json["tracks"] = jc.tracksToAdd
    if jc.usejson:
        jc.write_config()
    jc.add_default_session(default_session_data)

    # jc.text_index() not sure what broke here.