comparison jbrowse2.py @ 98:b1260bca5fdc draft

planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/jbrowse2 commit 44d8fc559ecf5463a8f753561976fa26686c96f6

97:74074746ccd8

#!/usr/bin/env python

import argparse
import binascii
import datetime
import json
import logging
import os
import re
import shutil

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger("jbrowse")

JB2VER = "v2.11.0"
# version pinned if cloning - but not cloning now
logCommands = True
# useful for seeing what's being written but NOT for production setups
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
SELF_LOCATION = os.path.dirname(os.path.realpath(__file__))
GALAXY_INFRASTRUCTURE_URL = None
mapped_chars = {
    ">": "__gt__",

    "}": "__cc__",
    "@": "__at__",
    "#": "__pd__",
    "": "__cn__",
}


class ColorScaling(object):

    COLOR_FUNCTION_TEMPLATE = """

    def get_cwd(self, cwd):
        if cwd:
            return self.outdir
        else:
            return subprocess.check_output(["pwd"]).decode("utf-8").strip()
            # return None

    def subprocess_check_call(self, command, output=None, cwd=True):
        if output:
            if logCommands:
                log.debug(

        retcode = p.returncode
        if retcode != 0:
            log.error(command)
            log.error(output)
            log.error(err)
            raise RuntimeError("Command failed with exit code %s" % (retcode))

    def subprocess_check_output(self, command):
        if logCommands:
            log.debug(" ".join(command))
        return subprocess.check_output(command, cwd=self.outdir)

                with urllib.request.urlopen(url, context=scontext) as f:
                    fl = f.readlines()
                nrow = len(fl)
            except Exception:
                nrow = 0
        logging.debug("### getNrow %s returning %d" % (url, nrow))
        return nrow

    def process_genomes(self, genomes):
        assembly = []
        assmeta = []

        self.assmeta[primaryGenome] = assmeta
        self.tracksToAdd[primaryGenome] = []
        return primaryGenome

    def make_assembly(self, fapath, gname, useuri):
        """added code to grab the first contig name and length for broken default session from Anthony and Helena's code
        that poor Bjoern is trying to figure out.
        """
        if useuri:
            faname = fapath
            scontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
            scontext.check_hostname = False
            scontext.verify_mode = ssl.VerifyMode.CERT_NONE

                    "url": "https://unpkg.com/jbrowse-plugin-mafviewer/dist/jbrowse-plugin-mafviewer.umd.production.min.js",
                }
            ]
        }
        categ = trackData["category"]
        fname = tId
        dest = "%s/%s" % (self.outdir, fname)
        gname = trackData["assemblyNames"]

        cmd = [
            "bash",
            os.path.join(INSTALLED_TO, "convertMAF.sh"),

        self.tracksToAdd[trackData["assemblyNames"]].append(trackDict)
        self.trackIdlist.append(tId)

    def add_vcf(self, data, trackData):
        tId = trackData["label"]
        # url = "%s/api/datasets/%s/display" % (
        # self.giURL,
        # trackData["metadata"]["dataset_id"],
        # )
        categ = trackData["category"]
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:

                track_human_label = track_human_label.replace(" ", "_")
            outputTrackConfig = {
                "category": category,
                "style": {},
            }
            outputTrackConfig["assemblyNames"] = track["assemblyNames"]
            outputTrackConfig["key"] = track_human_label
            outputTrackConfig["useuri"] = useuri
            outputTrackConfig["path"] = dataset_path
            outputTrackConfig["ext"] = dataset_ext

            outputTrackConfig["trackset"] = track.get("trackset", {})
            outputTrackConfig["label"] = track["label"]
            outputTrackConfig["metadata"] = extra_metadata
            outputTrackConfig["name"] = track_human_label
            if track["label"] in self.trackIdlist:
                logging.error(
                    "### not adding %s already in %s"

                        "hic",
                        dataset_path,
                        hic_path,
                    ]
                )
                logging.debug(
                    "### ext=cool: wasCool=%s, hic_path=%s"
                    % (outputTrackConfig["wasCool"], hic_path)
                )
                self.add_hic(
                    hic_path,
                    outputTrackConfig,
                )
            elif dataset_ext in ("bed",):

                    dataset_path,
                    outputTrackConfig,
                )
            elif dataset_ext == "bam":
                real_indexes = track["conf"]["options"]["bam"]["bam_index"]
                logging.debug("**** add bam got %s for indexes" % real_indexes)
                self.add_bam(
                    dataset_path,
                    outputTrackConfig,
                    bam_indexes=real_indexes,
                )
            elif dataset_ext == "cram":
                real_indexes = track["conf"]["options"]["cram"]["cram_index"]
                logging.debug("**** add cram got %s for indexes" % real_indexes)
                self.add_cram(
                    dataset_path,
                    outputTrackConfig,
                    cram_indexes=real_indexes,
                )

                    dataset_path,
                    outputTrackConfig,
                    track["conf"]["options"]["paf"],
                )
            else:
                logging.warn("Do not know how to handle %s", dataset_ext)
            # Return non-human label for use in other fields
            yield outputTrackConfig["label"]

    def add_default_session(self, default_data):
        """
        default session settings are hard and fragile.
        .add_default_view() and other configuration code adapted from
         https://github.com/abretaud/tools-iuc/blob/jbrowse2/tools/jbrowse2/jbrowse2.py
        """
        # TODO using the default session for now, but check out session specs in the future https://github.com/GMOD/jbrowse-components/issues/2708
        bpPerPx = 50 # this is tricky since browser window width is unknown - this seems a compromise that sort of works....
        track_types = {}
        with open(self.config_json_file, "r") as config_file:
            config_json = json.load(config_file)
        if self.config_json:
            config_json.update(self.config_json)

                            "type": track_types[tId],
                            "configuration": tId,
                            "displays": [style_data],
                        }
                    )
            first = [x for x in self.ass_first_contigs if x[0] == gnome]
            drdict = {
                    "reversed": False,
                    "assemblyName": gnome,
                }
            if len(first) > 0:
                [gnome, refName, end] = first[0]
                drdict["refName"] = refName
                drdict["start"] = 0
                end = int(end)
                drdict["end"] = end        
            else:
                ddl = default_data.get("defaultLocation", None)
                if ddl:
                    loc_match = re.search(r"^([^:]+):([\d,]*)\.*([\d,]*)$", ddl)
                    # allow commas like 100,000 but ignore as integer

                    else:
                        logging.info(
                            "@@@ regexp could not match contig:start..end in the supplied location %s - please fix"
                            % ddl
                        )
            view_json = {
                "type": "LinearGenomeView",
                "offsetPx": 0,
                "bpPerPx" : bpPerPx,
                "minimized": False,
                "tracks": tracks_data
            }
            if drdict.get("refName", None):
                # TODO displayedRegions is not just zooming to the region, it hides the rest of the chromosome
                view_json["displayedRegions"] = [
                    drdict,
                ]

            session_views.append(view_json)
        session_name = default_data.get("session_name", "New session")
        for key, value in mapped_chars.items():
            session_name = session_name.replace(value, key)
        session_json["name"] = session_name
        if "views" not in session_json:
            session_json["views"] = session_views
        else:
            session_json["views"] += session_views
        pp = json.dumps(session_views, indent=2)
        config_json["defaultSession"] = session_json
        self.config_json.update(config_json)
        logging.debug("defaultSession=%s" % (pp))
        with open(self.config_json_file, "w") as config_file:

        Xrefs to understand the choices:
        https://github.com/GMOD/jbrowse-components/issues/2708
        https://github.com/GMOD/jbrowse-components/discussions/3568
        https://github.com/GMOD/jbrowse-components/pull/4148
        """


        INDEX_TEMPLATE = """<!doctype html>
        <html lang="en" style="height:100%">
        <head>
        <meta charset="utf-8"/>
        <link rel="shortcut icon" href="./favicon.ico"/>
        <meta name="viewport" content="width=device-width,initial-scale=1"/>
        <meta name="theme-color" content="#000000"/>
        <meta name="description" content="A fast and flexible genome browser"/>
        <link rel="manifest" href="./manifest.json"/>
        <title>JBrowse</title>
        </script>
        </head>
        <body style="overscroll-behavior:none; height:100%; margin: 0;">
        <iframe
        id="jbframe"
        title="JBrowse2"
        frameborder="0"
        width="100%"
        height="100%"
        src='index_noview.html?config=config.json__SESSION_SPEC__'>
        </iframe>
        </body>
        </html>
        """

        new_index = "Nothing written"
        session_spec = {"views": []}
        logging.debug("def ass_first=%s\ndata=%s" % (self.ass_first_contigs, data))
        for first_contig in self.ass_first_contigs:
            logging.debug("first contig=%s" % self.ass_first_contigs)

        self.config_json.update(config_json)
        with open(config_path, "w") as config_file:
            json.dump(self.config_json, config_file, indent=2)

    def clone_jbrowse(self, realclone=False):
        """Clone a JBrowse directory into a destination directory. This also works in Biocontainer testing now
        Leave as True between version updates on temporary tools - requires manual conda trigger :(
        """
        dest = self.outdir
        if realclone:
            self.subprocess_check_call(
                ["jbrowse", "create", dest, "-f", "--tag", f"{JB2VER}"]

98:b1260bca5fdc

#!/usr/bin/env python

import argparse
import binascii
import datetime
# import hashlib
import json
import logging
import os
import re
import shutil

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger("jbrowse")

JB2VER = "v2.11.0"
# version pinned if cloning - but not used until now
logCommands = True
# useful for seeing what's being written but not for production setups
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
SELF_LOCATION = os.path.dirname(os.path.realpath(__file__))
GALAXY_INFRASTRUCTURE_URL = None
mapped_chars = {
    ">": "__gt__",

    "}": "__cc__",
    "@": "__at__",
    "#": "__pd__",
    "": "__cn__",
}


INDEX_TEMPLATE = """<!doctype html>
<html lang="en" style="height:100%">
<head>
<meta charset="utf-8"/>
<link rel="shortcut icon" href="./favicon.ico"/>
<meta name="viewport" content="width=device-width,initial-scale=1"/>
<meta name="theme-color" content="#000000"/>
<meta name="description" content="A fast and flexible genome browser"/>
<link rel="manifest" href="./manifest.json"/>
<title>JBrowse</title>
</script>
</head>
<body style="overscroll-behavior:none; height:100%; margin: 0;">
<iframe
  id="jbframe"
  title="JBrowse2"
  frameborder="0"
  width="100%"
  height="100%"
  src='index_noview.html?config=config.json__SESSION_SPEC__'>
</iframe>
</body>
</html>
"""


class ColorScaling(object):

    COLOR_FUNCTION_TEMPLATE = """

    def get_cwd(self, cwd):
        if cwd:
            return self.outdir
        else:
            return subprocess.check_output(["pwd"]).decode("utf-8").strip()

    def subprocess_check_call(self, command, output=None, cwd=True):
        if output:
            if logCommands:
                log.debug(

        retcode = p.returncode
        if retcode != 0:
            log.error(command)
            log.error(output)
            log.error(err)
            raise RuntimeError(f"Command ( {command} ) failed with exit code {retcode}")

    def subprocess_check_output(self, command):
        if logCommands:
            log.debug(" ".join(command))
        return subprocess.check_output(command, cwd=self.outdir)

                with urllib.request.urlopen(url, context=scontext) as f:
                    fl = f.readlines()
                nrow = len(fl)
            except Exception:
                nrow = 0
        logging.debug("getNrow %s returning %d" % (url, nrow))
        return nrow

    def process_genomes(self, genomes):
        assembly = []
        assmeta = []

        self.assmeta[primaryGenome] = assmeta
        self.tracksToAdd[primaryGenome] = []
        return primaryGenome

    def make_assembly(self, fapath, gname, useuri):
        if useuri:
            faname = fapath
            scontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
            scontext.check_hostname = False
            scontext.verify_mode = ssl.VerifyMode.CERT_NONE

                    "url": "https://unpkg.com/jbrowse-plugin-mafviewer/dist/jbrowse-plugin-mafviewer.umd.production.min.js",
                }
            ]
        }
        categ = trackData["category"]
        fname = f"{tId}"
        dest = os.path.join(self.outdir, fname)
        gname = trackData["assemblyNames"]

        cmd = [
            "bash",
            os.path.join(INSTALLED_TO, "convertMAF.sh"),

        self.tracksToAdd[trackData["assemblyNames"]].append(trackDict)
        self.trackIdlist.append(tId)

    def add_vcf(self, data, trackData):
        tId = trackData["label"]
        categ = trackData["category"]
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:

                track_human_label = track_human_label.replace(" ", "_")
            outputTrackConfig = {
                "category": category,
                "style": {},
            }

            # hashData = [
            #    str(dataset_path),
            #    track_human_label,
            #    track["category"],
            # ]
            # hashData = "|".join(hashData).encode("utf-8")
            # hash_string = hashlib.md5(hashData).hexdigest()

            outputTrackConfig["assemblyNames"] = track["assemblyNames"]
            outputTrackConfig["key"] = track_human_label
            outputTrackConfig["useuri"] = useuri
            outputTrackConfig["path"] = dataset_path
            outputTrackConfig["ext"] = dataset_ext
            outputTrackConfig["trackset"] = track.get("trackset", {})
            outputTrackConfig["label"] = track["label"]
            # outputTrackConfig["label"] = "%s_%i_%s_%s" % (
            #    dataset_ext,
            #    trackIndex,
            #    track_human_label,
            #    hash_string,
            # )

            outputTrackConfig["metadata"] = extra_metadata
            outputTrackConfig["name"] = track_human_label
            if track["label"] in self.trackIdlist:
                logging.error(
                    "### not adding %s already in %s"

                        "hic",
                        dataset_path,
                        hic_path,
                    ]
                )
                self.add_hic(
                    hic_path,
                    outputTrackConfig,
                )
            elif dataset_ext in ("bed",):

                    dataset_path,
                    outputTrackConfig,
                )
            elif dataset_ext == "bam":
                real_indexes = track["conf"]["options"]["bam"]["bam_index"]
                self.add_bam(
                    dataset_path,
                    outputTrackConfig,
                    bam_indexes=real_indexes,
                )
            elif dataset_ext == "cram":
                real_indexes = track["conf"]["options"]["cram"]["cram_index"]
                self.add_cram(
                    dataset_path,
                    outputTrackConfig,
                    cram_indexes=real_indexes,
                )

                    dataset_path,
                    outputTrackConfig,
                    track["conf"]["options"]["paf"],
                )
            else:
                logging.warning("Do not know how to handle %s", dataset_ext)
            # Return non-human label for use in other fields
            yield outputTrackConfig["label"]

    def add_default_session(self, default_data):
        """
        default session settings are hard and fragile.
        .add_default_view() and other configuration code adapted from
         https://github.com/abretaud/tools-iuc/blob/jbrowse2/tools/jbrowse2/jbrowse2.py
        """
        # TODO using the default session for now, but check out session specs in the future https://github.com/GMOD/jbrowse-components/issues/2708
        track_types = {}
        with open(self.config_json_file, "r") as config_file:
            config_json = json.load(config_file)
        if self.config_json:
            config_json.update(self.config_json)

                            "type": track_types[tId],
                            "configuration": tId,
                            "displays": [style_data],
                        }
                    )
            view_json = {
                "type": "LinearGenomeView",
                "offsetPx": 0,
                "minimized": False,
                "tracks": tracks_data,
            }
            first = [x for x in self.ass_first_contigs if x[0] == gnome]
            if len(first) > 0:
                [gnome, refName, end] = first[0]
                start = 0
                end = int(end)
                drdict = {
                    "refName": refName,
                    "start": start,
                    "end": end,
                    "reversed": False,
                    "assemblyName": gnome,
                }
            else:
                ddl = default_data.get("defaultLocation", None)
                if ddl:
                    loc_match = re.search(r"^([^:]+):([\d,]*)\.*([\d,]*)$", ddl)
                    # allow commas like 100,000 but ignore as integer

                    else:
                        logging.info(
                            "@@@ regexp could not match contig:start..end in the supplied location %s - please fix"
                            % ddl
                        )
            if drdict.get("refName", None):
                # TODO displayedRegions is not just zooming to the region, it hides the rest of the chromosome
                view_json["displayedRegions"] = [
                    drdict,
                ]

            session_views.append(view_json)
        session_name = default_data.get("session_name", "New session")
        for key, value in mapped_chars.items():
            session_name = session_name.replace(value, key)
        session_json["name"] = session_name

        if "views" not in session_json:
            session_json["views"] = session_views
        else:
            session_json["views"] += session_views

        pp = json.dumps(session_views, indent=2)
        config_json["defaultSession"] = session_json
        self.config_json.update(config_json)
        logging.debug("defaultSession=%s" % (pp))
        with open(self.config_json_file, "w") as config_file:

        Xrefs to understand the choices:
        https://github.com/GMOD/jbrowse-components/issues/2708
        https://github.com/GMOD/jbrowse-components/discussions/3568
        https://github.com/GMOD/jbrowse-components/pull/4148
        """
        new_index = "Nothing written"
        session_spec = {"views": []}
        logging.debug("def ass_first=%s\ndata=%s" % (self.ass_first_contigs, data))
        for first_contig in self.ass_first_contigs:
            logging.debug("first contig=%s" % self.ass_first_contigs)

        self.config_json.update(config_json)
        with open(config_path, "w") as config_file:
            json.dump(self.config_json, config_file, indent=2)

    def clone_jbrowse(self, realclone=False):
        """
            Clone a JBrowse directory into a destination directory.

            `realclone=true` will use the `jbrowse create` command.
            To allow running on internet-less compute and for reproducibility
            use frozen code with `realclone=false

        """
        dest = self.outdir
        if realclone:
            self.subprocess_check_call(
                ["jbrowse", "create", dest, "-f", "--tag", f"{JB2VER}"]