comparison jbrowse2.py @ 57:94264fe60478 draft

planemo upload for repository https://github.com/usegalaxy-eu/temporary-tools/tree/master/jbrowse2 commit 4b5df41484f6bdf316edaf95b53c92d328ec1674-dirty

56:c0097a584a8a

#!/usr/bin/env python
# change to accumulating all configuration for config.json based on the default from the clone
import argparse
import binascii
import datetime
import json

import os
import re
import shutil
import struct
import subprocess
import sys
import tempfile
import urllib.request
import xml.etree.ElementTree as ET
from collections import defaultdict

                            if len(fl.split()) > 1:
                                self.genome_firstcontig = fl.split()[0].strip()
                            else:
                                self.genome_firstcontig = fl
                    else:
                        fl = urllib.request.urlopen(fapath+".fai").readline()
                        if fl: # is first row of the text fai so the first contig name
                            self.genome_firstcontig = fl.decode('utf8').strip().split()[0]
        if self.config_json.get("assemblies", None):
            self.config_json["assemblies"] += assemblies
        else:
            self.config_json["assemblies"] = assemblies

                "type": "ReferenceSequenceTrack",
                "trackId": gname,
                "adapter": adapter,
            },
            "rendering": {"type": "DivSequenceRenderer"},
        }
        return trackDict

    def add_default_view(self):
        cmd = [

        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            uri = data
        else:
            uri = trackData["hic_url"]
        categ = trackData['category']
        trackDict = {
            "type": "HicTrack",
            "trackId": tId,
            "name": uri,
            "assemblyNames": [self.genome_name],
            "category": [categ,],
            "adapter": {
                "type": "HicAdapter",
                "hicLocation": uri,
            },
            "displays": [

                    "type": "LinearHicDisplay",
                    "displayId": "%s-LinearHicDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_maf(self, data, trackData):
        """

                    "name": "MafViewer",
                    "url": "https://unpkg.com/jbrowse-plugin-mafviewer/dist/jbrowse-plugin-mafviewer.umd.production.min.js",
                }
            ]
        }
        categ = trackData['category']
        fname = "%s.bed" % tId
        dest = "%s/%s" % (self.outdir, fname)
        gname = self.genome_name
        cmd = [
            "bash",

        ps.wait()
        outp = output.decode("ascii")
        soutp = outp.split("\n")
        samp = [x.split("s ")[1] for x in soutp if x.startswith("s ")]
        samples = [x.split(".")[0] for x in samp]
        trackDict = {
            "type": "MafTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [categ,],
            "adapter": {
                "type": "MafTabixAdapter",
                "samples": samples,
                "bedGzLocation": {
                    "uri": fname + ".sorted.bed.gz",

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)
        if self.config_json.get("plugins", None):
            self.config_json["plugins"].append(mafPlugin[0])
        else:

            str(min_gap),
            xml,
        ]
        subprocess.check_call(cmd, cwd=self.outdir, stdout=gff3_unrebased)
        gff3_unrebased.close()
        return gff3_unrebased.name

    def add_blastxml(self, data, trackData, blastOpts, **kwargs):
        gff3 = self._blastxml_to_gff3(data, min_gap=blastOpts["min_gap"])

        if "parent" in blastOpts and blastOpts["parent"] != "None":
            gff3_rebased = tempfile.NamedTemporaryFile(delete=False)
            cmd = ["python", os.path.join(INSTALLED_TO, "gff3_rebase.py")]
            if blastOpts.get("protein", "false") == "true":
                cmd.append("--protein2dna")
            cmd.extend([os.path.realpath(blastOpts["parent"]), gff3])
            subprocess.check_call(cmd, cwd=self.outdir, stdout=gff3_rebased)
            gff3_rebased.close()

            # Replace original gff3 file
            shutil.copy(gff3_rebased.name, gff3)
            os.unlink(gff3_rebased.name)
        url = "%s.gff3" % trackData["label"]
        dest = "%s/%s" % (self.outdir, url)
        self._sort_gff(gff3, dest)
        url = url + ".gz"
        tId = trackData["label"]
        categ = trackData['category']
        trackDict = {
            "type": "FeatureTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "category": [categ,],
            "adapter": {
                "type": "Gff3TabixAdapter",
                "gffGzLocation": {
                    "uri": url,
                },

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)
        os.unlink(gff3)

    def add_bigwig(self, data, trackData):

            dest = os.path.join(self.outdir, url)
            cmd = ["cp", data, dest]
            self.subprocess_check_call(cmd)
        bwloc = {"uri": url}
        tId = trackData["label"]
        categ = trackData['category']
        trackDict = {
            "type": "QuantitativeTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [categ,],
            "assemblyNames": [
                self.genome_name,
            ],
            "adapter": {
                "type": "BigWigAdapter",

                    "type": "LinearWiggleDisplay",
                    "displayId": "%s-LinearWiggleDisplay" % tId,
                }
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_bam(self, data, trackData, bam_index=None, **kwargs):
        tId = trackData["label"]
        useuri = trackData["useuri"].lower() == "yes"
        bindex = bam_index
        categ = trackData['category']
        if useuri:
            url = data
        else:
            fname = "%s.bam" % trackData["label"]
            dest = "%s/%s" % (self.outdir, fname)
            url = fname
            bindex = fname + '.bai'
            self.subprocess_check_call(["cp", data, dest])
            if bam_index is not None and os.path.exists(bam_index):
                if not os.path.exists(bindex):
                    # bai most probably made by galaxy and stored in galaxy dirs, need to copy it to dest
                    self.subprocess_check_call(
                        ["cp", bam_index, bindex]
                    )
                else:
                    # Can happen in exotic condition
                    # e.g. if bam imported as symlink with datatype=unsorted.bam, then datatype changed to bam
                    #      => no index generated by galaxy, but there might be one next to the symlink target
                    #      this trick allows to skip the bam sorting made by galaxy if already done outside

                        log.warn("Could not find a bam index (.bai file) for %s", data)
        trackDict = {
            "type": "AlignmentsTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [categ,],
            "assemblyNames": [self.genome_name],
            "adapter": {
                "type": "BamAdapter",
                "bamLocation": {"uri": url},
                "index": {

                    "type": "LinearAlignmentsDisplay",
                    "displayId": "%s-LinearAlignmentsDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_cram(self, data, trackData, cram_index=None, **kwargs):
        tId = trackData["label"]
        categ = trackData['category']
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:
            fname = "%s.cram" % trackData["label"]
            dest = "%s/%s" % (self.outdir, fname)
            url = fname
            self.subprocess_check_call(["cp", data, dest])
            if cram_index is not None and os.path.exists(cram_index):
                if not os.path.exists(dest+'.crai'):
                    # most probably made by galaxy and stored in galaxy dirs, need to copy it to dest
                    self.subprocess_check_call(
                        ["cp", os.path.realpath(cram_index), dest + ".crai"]
                    )
            else:
                cpath = os.path.realpath(dest) + '.crai'
                cmd = ["samtools", "index", "-c", "-o", cpath, os.path.realpath(dest)]
                logging.debug('executing cmd %s' % ' '.join(cmd))
                self.subprocess_check_call(cmd)
        trackDict = {
            "type": "AlignmentsTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [categ,],
            "assemblyNames": [self.genome_name],
            "adapter": {
                "type": "CramAdapter",
                "cramLocation": {"uri": url},
                "craiLocation": {
                    "uri": url + '.crai',
                },
                "sequenceAdapter": self.genome_sequence_adapter,
            },
            "displays": [
                {
                    "type": "LinearAlignmentsDisplay",
                    "displayId": "%s-LinearAlignmentsDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_vcf(self, data, trackData):
        tId = trackData["label"]
        # url = "%s/api/datasets/%s/display" % (
        # self.giURL,
        # trackData["metadata"]["dataset_id"],
        # )
        categ = trackData['category']
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:
            url = "%s.vcf.gz" % tId

        trackDict = {
            "type": "VariantTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "category": [categ,],
            "adapter": {
                "type": "VcfTabixAdapter",
                "vcfGzLocation": {
                    "uri": url
                },
                "index": {
                    "location": {
                        "uri": url + ".tbi",
                    }
                },

                    "type": "LinearPairedArcDisplay",
                    "displayId": "%s-LinearPairedArcDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def _sort_gff(self, data, dest):
        # Only index if not already done

        else:
            url = "%s.%s.gz" % (trackData["label"], ext)
            dest = "%s/%s" % (self.outdir, url)
            self._sort_gff(data, dest)
        tId = trackData["label"]
        categ = trackData['category']
        trackDict = {
            "type": "FeatureTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "category": [categ,],
            "adapter": {
                "type": "Gff3TabixAdapter",
                "gffGzLocation": {
                    "uri": url,
                },

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_bed(self, data, ext, trackData):
        tId = trackData["label"]
        categ = trackData['category']
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:
            url = "%s.%s.gz" % (trackData["label"], ext)

            "type": "FeatureTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "adapter": {
            "category": [categ,],
                "type": "BedTabixAdapter",
                "bedGzLocation": {
                    "uri": url,
                },
                "index": {

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_paf(self, data, trackData, pafOpts, **kwargs):
        tname = trackData["name"]
        tId = trackData["label"]
        categ = trackData['category']
        pgnames = [x.strip() for x in pafOpts["genome_label"].split(",")]
        pgpaths = [x.strip() for x in pafOpts["genome"].split(",")]
        passnames = [self.genome_name]  # always first
        for i, gname in enumerate(pgnames):
            if len(gname.split()) > 1:
                gname = gname.split()[0]
                passnames.append(gname)
                # trouble from spacey names in command lines avoidance
                if gname not in self.genome_names:
                    # ignore if already there - eg for duplicates among pafs.
                    useuri = pgpaths[i].startswith('http://') or pgpaths[i].startswith('https://')
                    asstrack = self.make_assembly(pgpaths[i], gname, useuri)
                    self.genome_names.append(gname)
                    if self.config_json.get("assemblies", None):
                        self.config_json["assemblies"].append(asstrack)
                    else:

        self.symlink_or_copy(os.path.realpath(data), dest)
        trackDict = {
            "type": "SyntenyTrack",
            "trackId": tId,
            "assemblyNames": passnames,
            "category": [categ,],
            "name": tname,
            "adapter": {
                "type": "PAFAdapter",
                "pafLocation": {"uri": url},
                "assemblyNames": passnames,
            },
            "displays": [
            {
            "type": "LinearSyntenyDisplay",
            "displayId": "%s-LinearSyntenyDisplay" % tId,
            },
            {
            "type": "DotPlotDisplay",
            "displayId": "%s-DotPlotDisplay" % tId,
            },
            ],
        }
        style_json = self._prepare_track_style(trackDict)
        trackDict["style"] = style_json
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def process_annotations(self, track):
        category = track["category"].replace("__pd__date__pd__", TODAY)

            else:
                logging.warn("Do not know how to handle %s", dataset_ext)
            # Return non-human label for use in other fields
            yield outputTrackConfig["label"]

    def add_default_session(self, data):
        """
        Add some default session settings: set some assemblies/tracks on/off
        """
        tracks_data = []

            config_json.update(self.config_json)

        for track_conf in self.tracksToAdd:
            track_types[track_conf["trackId"]] = track_conf["type"]
            tId = track_conf["trackId"]
            if tId in data["visibility"]["default_on"]:
                style_data = {"type": "LinearBasicDisplay"}
                if "displays" in track_conf:
                    style_data["type"] = track_conf["displays"][0]["type"]
                if track_conf.get("style_labels", None):
                    # TODO fix this: it should probably go in a renderer block (SvgFeatureRenderer) but still does not work
                    # TODO move this to per track displays?
                    style_data["labels"] = track_conf["style_labels"]
                tracks_data.append(
                    {
                        "type": track_types[tId],
                        "configuration": tId,
                        "displays": [style_data],
                    }
                )

        # The view for the assembly we're adding
        view_json = {"type": "LinearGenomeView", "tracks": tracks_data}

        refName = None
        drdict = {
            "reversed": False,
            "assemblyName": self.genome_name,
            "start": 2000,
            "end": 200000,
            "refName": "x",
        }

        if data.get("defaultLocation", ""):
            ddl = data["defaultLocation"]
            loc_match = re.search(r"^([^:]+):([\d,]*)\.*([\d,]*)$", ddl)
            # allow commas like 100,000 but ignore as integer
            if loc_match:
                refName = loc_match.group(1)
                drdict["refName"] = refName

            logging.info("@@@ defaultlocation %s for default session" % drdict)
        else:
            logging.info(
                "@@@ no contig name found for default session - please add one!"
            )
        session_name = data.get("session_name", "New session")
        for key, value in mapped_chars.items():
            session_name = session_name.replace(value, key)
        # Merge with possibly existing defaultSession (if upgrading a jbrowse instance)
        session_json = {}
        if "defaultSession" in config_json:

    def clone_jbrowse(self, realclone=True):
        """Clone a JBrowse directory into a destination directory. This also works in Biocontainer testing now"""
        dest = self.outdir
        if realclone:
            self.subprocess_check_call(['jbrowse', 'create', dest,"-f", '--tag', f"{JB2VER}"])
        else:
            shutil.copytree(self.jbrowse2path, dest, dirs_exist_ok=True)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",

        cmd = ["cp", os.path.join(INSTALLED_TO, "jb2_webserver.py"), dest]
        self.subprocess_check_call(cmd)


def parse_style_conf(item):
    if "type" in item.attrib and item.attrib["type"] in [
        "boolean",
        "integer",
    ]:
        if item.attrib["type"] == "boolean":
            return item.text in ("yes", "true", "True")
        elif item.attrib["type"] == "integer":
            return int(item.text)
    else:
        return item.text


if __name__ == "__main__":

            pass

        trackfiles = track.findall("files/trackFile")
        if trackfiles:
            for x in track.findall("files/trackFile"):
                track_conf["useuri"] = x.attrib["useuri"]
                if is_multi_bigwig:
                    multi_bigwig_paths.append(
                        (
                            x.attrib["label"],

        track_conf["format"] = track.attrib["format"]
        if track.find("options/style"):
            track_conf["style"] = {
                item.tag: parse_style_conf(item) for item in track.find("options/style")
            }
        if track.find("options/style_labels"):
            track_conf["style_labels"] = {
                item.tag: parse_style_conf(item)
                for item in track.find("options/style_labels")
            }

        track_conf["conf"] = etree_to_dict(track.find("options"))
        track_conf["category"] = track.attrib["cat"]
        track_conf["format"] = track.attrib["format"]
        try:
            # Only pertains to gff3 + blastxml. TODO?
            track_conf["style"] = {t.tag: t.text for t in track.find("options/style")}
        except TypeError:
            track_conf["style"] = {}
            pass
        keys = jc.process_annotations(track_conf)

        if keys:
            for key in keys:
                default_session_data["visibility"][
                    track.attrib.get("visibility", "default_off")
                ].append(key)
                if track_conf.get("style", None):
                    default_session_data["style"][key] = track_conf[
                        "style"
                    ]  # TODO do we need this anymore?
                if track_conf.get("style_lables", None):
                    default_session_data["style_labels"][key] = track_conf.get(
                        "style_labels", None
                    )
    default_session_data["defaultLocation"] = root.find(

57:94264fe60478

 #!/usr/bin/env python
# change to accumulating all configuration for config.json based on the default from the clone
import argparse
import binascii
import datetime
import json

import os
import re
import shutil
import struct
import subprocess
import tempfile
import urllib.request
import xml.etree.ElementTree as ET
from collections import defaultdict

                            if len(fl.split()) > 1:
                                self.genome_firstcontig = fl.split()[0].strip()
                            else:
                                self.genome_firstcontig = fl
                    else:
                        fl = urllib.request.urlopen(fapath + ".fai").readline()
                        if fl:  # is first row of the text fai so the first contig name
                            self.genome_firstcontig = (
                                fl.decode("utf8").strip().split()[0]
                            )
        if self.config_json.get("assemblies", None):
            self.config_json["assemblies"] += assemblies
        else:
            self.config_json["assemblies"] = assemblies

                "type": "ReferenceSequenceTrack",
                "trackId": gname,
                "adapter": adapter,
            },
            "rendering": {"type": "DivSequenceRenderer"},
            "displays": [
                {
                    "type": "LinearReferenceSequenceDisplay",
                    "displayId": "%s-LinearReferenceSequenceDisplay" % gname,
                },
                {
                    "type": "LinearGCContentDisplay",
                    "displayId": "%s-LinearGCContentDisplay" % gname,
                },
            ],
        }
        return trackDict

    def add_default_view(self):
        cmd = [

        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            uri = data
        else:
            uri = trackData["hic_url"]
        categ = trackData["category"]
        trackDict = {
            "type": "HicTrack",
            "trackId": tId,
            "name": uri,
            "assemblyNames": [self.genome_name],
            "category": [
                categ,
            ],
            "adapter": {
                "type": "HicAdapter",
                "hicLocation": uri,
            },
            "displays": [

                    "type": "LinearHicDisplay",
                    "displayId": "%s-LinearHicDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_maf(self, data, trackData):
        """

                    "name": "MafViewer",
                    "url": "https://unpkg.com/jbrowse-plugin-mafviewer/dist/jbrowse-plugin-mafviewer.umd.production.min.js",
                }
            ]
        }
        categ = trackData["category"]
        fname = "%s.bed" % tId
        dest = "%s/%s" % (self.outdir, fname)
        gname = self.genome_name
        cmd = [
            "bash",

        ps.wait()
        outp = output.decode("ascii")
        soutp = outp.split("\n")
        samp = [x.split("s ")[1] for x in soutp if x.startswith("s ")]
        samples = [x.split(".")[0] for x in samp]
        logging.warn("### maf convert cmd = %s,\nsamples=%s" % (' '.join(cmd), samples))
        trackDict = {
            "type": "MafTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [
                categ,
            ],
            "adapter": {
                "type": "MafTabixAdapter",
                "samples": samples,
                "bedGzLocation": {
                    "uri": fname + ".sorted.bed.gz",

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)
        if self.config_json.get("plugins", None):
            self.config_json["plugins"].append(mafPlugin[0])
        else:

            str(min_gap),
            xml,
        ]
        subprocess.check_call(cmd, cwd=self.outdir, stdout=gff3_unrebased)
        gff3_unrebased.close()
        logging.warn("### blastxml to gff3 cmd = %s" % ' '.join(cmd))
        return gff3_unrebased.name

    def add_blastxml(self, data, trackData, blastOpts, **kwargs):
        gff3 = self._blastxml_to_gff3(data, min_gap=blastOpts["min_gap"])
        if "parent" in blastOpts and blastOpts["parent"] != "None":
            gff3_rebased = tempfile.NamedTemporaryFile(delete=False)
            cmd = ["python", os.path.join(INSTALLED_TO, "gff3_rebase.py")]
            if blastOpts.get("protein", "false") == "true":
                cmd.append("--protein2dna")
            cmd.extend([os.path.realpath(blastOpts["parent"]), gff3])
            subprocess.check_call(cmd, cwd=self.outdir, stdout=gff3_rebased)
            logging.warn("### gff3rebase cmd = %s" % ' '.join(cmd))
            gff3_rebased.close()
            # Replace original gff3 file
            shutil.copy(gff3_rebased.name, gff3)
            os.unlink(gff3_rebased.name)
        url = "%s.gff3.gz" % trackData["label"]
        dest = "%s/%s" % (self.outdir, url)
        self._sort_gff(gff3, dest)
        tId = trackData["label"]
        categ = trackData["category"]
        trackDict = {
            "type": "FeatureTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "category": [
                categ,
            ],
            "adapter": {
                "type": "Gff3TabixAdapter",
                "gffGzLocation": {
                    "uri": url,
                },

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)
        os.unlink(gff3)

    def add_bigwig(self, data, trackData):

            dest = os.path.join(self.outdir, url)
            cmd = ["cp", data, dest]
            self.subprocess_check_call(cmd)
        bwloc = {"uri": url}
        tId = trackData["label"]
        categ = trackData["category"]
        trackDict = {
            "type": "QuantitativeTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [
                categ,
            ],
            "assemblyNames": [
                self.genome_name,
            ],
            "adapter": {
                "type": "BigWigAdapter",

                    "type": "LinearWiggleDisplay",
                    "displayId": "%s-LinearWiggleDisplay" % tId,
                }
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_bam(self, data, trackData, bam_index=None, **kwargs):
        tId = trackData["label"]
        useuri = trackData["useuri"].lower() == "yes"
        bindex = bam_index
        categ = trackData["category"]
        if useuri:
            url = data
        else:
            fname = "%s.bam" % trackData["label"]
            dest = "%s/%s" % (self.outdir, fname)
            url = fname
            bindex = fname + ".bai"
            self.subprocess_check_call(["cp", data, dest])
            if bam_index is not None and os.path.exists(bam_index):
                if not os.path.exists(bindex):
                    # bai most probably made by galaxy and stored in galaxy dirs, need to copy it to dest
                    self.subprocess_check_call(["cp", bam_index, bindex])
                else:
                    # Can happen in exotic condition
                    # e.g. if bam imported as symlink with datatype=unsorted.bam, then datatype changed to bam
                    #      => no index generated by galaxy, but there might be one next to the symlink target
                    #      this trick allows to skip the bam sorting made by galaxy if already done outside

                        log.warn("Could not find a bam index (.bai file) for %s", data)
        trackDict = {
            "type": "AlignmentsTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [
                categ,
            ],
            "assemblyNames": [self.genome_name],
            "adapter": {
                "type": "BamAdapter",
                "bamLocation": {"uri": url},
                "index": {

                    "type": "LinearAlignmentsDisplay",
                    "displayId": "%s-LinearAlignmentsDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_cram(self, data, trackData, cram_index=None, **kwargs):
        tId = trackData["label"]
        categ = trackData["category"]
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:
            fname = "%s.cram" % trackData["label"]
            dest = "%s/%s" % (self.outdir, fname)
            url = fname
            self.subprocess_check_call(["cp", data, dest])
            if cram_index is not None and os.path.exists(cram_index):
                if not os.path.exists(dest + ".crai"):
                    # most probably made by galaxy and stored in galaxy dirs, need to copy it to dest
                    self.subprocess_check_call(
                        ["cp", os.path.realpath(cram_index), dest + ".crai"]
                    )
            else:
                cpath = os.path.realpath(dest) + ".crai"
                cmd = ["samtools", "index", "-c", "-o", cpath, os.path.realpath(dest)]
                logging.debug("executing cmd %s" % " ".join(cmd))
                self.subprocess_check_call(cmd)
        trackDict = {
            "type": "AlignmentsTrack",
            "trackId": tId,
            "name": trackData["name"],
            "category": [
                categ,
            ],
            "assemblyNames": [self.genome_name],
            "adapter": {
                "type": "CramAdapter",
                "cramLocation": {"uri": url},
                "craiLocation": {
                    "uri": url + ".crai",
                },
                "sequenceAdapter": self.genome_sequence_adapter,
            },
            "displays": [
                {
                    "type": "LinearAlignmentsDisplay",
                    "displayId": "%s-LinearAlignmentsDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_vcf(self, data, trackData):
        tId = trackData["label"]
        # url = "%s/api/datasets/%s/display" % (
        # self.giURL,
        # trackData["metadata"]["dataset_id"],
        # )
        categ = trackData["category"]
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:
            url = "%s.vcf.gz" % tId

        trackDict = {
            "type": "VariantTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "category": [
                categ,
            ],
            "adapter": {
                "type": "VcfTabixAdapter",
                "vcfGzLocation": {"uri": url},
                "index": {
                    "location": {
                        "uri": url + ".tbi",
                    }
                },

                    "type": "LinearPairedArcDisplay",
                    "displayId": "%s-LinearPairedArcDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def _sort_gff(self, data, dest):
        # Only index if not already done

        else:
            url = "%s.%s.gz" % (trackData["label"], ext)
            dest = "%s/%s" % (self.outdir, url)
            self._sort_gff(data, dest)
        tId = trackData["label"]
        categ = trackData["category"]
        trackDict = {
            "type": "FeatureTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "category": [
                categ,
            ],
            "adapter": {
                "type": "Gff3TabixAdapter",
                "gffGzLocation": {
                    "uri": url,
                },

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_bed(self, data, ext, trackData):
        tId = trackData["label"]
        categ = trackData["category"]
        useuri = trackData["useuri"].lower() == "yes"
        if useuri:
            url = data
        else:
            url = "%s.%s.gz" % (trackData["label"], ext)

            "type": "FeatureTrack",
            "trackId": tId,
            "name": trackData["name"],
            "assemblyNames": [self.genome_name],
            "adapter": {
                "category": [
                    categ,
                ],
                "type": "BedTabixAdapter",
                "bedGzLocation": {
                    "uri": url,
                },
                "index": {

                    "type": "LinearArcDisplay",
                    "displayId": "%s-LinearArcDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def add_paf(self, data, trackData, pafOpts, **kwargs):
        tname = trackData["name"]
        tId = trackData["label"]
        categ = trackData["category"]
        pgnames = [x.strip() for x in pafOpts["genome_label"].split(",")]
        pgpaths = [x.strip() for x in pafOpts["genome"].split(",")]
        passnames = [self.genome_name]  # always first
        for i, gname in enumerate(pgnames):
            if len(gname.split()) > 1:
                gname = gname.split()[0]
                passnames.append(gname)
                # trouble from spacey names in command lines avoidance
                if gname not in self.genome_names:
                    # ignore if already there - eg for duplicates among pafs.
                    useuri = pgpaths[i].startswith("http://") or pgpaths[i].startswith(
                        "https://"
                    )
                    asstrack = self.make_assembly(pgpaths[i], gname, useuri)
                    self.genome_names.append(gname)
                    if self.config_json.get("assemblies", None):
                        self.config_json["assemblies"].append(asstrack)
                    else:

        self.symlink_or_copy(os.path.realpath(data), dest)
        trackDict = {
            "type": "SyntenyTrack",
            "trackId": tId,
            "assemblyNames": passnames,
            "category": [
                categ,
            ],
            "name": tname,
            "adapter": {
                "type": "PAFAdapter",
                "pafLocation": {"uri": url},
                "assemblyNames": passnames,
            },
            "displays": [
                {
                    "type": "LinearSyntenyDisplay",
                    "displayId": "%s-LinearSyntenyDisplay" % tId,
                },
                {
                    "type": "DotPlotDisplay",
                    "displayId": "%s-DotPlotDisplay" % tId,
                },
            ],
        }
        self.tracksToAdd.append(trackDict)
        self.trackIdlist.append(tId)

    def process_annotations(self, track):
        category = track["category"].replace("__pd__date__pd__", TODAY)

            else:
                logging.warn("Do not know how to handle %s", dataset_ext)
            # Return non-human label for use in other fields
            yield outputTrackConfig["label"]

    def add_default_session(self, default_data):
        """
        Add some default session settings: set some assemblies/tracks on/off
        """
        tracks_data = []

            config_json.update(self.config_json)

        for track_conf in self.tracksToAdd:
            track_types[track_conf["trackId"]] = track_conf["type"]
            tId = track_conf["trackId"]
            #if tId in data["visibility"]["default_on"]:
            style_data = default_data["style"].get(tId,  None)
            if not style_data:
                logging.warn("### No style data in default data for %s" % tId)
                style_data = {"type": "LinearBasicDisplay"}
            if "displays" in track_conf:
                disp = track_conf["displays"][0]["type"]
                style_data["type"] = disp
                style_data["configuration"] = "%s-%s" % (tId, disp)
            if track_conf.get("style_labels", None):
                # TODO fix this: it should probably go in a renderer block (SvgFeatureRenderer) but still does not work
                # TODO move this to per track displays?
                style_data["labels"] = track_conf["style_labels"]
            tracks_data.append(
                {
                    "type": track_types[tId],
                    "configuration": tId,
                    "displays": [style_data],
                }
            )

        # The view for the assembly we're adding
        view_json = {"type": "LinearGenomeView", "tracks": tracks_data}

        refName = None
        drdict = {
            "reversed": False,
            "assemblyName": self.genome_name,
            "start": 1,
            "end": 100000,
            "refName": "x",
        }

        if default_data.get("defaultLocation", ""):
            ddl = default_data["defaultLocation"]
            loc_match = re.search(r"^([^:]+):([\d,]*)\.*([\d,]*)$", ddl)
            # allow commas like 100,000 but ignore as integer
            if loc_match:
                refName = loc_match.group(1)
                drdict["refName"] = refName

            logging.info("@@@ defaultlocation %s for default session" % drdict)
        else:
            logging.info(
                "@@@ no contig name found for default session - please add one!"
            )
        session_name = default_data.get("session_name", "New session")
        for key, value in mapped_chars.items():
            session_name = session_name.replace(value, key)
        # Merge with possibly existing defaultSession (if upgrading a jbrowse instance)
        session_json = {}
        if "defaultSession" in config_json:

    def clone_jbrowse(self, realclone=True):
        """Clone a JBrowse directory into a destination directory. This also works in Biocontainer testing now"""
        dest = self.outdir
        if realclone:
            self.subprocess_check_call(
                ["jbrowse", "create", dest, "-f", "--tag", f"{JB2VER}"]
            )
        else:
            shutil.copytree(self.jbrowse2path, dest, dirs_exist_ok=True)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",

        cmd = ["cp", os.path.join(INSTALLED_TO, "jb2_webserver.py"), dest]
        self.subprocess_check_call(cmd)


def parse_style_conf(item):
    if item.text.lower() in ['false','true','yes','no']:
            return item.text.lower in ("yes", "true")
    else:
        return item.text


if __name__ == "__main__":

            pass

        trackfiles = track.findall("files/trackFile")
        if trackfiles:
            for x in track.findall("files/trackFile"):
                track_conf["label"] = x.attrib["label"]
                track_conf["useuri"] = x.attrib["useuri"]
                if is_multi_bigwig:
                    multi_bigwig_paths.append(
                        (
                            x.attrib["label"],

        track_conf["format"] = track.attrib["format"]
        if track.find("options/style"):
            track_conf["style"] = {
                item.tag: parse_style_conf(item) for item in track.find("options/style")
            }
        else:
            track_conf["style"] = {}
        tst = track_conf["style"].get("type", None)
        if tst:
            track_conf["style"]["configuration"] = "%s-%s" % (track_conf["label"], tst)
        if track.find("options/style_labels"):
            track_conf["style_labels"] = {
                item.tag: parse_style_conf(item)
                for item in track.find("options/style_labels")
            }

        track_conf["conf"] = etree_to_dict(track.find("options"))
        track_conf["category"] = track.attrib["cat"]
        track_conf["format"] = track.attrib["format"]
        keys = jc.process_annotations(track_conf)

        if keys:
            for key in keys:
                default_session_data["visibility"][
                    track.attrib.get("visibility", "default_off")
                ].append(key)
                if track_conf.get("style", None):
                    default_session_data["style"][key] = track_conf["style"]
                if track_conf.get("style_lables", None):
                    default_session_data["style_labels"][key] = track_conf.get(
                        "style_labels", None
                    )
    default_session_data["defaultLocation"] = root.find(