comparison jbrowse2/jbrowse2.py @ 10:0db895a99532 draft default tip

Uploaded

9:6a41f87b5d7f

    )
    return metadata


class JbrowseConnector(object):
    def __init__(self, jbrowse, outdir, genomes):
        self.debug = False
        self.usejson = True
        self.giURL = GALAXY_INFRASTRUCTURE_URL
        self.jbrowse = jbrowse
        self.outdir = outdir
        os.makedirs(self.outdir, exist_ok=True)
        self.genome_paths = genomes
        self.trackIdlist = []
        self.tracksToAdd = []
        self.config_json = {}
        self.config_json_file = os.path.realpath(os.path.join(outdir, "config.json"))
        self.clone_jbrowse(self.jbrowse, self.outdir)

    def subprocess_check_call(self, command, output=None):
        if output:
            if self.debug:
                log.debug("cd %s && %s >  %s", self.outdir, " ".join(command), output)

            log.debug("cd %s && %s", self.outdir, " ".join(command))
            subprocess.check_call(command, cwd=self.outdir)

    def subprocess_popen(self, command):
        if self.debug:
            log.debug("cd %s && %s", self.outdir, command)
        p = subprocess.Popen(
            command,
            shell=True,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        output, err = p.communicate()
        retcode = p.returncode
        if retcode != 0:
            log.error("cd %s && %s", self.outdir, command)
            log.error(output)
            log.error(err)
            raise RuntimeError("Command failed with exit code %s" % (retcode))

    def subprocess_check_output(self, command):
        if self.debug:
            log.debug("cd %s && %s", self.outdir, " ".join(command))
        return subprocess.check_output(command, cwd=self.outdir)

    def _jbrowse_bin(self, command):
        return os.path.realpath(os.path.join(self.jbrowse, "bin", command))

    def symlink_or_copy(self, src, dest):
        if "GALAXY_JBROWSE_SYMLINKS" in os.environ and bool(
            os.environ["GALAXY_JBROWSE_SYMLINKS"]
        ):

            if self.debug:
                log.info("genome_node=%s" % str(genome_node))
            genome_name = genome_node["meta"]["dataset_dname"].strip().split()[0]
            fapath = genome_node["path"]
            faname = genome_name + ".fa.gz"
            fadest = os.path.realpath(os.path.join(self.outdir, faname))
            cmd = "bgzip -i -c %s > %s && samtools faidx %s" % (
                fapath,
                fadest,
                fadest
            )
            log.info("### cmd = %s" % ' '.join(cmd))
            self.subprocess_popen(cmd)
            adapter = {
                "type": "BgzipFastaAdapter",
                "fastaLocation": {
                    "uri": faname,

        url = "%s/api/datasets/%s/display?to_ext=hic " % (
            self.giURL,
            dsId,
        )
        hname = trackData["name"]
        dest = os.path.realpath(os.path.join(self.outdir, hname))
        url = hname
        cmd = ["cp", data, dest]
        self.subprocess_check_call(cmd)
        floc = {
            "uri": hname,

                }
            ]
        }
        tId = trackData["label"]
        fname = "%s.bed" % tId
        dest = os.path.realpath("%s/%s" % (self.outdir, fname))
        # self.symlink_or_copy(data, dest)
        # Process MAF to bed-like. Need build to munge chromosomes
        gname = self.genome_name
        cmd = [
            "bash",

            # Replace original gff3 file
            shutil.copy(gff3_rebased.name, gff3)
            os.unlink(gff3_rebased.name)
        url = "%s.gff3" % trackData["label"]
        dest = os.path.realpath("%s/%s" % (self.outdir, url))
        self._sort_gff(gff3, dest)
        url = url + ".gz"
        tId = trackData["label"]
        trackDict = {
            "type": "FeatureTrack",

            self.subprocess_check_call(cmd)
        os.unlink(gff3)

    def add_bigwig(self, data, trackData):
        url = "%s.bw" % trackData["name"]
        dest = os.path.realpath(os.path.join(self.outdir, url))
        cmd = ["cp", data, dest]
        self.subprocess_check_call(cmd)
        bwloc = {"uri": url}
        tId = trackData["label"]
        trackDict = {

            self.subprocess_check_call(cmd)

    def add_bam(self, data, trackData, bamOpts, bam_index=None, **kwargs):
        tId = trackData["label"]
        fname = "%s.bam" % trackData["label"]
        dest = os.path.realpath("%s/%s" % (self.outdir, fname))
        url = fname
        self.subprocess_check_call(["cp", data, dest])
        log.info("### copied %s to %s" % (data, dest))
        bloc = {"uri": url}
        if bam_index is not None and os.path.exists(os.path.realpath(bam_index)):

        url = "%s/api/datasets/%s/display" % (
            self.giURL,
            trackData["metadata"]["dataset_id"],
        )
        url = "%s.vcf.gz" % tId
        dest = os.path.realpath("%s/%s" % (self.outdir, url))
        cmd = "bgzip -c %s  > %s" % (data, dest)
        self.subprocess_popen(cmd)
        cmd = ["tabix", "-p", "vcf", dest]
        self.subprocess_check_call(cmd)
        trackDict = {

            cmd = ["tabix", "-f", "-p", "bed", dest]
            self.subprocess_check_call(cmd)

    def add_gff(self, data, ext, trackData):
        url = "%s.%s" % (trackData["label"], ext)
        dest = os.path.realpath("%s/%s" % (self.outdir, url))
        self._sort_gff(data, dest)
        url = url + ".gz"
        tId = trackData["label"]
        trackDict = {
            "type": "FeatureTrack",

            ]
            self.subprocess_check_call(cmd)

    def add_bed(self, data, ext, trackData):
        url = "%s.%s" % (trackData["label"], ext)
        dest = os.path.realpath("%s/%s.gz" % (self.outdir, url))
        self._sort_bed(data, dest)
        tId = trackData["label"]
        url = url + ".gz"
        trackDict = {
            "type": "FeatureTrack",

            elif dataset_ext == "vcf":
                self.add_vcf(dataset_path, outputTrackConfig)
            else:
                log.warn("Do not know how to handle %s", dataset_ext)

    def clone_jbrowse(self, jbrowse_dir, destination):
        """Clone a JBrowse directory into a destination directory."""
        cmd = ["jbrowse", "create", "-f", self.outdir]
        self.subprocess_check_call(cmd)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",
            "robots.txt",

            "version.txt",
            "test_data",
        ]:
            cmd = ["rm", "-rf", os.path.join(self.outdir, fn)]
            self.subprocess_check_call(cmd)
        cmd = ['cp', os.path.join(INSTALLED_TO, "servejb2.py"), self.outdir]
        self.subprocess_check_call(cmd)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="", epilog="")
    parser.add_argument("xml", type=argparse.FileType("r"), help="Track Configuration")

    parser.add_argument("--jbrowse", help="Folder containing a jbrowse release")
    parser.add_argument("--outdir", help="Output directory", default="out")
    parser.add_argument("--version", "-V", action="version", version="%(prog)s 0.8.0")
    args = parser.parse_args()

    tree = ET.parse(args.xml.name)
    root = tree.getroot()

    # This should be done ASAP
    GALAXY_INFRASTRUCTURE_URL = root.find("metadata/galaxyUrl").text
    # Sometimes this comes as `localhost` without a protocol
    if not GALAXY_INFRASTRUCTURE_URL.startswith("http"):
        # so we'll prepend `http://` and hope for the best. Requests *should*
        # be GET and not POST so it should redirect OK
        GALAXY_INFRASTRUCTURE_URL = "http://" + GALAXY_INFRASTRUCTURE_URL

    jc = JbrowseConnector(
        jbrowse=args.jbrowse,
        outdir=args.outdir,
        genomes=[
            {
                "path": os.path.realpath(x.attrib["path"]),
                "meta": metadata_from_node(x.find("metadata")),

10:0db895a99532

    )
    return metadata


class JbrowseConnector(object):
    def __init__(self, outdir, genomes):
        self.debug = False
        self.usejson = True
        self.giURL = GALAXY_INFRASTRUCTURE_URL
        self.outdir = outdir
        os.makedirs(self.outdir, exist_ok=True)
        self.genome_paths = genomes
        self.trackIdlist = []
        self.tracksToAdd = []
        self.config_json = {}
        self.config_json_file = os.path.join(outdir, "config.json")
        self.clone_jbrowse(destination=self.outdir)

    def subprocess_check_call(self, command, output=None):
        if output:
            if self.debug:
                log.debug("cd %s && %s >  %s", self.outdir, " ".join(command), output)

            log.debug("cd %s && %s", self.outdir, " ".join(command))
            subprocess.check_call(command, cwd=self.outdir)

    def subprocess_popen(self, command):
        if self.debug:
            log.debug(command)
        p = subprocess.Popen(
            command,
            cwd=self.outdir,
            shell=True,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        output, err = p.communicate()
        retcode = p.returncode
        if retcode != 0:
            log.error(command)
            log.error(output)
            log.error(err)
            raise RuntimeError("Command failed with exit code %s" % (retcode))

    def subprocess_check_output(self, command):
        if self.debug:
            log.debug(" ".join(command))
        return subprocess.check_output(command, cwd=self.outdir)

    def symlink_or_copy(self, src, dest):
        if "GALAXY_JBROWSE_SYMLINKS" in os.environ and bool(
            os.environ["GALAXY_JBROWSE_SYMLINKS"]
        ):

            if self.debug:
                log.info("genome_node=%s" % str(genome_node))
            genome_name = genome_node["meta"]["dataset_dname"].strip().split()[0]
            fapath = genome_node["path"]
            faname = genome_name + ".fa.gz"
            fadest = os.path.join(self.outdir, faname)
            # fadest = os.path.realpath(os.path.join(self.outdir, faname))
            cmd = "bgzip -i -c %s -I %s.gzi > %s && samtools faidx %s" % (
                fapath,
                fadest,
                fadest,
                fadest,
            )
            if self.debug:
                log.info("### cmd = %s" % cmd)
            self.subprocess_popen(cmd)
            adapter = {
                "type": "BgzipFastaAdapter",
                "fastaLocation": {
                    "uri": faname,

        url = "%s/api/datasets/%s/display?to_ext=hic " % (
            self.giURL,
            dsId,
        )
        hname = trackData["name"]
        dest = os.path.join(self.outdir, hname)
        url = hname
        cmd = ["cp", data, dest]
        self.subprocess_check_call(cmd)
        floc = {
            "uri": hname,

                }
            ]
        }
        tId = trackData["label"]
        fname = "%s.bed" % tId
        dest = "%s/%s" % (self.outdir, fname)
        # self.symlink_or_copy(data, dest)
        # Process MAF to bed-like. Need build to munge chromosomes
        gname = self.genome_name
        cmd = [
            "bash",

            # Replace original gff3 file
            shutil.copy(gff3_rebased.name, gff3)
            os.unlink(gff3_rebased.name)
        url = "%s.gff3" % trackData["label"]
        dest = "%s/%s" % (self.outdir, url)
        self._sort_gff(gff3, dest)
        url = url + ".gz"
        tId = trackData["label"]
        trackDict = {
            "type": "FeatureTrack",

            self.subprocess_check_call(cmd)
        os.unlink(gff3)

    def add_bigwig(self, data, trackData):
        url = "%s.bw" % trackData["name"]
        dest = os.path.join(self.outdir, url)
        cmd = ["cp", data, dest]
        self.subprocess_check_call(cmd)
        bwloc = {"uri": url}
        tId = trackData["label"]
        trackDict = {

            self.subprocess_check_call(cmd)

    def add_bam(self, data, trackData, bamOpts, bam_index=None, **kwargs):
        tId = trackData["label"]
        fname = "%s.bam" % trackData["label"]
        dest = "%s/%s" % (self.outdir, fname)
        url = fname
        self.subprocess_check_call(["cp", data, dest])
        log.info("### copied %s to %s" % (data, dest))
        bloc = {"uri": url}
        if bam_index is not None and os.path.exists(os.path.realpath(bam_index)):

        url = "%s/api/datasets/%s/display" % (
            self.giURL,
            trackData["metadata"]["dataset_id"],
        )
        url = "%s.vcf.gz" % tId
        dest = "%s/%s" % (self.outdir, url)
        cmd = "bgzip -c %s  > %s" % (data, dest)
        self.subprocess_popen(cmd)
        cmd = ["tabix", "-p", "vcf", dest]
        self.subprocess_check_call(cmd)
        trackDict = {

            cmd = ["tabix", "-f", "-p", "bed", dest]
            self.subprocess_check_call(cmd)

    def add_gff(self, data, ext, trackData):
        url = "%s.%s" % (trackData["label"], ext)
        dest = "%s/%s" % (self.outdir, url)
        self._sort_gff(data, dest)
        url = url + ".gz"
        tId = trackData["label"]
        trackDict = {
            "type": "FeatureTrack",

            ]
            self.subprocess_check_call(cmd)

    def add_bed(self, data, ext, trackData):
        url = "%s.%s" % (trackData["label"], ext)
        dest = "%s/%s.gz" % (self.outdir, url)
        self._sort_bed(data, dest)
        tId = trackData["label"]
        url = url + ".gz"
        trackDict = {
            "type": "FeatureTrack",

            elif dataset_ext == "vcf":
                self.add_vcf(dataset_path, outputTrackConfig)
            else:
                log.warn("Do not know how to handle %s", dataset_ext)

    def clone_jbrowse(self, destination):
        """Clone a JBrowse directory into a destination directory."""
        cmd = ["jbrowse", "create", "-f", os.path.realpath(destination)]
        self.subprocess_check_call(cmd)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",
            "robots.txt",

            "version.txt",
            "test_data",
        ]:
            cmd = ["rm", "-rf", os.path.join(self.outdir, fn)]
            self.subprocess_check_call(cmd)
        cmd = ["cp", os.path.join(INSTALLED_TO, "servejb2.py"), self.outdir]
        self.subprocess_check_call(cmd)

    def clone_jbrowse2(self, jbrowse_dir, destination):
        """Clone a JBrowse directory into a destination directory."""
        cmd = ["cp", "-rv", jbrowse_dir + "/*", self.outdir]
        self.subprocess_check_call(cmd)
        for fn in [
            "asset-manifest.json",
            "favicon.ico",
            "robots.txt",
            "umd_plugin.js",
            "version.txt",
            "test_data",
        ]:
            cmd = ["rm", "-rf", os.path.join(self.outdir, fn)]
            self.subprocess_check_call(cmd)
        cmd = ["cp", os.path.join(INSTALLED_TO, "servejb2.py"), self.outdir]
        self.subprocess_check_call(cmd)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="", epilog="")
    parser.add_argument("--xml", help="Track Configuration")
    parser.add_argument("--jbrowse", help="Output from 'which jbrowse'")
    parser.add_argument("--outdir", help="Output directory", default="out")
    parser.add_argument("--version", "-V", action="version", version="%(prog)s 2.0.1")
    args = parser.parse_args()

    tree = ET.parse(args.xml)
    root = tree.getroot()

    # This should be done ASAP
    GALAXY_INFRASTRUCTURE_URL = root.find("metadata/galaxyUrl").text
    # Sometimes this comes as `localhost` without a protocol
    if not GALAXY_INFRASTRUCTURE_URL.startswith("http"):
        # so we'll prepend `http://` and hope for the best. Requests *should*
        # be GET and not POST so it should redirect OK
        GALAXY_INFRASTRUCTURE_URL = "http://" + GALAXY_INFRASTRUCTURE_URL
    jc = JbrowseConnector(
        outdir=args.outdir,
        genomes=[
            {
                "path": os.path.realpath(x.attrib["path"]),
                "meta": metadata_from_node(x.find("metadata")),