Mercurial > repos > dave > data_manager_build_kraken2_database

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/kraken2_build_database.py	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,382 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import datetime
+import errno
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+from enum import Enum
+
+try:
+    # Python3
+    from urllib.request import urlopen
+except ImportError:
+    from urllib2 import urlopen
+
+
+DATA_TABLE_NAME = "kraken2_databases"
+
+
+class KrakenDatabaseTypes(Enum):
+    standard = 'standard'
+    minikraken = 'minikraken'
+    special = 'special'
+    custom = 'custom'
+
+    def __str__(self):
+        return self.value
+
+
+class SpecialDatabaseTypes(Enum):
+    rdp = 'rdp'
+    greengenes = 'greengenes'
+    silva = 'silva'
+
+    def __str__(self):
+        return self.value
+
+
+class Minikraken2Versions(Enum):
+    v1 = 'v1'
+    v2 = 'v2'
+
+    def __str__(self):
+        return self.value
+
+class Minikraken2Releases(Enum):
+    March_2020 = 'March_2020'
+    April_2019 = 'April_2019'
+
+    def __str__(self):
+        return self.value
+
+
+def kraken2_build_standard(kraken2_args, target_directory, data_table_name=DATA_TABLE_NAME):
+    now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%SZ")
+
+    database_value = "_".join([
+        now,
+        "standard",
+        "kmer-len", str(kraken2_args["kmer_len"]),
+        "minimizer-len", str(kraken2_args["minimizer_len"]),
+        "minimizer-spaces", str(kraken2_args["minimizer_spaces"]),
+        "load-factor", str(kraken2_args["load_factor"]),
+    ])
+
+    database_name = " ".join([
+        "Standard",
+        "(Created:",
+        now + ",",
+        "kmer-len=" + str(kraken2_args["kmer_len"]) + ",",
+        "minimizer-len=" + str(kraken2_args["minimizer_len"]) + ",",
+        "minimizer-spaces=" + str(kraken2_args["minimizer_spaces"]) + ")",
+        "load-factor", str(kraken2_args["load_factor"]),
+    ])
+
+    database_path = database_value
+
+    args = [
+        '--threads', str(kraken2_args["threads"]),
+        '--standard',
+        '--kmer-len', str(kraken2_args["kmer_len"]),
+        '--minimizer-len', str(kraken2_args["minimizer_len"]),
+        '--minimizer-spaces', str(kraken2_args["minimizer_spaces"]),
+        '--load-factor', str(kraken2_args["load_factor"]),
+        '--db', database_path
+    ]
+
+    subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    if kraken2_args["clean"]:
+        args = [
+            '--threads', str(kraken2_args["threads"]),
+            '--clean',
+            '--db', database_path
+        ]
+
+        subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    data_table_entry = {
+        'data_tables': {
+            data_table_name: [
+                {
+                    "value": database_value,
+                    "name": database_name,
+                    "path": database_path,
+                }
+            ]
+        }
+    }
+
+    return data_table_entry
+
+
+def kraken2_build_minikraken(minikraken2_version, minikraken2_release, target_directory, data_table_name=DATA_TABLE_NAME):
+
+    now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%SZ")
+
+    value_parts = [now, "minikraken2", minikraken2_release, "8GB"]
+    name_parts = ["Minikraken2", minikraken2_release, "8GB", "(Created: %s)" % now]
+
+    if minikraken2_release == 'April_2019':
+        value_parts.insert(3, minikraken2_version)
+        name_parts.insert(2, minikraken2_version)
+        src = urlopen(
+            'ftp://ftp.ccb.jhu.edu/pub/data/kraken2_dbs/old/minikraken2_%s_8GB_201904.tgz'
+            % minikraken2_version
+        )
+    else:
+        src = urlopen('ftp://ftp.ccb.jhu.edu/pub/data/kraken2_dbs/minikraken_8GB_202003.tgz')
+
+    database_value = "_".join(value_parts)
+
+    database_name = " ".join(name_parts)
+
+    database_path = database_value
+
+    # download the minikraken2 data
+    with open('tmp_data.tar.gz', 'wb') as dst:
+        shutil.copyfileobj(src, dst)
+    # unpack the downloaded archive to the target directory
+    with tarfile.open('tmp_data.tar.gz', 'r:gz') as fh:
+        for member in fh.getmembers():
+            if member.isreg():
+                member.name = os.path.basename(member.name)
+                fh.extract(member, os.path.join(target_directory, database_path))
+
+    data_table_entry = {
+        'data_tables': {
+            data_table_name: [
+                {
+                    "value": database_value,
+                    "name": database_name,
+                    "path": database_path,
+                }
+            ]
+        }
+    }
+
+    return data_table_entry
+
+
+def kraken2_build_special(kraken2_args, target_directory, data_table_name=DATA_TABLE_NAME):
+
+    now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%SZ")
+
+    special_database_names = {
+        "rdp": "RDP",
+        "greengenes": "Greengenes",
+        "silva": "Silva",
+    }
+
+    database_value = "_".join([
+        now,
+        kraken2_args["special_database_type"],
+        "kmer-len", str(kraken2_args["kmer_len"]),
+        "minimizer-len", str(kraken2_args["minimizer_len"]),
+        "minimizer-spaces", str(kraken2_args["minimizer_spaces"]),
+        "load-factor", str(kraken2_args["load_factor"]),
+    ])
+
+    database_name = " ".join([
+        special_database_names[kraken2_args["special_database_type"]],
+        "(Created:",
+        now + ",",
+        "kmer-len=" + str(kraken2_args["kmer_len"]) + ",",
+        "minimizer-len=" + str(kraken2_args["minimizer_len"]) + ",",
+        "minimizer-spaces=" + str(kraken2_args["minimizer_spaces"]) + ")",
+        "load-factor=" + str(kraken2_args["load_factor"]) + ")",
+    ])
+
+    database_path = database_value
+
+    args = [
+        '--threads', str(kraken2_args["threads"]),
+        '--special', kraken2_args["special_database_type"],
+        '--kmer-len', str(kraken2_args["kmer_len"]),
+        '--minimizer-len', str(kraken2_args["minimizer_len"]),
+        '--minimizer-spaces', str(kraken2_args["minimizer_spaces"]),
+        '--load-factor', str(kraken2_args["load_factor"]),
+        '--db', database_path
+    ]
+
+    subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    if kraken2_args["clean"]:
+        args = [
+            '--threads', str(kraken2_args["threads"]),
+            '--clean',
+            '--db', database_path
+        ]
+
+        subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    data_table_entry = {
+        'data_tables': {
+            data_table_name: [
+                {
+                    "value": database_value,
+                    "name": database_name,
+                    "path": database_path,
+                }
+            ]
+        }
+    }
+
+    return data_table_entry
+
+
+def kraken2_build_custom(kraken2_args, custom_database_name, target_directory, data_table_name=DATA_TABLE_NAME):
+
+    args = [
+        '--threads', str(kraken2_args["threads"]),
+        '--download-taxonomy',
+        '--db', custom_database_name,
+    ]
+
+    if kraken2_args['skip_maps']:
+        args.append('--skip-maps')
+
+    subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    args = [
+        '--threads', str(kraken2_args["threads"]),
+        '--add-to-library', kraken2_args["custom_fasta"],
+        '--db', custom_database_name
+    ]
+
+    subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    args = [
+        '--threads', str(kraken2_args["threads"]),
+        '--build',
+        '--kmer-len', str(kraken2_args["kmer_len"]),
+        '--minimizer-len', str(kraken2_args["minimizer_len"]),
+        '--minimizer-spaces', str(kraken2_args["minimizer_spaces"]),
+        '--load-factor', str(kraken2_args["load_factor"]),
+        '--db', custom_database_name
+    ]
+
+    subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    if kraken2_args["clean"]:
+        args = [
+            '--threads', str(kraken2_args["threads"]),
+            '--clean',
+            '--db', custom_database_name
+        ]
+
+        subprocess.check_call(['kraken2-build'] + args, cwd=target_directory)
+
+    data_table_entry = {
+        'data_tables': {
+            data_table_name: [
+                {
+                    "value": custom_database_name,
+                    "name": custom_database_name,
+                    "path": custom_database_name
+                }
+            ]
+        }
+    }
+
+    return data_table_entry
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('data_manager_json')
+    parser.add_argument('--kmer-len', dest='kmer_len', type=int, default=35, help='kmer length')
+    parser.add_argument('--minimizer-len', dest='minimizer_len', type=int, default=31, help='minimizer length')
+    parser.add_argument('--minimizer-spaces', dest='minimizer_spaces', default=6, help='minimizer spaces')
+    parser.add_argument('--load-factor', dest='load_factor', type=float, default=0.7, help='load factor')
+    parser.add_argument('--threads', dest='threads', default=1, help='threads')
+    parser.add_argument('--database-type', dest='database_type', type=KrakenDatabaseTypes, choices=list(KrakenDatabaseTypes), required=True, help='type of kraken database to build')
+    parser.add_argument('--minikraken2-version', dest='minikraken2_version', type=Minikraken2Versions, choices=list(Minikraken2Versions), help='MiniKraken2 version (only applies to --database-type minikraken) and the Mar2019 release')
+    parser.add_argument('--minikraken2-release', dest='minikraken2_release', type=Minikraken2Releases, choices=list(Minikraken2Releases), help='MiniKraken2 release (only applies to --database-type minikraken)')
+    parser.add_argument('--special-database-type', dest='special_database_type', type=SpecialDatabaseTypes, choices=list(SpecialDatabaseTypes), help='type of special database to build (only applies to --database-type special)')
+    parser.add_argument('--custom-fasta', dest='custom_fasta', help='fasta file for custom database (only applies to --database-type custom)')
+    parser.add_argument('--custom-database-name', dest='custom_database_name', help='Name for custom database (only applies to --database-type custom)')
+    parser.add_argument('--skip-maps', dest='skip_maps', action='store_true', help='')
+    parser.add_argument('--clean', dest='clean', action='store_true', help='Clean up extra files')
+    args = parser.parse_args()
+
+    with open(args.data_manager_json) as fh:
+        data_manager_input = json.load(fh)
+
+    target_directory = data_manager_input['output_data'][0]['extra_files_path']
+
+    try:
+        os.mkdir(target_directory)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(target_directory):
+            pass
+        else:
+            raise
+
+    data_manager_output = {}
+
+    if str(args.database_type) == 'standard':
+        kraken2_args = {
+            "kmer_len": args.kmer_len,
+            "minimizer_len": args.minimizer_len,
+            "minimizer_spaces": args.minimizer_spaces,
+            "load_factor": args.load_factor,
+            "threads": args.threads,
+            "clean": args.clean,
+        }
+        data_manager_output = kraken2_build_standard(
+            kraken2_args,
+            target_directory,
+        )
+    elif str(args.database_type) == 'minikraken':
+        data_manager_output = kraken2_build_minikraken(
+            str(args.minikraken2_version),
+            str(args.minikraken2_release),
+            target_directory
+        )
+    elif str(args.database_type) == 'special':
+        kraken2_args = {
+            "special_database_type": str(args.special_database_type),
+            "kmer_len": args.kmer_len,
+            "minimizer_len": args.minimizer_len,
+            "minimizer_spaces": args.minimizer_spaces,
+            "load_factor": args.load_factor,
+            "threads": args.threads,
+            "clean": args.clean,
+        }
+        data_manager_output = kraken2_build_special(
+            kraken2_args,
+            target_directory,
+        )
+    elif str(args.database_type) == 'custom':
+        kraken2_args = {
+            "custom_fasta": args.custom_fasta,
+            "skip_maps": args.skip_maps,
+            "kmer_len": args.kmer_len,
+            "minimizer_len": args.minimizer_len,
+            "minimizer_spaces": args.minimizer_spaces,
+            "load_factor": args.load_factor,
+            "threads": args.threads,
+            "clean": args.clean,
+        }
+        data_manager_output = kraken2_build_custom(
+            kraken2_args,
+            args.custom_database_name,
+            target_directory,
+        )
+    else:
+        sys.exit("Invalid database type")
+
+    with open(args.data_manager_json, 'w') as fh:
+        json.dump(data_manager_output, fh, sort_keys=True)
+
+
+if __name__ == "__main__":
+    main()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/kraken2_build_database.xml	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,116 @@
+<?xml version="1.0"?>
+<tool id="kraken2_build_database" name="Kraken2" tool_type="manage_data" version="@TOOL_VERSION@+galaxy0" profile="18.09">
+    <macros>
+        <token name="@TOOL_VERSION@">2.1.1</token>
+        <xml name="common_params">
+            <param name="kmer_len" type="integer" value="35" label="K-mer length in BP" />
+            <param name="minimizer_len" type="integer" value="31" label="Minimizer length" />
+            <param name="minimizer_spaces" type="integer" value="6" label="Minimizer spaces" />
+            <param name="load_factor" type="float" value="0.7" min="0" max="1" label="Load factor" help="Proportion of the hash table to be populated" />
+            <param name="clean" type="boolean" truevalue="--clean" falsevalue="" checked="true" label="Clean up extra files" />
+        </xml>
+    </macros>
+    <description>database builder</description>
+    <requirements>
+        <requirement type="package" version="@TOOL_VERSION@">kraken2</requirement>
+        <requirement type="package" version="3.7">python</requirement>
+    </requirements>
+    <version_command>kraken2 -version | head -n 1 | awk '{print $NF}'</version_command>
+    <command detect_errors="exit_code">
+    <![CDATA[
+        python '$__tool_directory__/kraken2_build_database.py'
+          '${out_file}'
+          --database-type ${database_type.database_type}
+          #if $database_type.database_type == "minikraken"
+            --minikraken2-release $database_type.kraken_release
+            #if $database_type.kraken_release == "April_2019":
+                --minikraken2-version ${database_type.kraken_release.minikraken2_version}
+            #end if
+          #else if $database_type.database_type == "standard"
+            --threads \${GALAXY_SLOTS:-1}
+            --kmer-len ${database_type.kmer_len}
+            --minimizer-len ${database_type.minimizer_len}
+            --minimizer-spaces ${database_type.minimizer_spaces}
+            --load-factor ${database_type.load_factor}
+            ${database_type.clean}
+          #else if $database_type.database_type == "special"
+            --threads \${GALAXY_SLOTS:-1}
+            --special-database-type ${database_type.special_database_type}
+            --kmer-len ${database_type.kmer_len}
+            --minimizer-len ${database_type.minimizer_len}
+            --minimizer-spaces ${database_type.minimizer_spaces}
+            --load-factor ${database_type.load_factor}
+            ${database_type.clean}
+          #else if $database_type.database_type == "custom"
+            --threads \${GALAXY_SLOTS:-1}
+            ${database_type.skip_maps}
+            --custom-fasta ${database_type.custom_fasta}
+            --custom-database-name ${database_type.custom_database_name}
+            --kmer-len ${database_type.kmer_len}
+            --minimizer-len ${database_type.minimizer_len}
+            --minimizer-spaces ${database_type.minimizer_spaces}
+            --load-factor ${database_type.load_factor}
+            ${database_type.clean}
+          #end if
+    ]]>
+    </command>
+    <inputs>
+        <conditional name="database_type">
+            <param name="database_type" type="select" multiple="false" label="Database Type">
+                <option value="standard">Standard</option>
+                <option value="minikraken">MiniKraken</option>
+                <option value="special">Special</option>
+                <option value="custom">Custom</option>
+            </param>
+            <when value="standard">
+                <expand macro="common_params" />
+            </when>
+            <when value="minikraken">
+                <conditional name="kraken_release">
+                    <param name="minikraken2_release" type="select" multiple="false" label="Select the MiniKraken2 release to download">
+                        <option value="April_2019">April 2019</option>
+                        <option value="March_2020">March 2020</option>
+                    </param>
+                    <when value="April_2019">
+                        <param name="minikraken2_version" type="select" multiple="false" label="Select MiniKraken2 database version to download">
+                            <option value="v2">Version 2</option>
+                            <option value="v1">Version 1</option>
+                        </param>
+                    </when>
+                    <when value="March_2020" />
+                </conditional>
+            </when>
+            <when value="special">
+                <param name="special_database_type" type="select" multiple="false" label="Select database to build">
+                    <option value="greengenes">Greengenes</option>
+                    <option value="silva">Silva</option>
+                    <option value="rdp">RDP</option>
+                </param>
+                <expand macro="common_params" />
+            </when>
+            <when value="custom">
+                <param name="custom_fasta" type="data" format="fasta" multiple="False" optional="true" label="Select history item" />
+                <param name="custom_database_name" type="text" label="Name for this database" />
+                <param name="skip_maps" type="boolean" truevalue="--skip-maps" falsevalue="" label="Skip downloading accession number to taxid maps during taxonomy download." />
+                <expand macro="common_params" />
+            </when>
+        </conditional>
+    </inputs>
+    <outputs>
+        <data name="out_file" format="data_manager_json" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="database_type" value="custom" />
+            <param name="custom_fasta" value="adapter.fa" />
+            <param name="custom_database_name" value="database" />
+            <param name="skip_maps" value="true" />
+            <output name="out_file" value="kraken2_custom_data_manager.json" />
+        </test>
+    </tests>
+    <help>
+    </help>
+    <citations>
+        <citation type="doi">10.1186/gb-2014-15-3-r46</citation>
+    </citations>
+</tool>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,18 @@
+<data_managers>
+    <data_manager tool_file="data_manager/kraken2_build_database.xml" id="kraken2_build_database" version="2.0.8_beta+galaxy1">
+        <data_table name="kraken2_databases">
+            <output>
+                <column name="value"/>
+                <column name="name"/>
+                <column name="path" output_ref="out_file">
+                    <move type="directory">
+                        <source>${path}</source>
+                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">kraken2_databases/${path}</target>
+                    </move>
+                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/kraken2_databases/${path}</value_translation>
+                    <value_translation type="function">abspath</value_translation>
+                </column>
+            </output>
+        </data_table>
+    </data_manager>
+</data_managers>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/adapter.fa	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,2 @@
+>sequence16|kraken:taxid|32630  Adapter sequence
+CAAGCAGAAGACGGCATACGAGATCTTCGAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/adapter.fastq	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,4 @@
+@sequence16
+CAAGCAGAAGACGGCATACGAGATCTTCGAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA
++
+IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/kraken2_custom_data_manager.json	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,1 @@
+{"data_tables": {"kraken2_databases": [{"name": "database", "path": "database", "value": "database"}]}}
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample	Tue Dec 01 16:07:40 2020 +0000
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<tables>
+    <!-- Locations of Kraken2 database in the required format -->
+    <table name="kraken2_databases" comment_char="#">
+        <columns>value, name, path</columns>
+        <file path="tool-data/kraken2_databases.loc" />
+    </table>
+</tables>