changeset 0:8b91891ae805 draft

planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_fetch_refseq commit a572f1f01161527ff6ed4af05bb2e073a8ca903b
author iuc
date Mon, 01 Oct 2018 15:36:01 -0400
parents
children 937f167631b1
files data_manager/fetch_refseq.py data_manager/fetch_refseq.xml data_manager_conf.xml test-data/plastid.json tool-data/all_fasta.loc.sample tool_data_table_conf.xml.sample
diffstat 6 files changed, 299 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/fetch_refseq.py	Mon Oct 01 15:36:01 2018 -0400
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+
+from __future__ import division, print_function
+
+import argparse
+import functools
+import gzip
+import json
+import os
+import os.path
+import sys
+from datetime import date
+from multiprocessing import Process, Queue
+
+import requests
+
+try:
+    from io import StringIO
+except ImportError:
+    from StringIO import StringIO
+# Refseq structure
+# - Release number
+# - Divisions
+#   1. archea
+#   2. bacteria
+#   3. fungi
+#   4. invertebrate
+#   5. mitochondrion
+#   6. other
+#   7. plant
+#   8. plasmid
+#   9. plastid
+#  10. protozoa
+#  11. vertebrate mammalian
+#  12. vertebrate other
+#  13. viral
+# within each division
+# DIVNAME.\d+(.\d+)?.(genomic|protein|rna).(fna|gbff|faa|gpff).gz
+#  where fna and faa are FASTA, gbff and gpff are Genbank
+
+
+def _add_data_table_entry(data_manager_dict, data_table_entry, data_table_name):
+    data_manager_dict['data_tables'] = data_manager_dict.get('data_tables', {})
+    data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get('all_fasta', [])
+    data_manager_dict['data_tables'][data_table_name].append(data_table_entry)
+    return data_manager_dict
+
+
+def unzip_to(conn, out_dir, output_filename, chunk_size=4096, debug=False, compress=False):
+    input_filename = conn.get()
+    if compress:
+        open_output = gzip.open
+    else:
+        open_output = open
+    with open_output(os.path.join(out_dir, output_filename), 'wb') as output_file:
+        while input_filename != 'STOP':
+            if debug:
+                print('Reading', input_filename, file=sys.stderr)
+            with gzip.open(input_filename, 'rb') as input_file:
+                read_chunk = functools.partial(input_file.read, (chunk_size))
+                for data in iter(read_chunk, b''):  # use b'' as a sentinel to stop the loop. note '' != b'' in Python 3
+                    output_file.write(data)
+            os.unlink(input_filename)
+            input_filename = conn.get()
+
+
+def get_refseq_division(division_name, mol_types, output_directory, debug=False, compress=False):
+    base_url = 'https://ftp.ncbi.nlm.nih.gov/refseq/release/'
+    valid_divisions = set(['archea', 'bacteria', 'complete', 'fungi', 'invertebrate', 'mitochondrion', 'other',
+                          'plant', 'plasmid', 'plastid', 'protozoa', 'vertebrate_mammalian', 'vertebrate_other', 'viral'])
+    ending_mappings = {
+        'genomic': '.genomic.fna.gz',
+        'protein': '.protein.faa.gz',
+        'rna': 'rna.fna.gz'
+    }
+    assert division_name in valid_divisions, "Unknown division name ({})".format(division_name)
+    for mol_type in mol_types:
+        assert mol_type in ending_mappings, "Unknown molecule type ({})".format(mol_type)
+    if not os.path.exists(output_directory):
+        os.mkdir(output_directory)
+    release_num_file = base_url + 'RELEASE_NUMBER'
+    r = requests.get(release_num_file)
+    release_num = r.text.strip()
+    division_base_url = base_url + division_name
+    if debug:
+        print('Retrieving {}'.format(division_base_url), file=sys.stderr)
+    r = requests.get(division_base_url)
+    listing_text = r.text
+
+    unzip_queues = {}
+    unzip_processes = []
+    final_output_filenames = []
+    for mol_type in mol_types:
+        q = unzip_queues[mol_type] = Queue()
+        output_filename = division_name + '.' + release_num + '.' + mol_type + '.fasta'
+        if compress:
+            output_filename += '.gz'
+        final_output_filenames.append(output_filename)
+        unzip_processes.append(Process(target=unzip_to, args=(q, output_directory, output_filename),
+                                       kwargs=dict(debug=debug, compress=compress)))
+        unzip_processes[-1].start()
+
+    # sample line: <a href="vertebrate_other.86.genomic.gbff.gz">vertebrate_other.86.genomic.gbff.gz</a>   2018-07-13 00:59   10M
+    for line in StringIO(listing_text):
+        if '.gz' not in line:
+            continue
+        parts = line.split('"')
+        assert len(parts) == 3, "Unexpected line format: {}".format(line.rstrip())
+        filename = parts[1]
+        for mol_type in mol_types:
+            ending = ending_mappings[mol_type]
+            if filename.endswith(ending):
+                if debug:
+                    print('Downloading:', filename, ending, mol_type, file=sys.stderr)
+                output_filename = os.path.join(output_directory, filename)
+                with open(output_filename, 'wb') as output_file:
+                    r = requests.get(division_base_url + '/' + filename)
+                    for chunk in r.iter_content(chunk_size=4096):
+                        output_file.write(chunk)
+                conn = unzip_queues[mol_type]
+                conn.put(output_filename)
+
+    for mol_type in mol_types:
+        conn = unzip_queues[mol_type]
+        conn.put('STOP')
+
+    return [release_num, final_output_filenames]
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Download RefSeq databases')
+    parser.add_argument('--debug', default=False, action='store_true', help='Print debugging output to stderr (verbose)')
+    parser.add_argument('--compress', default=False, action='store_true', help='Compress output files')
+    parser.add_argument('--output_directory', default='tmp', help='Directory to write output to')
+    parser.add_argument('--galaxy_datamanager_filename', help='Galaxy JSON format file describing data manager inputs')
+    parser.add_argument('--division_names', help='RefSeq divisions to download')
+    parser.add_argument('--mol_types', help='Molecule types (genomic, rna, protein) to fetch')
+    parser.add_argument('--pin_date', help='Force download date to this version string')
+    args = parser.parse_args()
+
+    division_names = args.division_names.split(',')
+    mol_types = args.mol_types.split(',')
+    if args.galaxy_datamanager_filename is not None:
+        dm_opts = json.loads(open(args.galaxy_datamanager_filename).read())
+        output_directory = dm_opts['output_data'][0]['extra_files_path']  # take the extra_files_path of the first output parameter
+        data_manager_dict = {}
+    else:
+        output_directory = args.output_directory
+    for division_name in division_names:
+        if args.pin_date is not None:
+            today_str = args.pin_date
+        else:
+            today_str = date.today().strftime('%Y-%m-%d')  # ISO 8601 date format
+        [release_num, fasta_files] = get_refseq_division(division_name, mol_types, output_directory, args.debug, args.compress)
+        if args.galaxy_datamanager_filename is not None:
+            for i, mol_type in enumerate(mol_types):
+                assert mol_type in fasta_files[i], "Filename does not contain expected mol_type ({}, {})".format(mol_type, fasta_files[i])
+                unique_key = 'refseq_' + division_name + '.' + release_num + '.' + mol_type  # note: this is now same as dbkey
+                dbkey = unique_key
+                desc = 'RefSeq ' + division_name + ' Release ' + release_num + ' ' + mol_type + ' (' + today_str + ')'
+                path = os.path.join(output_directory, fasta_files[i])
+                _add_data_table_entry(data_manager_dict=data_manager_dict,
+                                      data_table_entry=dict(value=unique_key, dbkey=dbkey, name=desc, path=path),
+                                      data_table_name='all_fasta')
+            open(args.galaxy_datamanager_filename, 'w').write(json.dumps(data_manager_dict, sort_keys=True))
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager/fetch_refseq.xml	Mon Oct 01 15:36:01 2018 -0400
@@ -0,0 +1,88 @@
+<tool id="data_manager_fetch_refseq" name="RefSeq data manager" version="0.0.18" tool_type="manage_data">
+    <description>Fetch FASTA data from NCBI RefSeq and update all_fasta data table</description>
+    <requirements>
+        <requirement type="package">python</requirement>
+        <requirement type="package">requests</requirement>
+    </requirements>
+    <command detect_errors="exit_code"><![CDATA[
+    python $__tool_directory__/fetch_refseq.py
+      #if str( $advanced.advanced_selector ) == 'advanced':
+        '${advanced.compress}'
+      #end if
+      --galaxy_datamanager_filename '${output_file}'
+      --division_names ${division_names}
+      --mol_types ${mol_types}
+      #if str( $pin_date ) != 'NO':
+        --pin_date '${pin_date}'
+      #end if
+    ]]></command>
+    <inputs>
+        <param argument="division_names" type="select" label="RefSeq division" multiple="true">
+            <option value="archea">Archea</option>
+            <option value="bacteria">Bacteria</option>
+            <option value="complete">Complete</option>
+            <option value="fungi">Fungi</option>
+            <option value="invertebrate">Invertebrate</option>
+            <option value="mitochondrion">Mitochondrion</option>
+            <option value="other">Other</option>
+            <option value="plant">Plant</option>
+            <option value="plasmid">Plasmid</option>
+            <option value="plastid">Plastid</option>
+            <option value="protozoa">Protozoa</option>
+            <option value="vertebrate_mammalian">Mammalian Vertebrate</option>
+            <option value="vertebrate_other">Other Vertebrate</option>
+            <option value="viral">Viral</option>
+        </param>
+        <param argument="mol_types" type="select" multiple="true" label="Molecule type" help="Select at least one of genomic, protein or rna sequence">
+            <option value="protein">Protein</option>
+            <option value="genomic">Genomic (DNA)</option>
+            <option value="rna">RNA</option>
+        </param>
+        <conditional name="advanced">
+            <param name="advanced_selector" type="select" label="Advanced Options">
+                <option value="basic" selected="True">Basic</option>
+                <option value="advanced">Advanced</option>
+            </param>
+            <when value="basic">
+            </when>
+            <when value="advanced">
+                <param type="boolean" argument="--compress" truevalue="--compress" falsevalue="" label="Compress FASTA files" 
+                    help="Compress downloaded FASTA files (with gzip). Limits compatibility with tools expecting uncompressed FASTA."/>
+            </when>
+        </conditional>
+        <param argument="--pin_date" type="hidden" value="NO" help="Used for testing"/>
+    </inputs>
+    <outputs>
+        <data name="output_file" format="data_manager_json"/>
+    </outputs>
+    <tests>
+        <test>
+            <param name="division_names" value="plastid"/>
+            <param name="mol_types" value="protein"/>
+            <param name="pin_date" value="2018-03-14"/>            
+            <param name="advanced_selector" value="basic"/>
+            <output name="output_file">
+                <assert_contents>
+                    <has_text text="2018-03-14"/>
+                    <has_text text="refseq_plastid"/>
+                    <has_text text="/refseq_plastid."/>
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+    <help><![CDATA[
+This data manager fetches FASTA format collections of proteins, nucleotides (genomic DNA) and RNA
+from NCBI's RefSeq_ data collection.
+
+RefSeq is released every two months and consists of a number of divisions. Some sequences are shared
+between multiple divisions. This data manager allows the Galaxy administrator to select which
+divisions and which molecule types within each division to download. Once downloaded the
+files are made accessible by adding an entry into the *all_fasta* data table.
+
+.. _RefSeq: https://www.ncbi.nlm.nih.gov/refseq/
+    ]]>
+    </help>
+    <citations>
+        <citation type="doi">10.1093/nar/gkv1189</citation>
+    </citations>
+</tool>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/data_manager_conf.xml	Mon Oct 01 15:36:01 2018 -0400
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<data_managers>    
+    <data_manager tool_file="data_manager/fetch_refseq.xml" id="fetch_genome_fetch_refseq">
+        <data_table name="all_fasta">
+            <output>
+                <column name="value" />
+                <column name="dbkey" />
+                <column name="name" />
+                <column name="path" output_ref="output_file" >
+                    <move type="file">
+                        <source>${path}</source>
+                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">refseq/#echo str($dbkey).split('.')[1]#/${value}.fasta</target>
+                    </move>
+                    <value_translation>refseq/#echo str($dbkey).split('.')[1]#/${value}.fasta</value_translation>
+                    <value_translation type="function">abspath</value_translation>
+                </column>
+            </output>
+        </data_table>
+    </data_manager>    
+</data_managers>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/plastid.json	Mon Oct 01 15:36:01 2018 -0400
@@ -0,0 +1,1 @@
+{"data_tables": {"all_fasta": [{"path": "tmp/plastid.89.protein.fasta.gz", "dbkey": "plastid.89.protein", "name": "RefSeq plastid Release 89 protein (2018-09-07)", "value": "plastid.89.protein.2018-03-14"}]}}
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool-data/all_fasta.loc.sample	Mon Oct 01 15:36:01 2018 -0400
@@ -0,0 +1,18 @@
+#This file lists the locations and dbkeys of all the fasta files
+#under the "genome" directory (a directory that contains a directory
+#for each build). The script extract_fasta.py will generate the file
+#all_fasta.loc. This file has the format (white space characters are
+#TAB characters):
+#
+#<unique_build_id>	<dbkey>		<display_name>	<file_path>
+#
+#So, all_fasta.loc could look something like this:
+#
+#apiMel3	apiMel3	Honeybee (Apis mellifera): apiMel3		/path/to/genome/apiMel3/apiMel3.fa
+#hg19canon	hg19		Human (Homo sapiens): hg19 Canonical		/path/to/genome/hg19/hg19canon.fa
+#hg19full	hg19		Human (Homo sapiens): hg19 Full			/path/to/genome/hg19/hg19full.fa
+#
+#Your all_fasta.loc file should contain an entry for each individual
+#fasta file. So there will be multiple fasta files for each build,
+#such as with hg19 above.
+#
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tool_data_table_conf.xml.sample	Mon Oct 01 15:36:01 2018 -0400
@@ -0,0 +1,7 @@
+<tables>
+    <!-- Locations of all fasta files under genome directory -->
+    <table name="all_fasta" comment_char="#">
+        <columns>value, dbkey, name, path</columns>
+        <file path="tool-data/all_fasta.loc" />
+    </table>
+</tables>
\ No newline at end of file