comparison data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py @ 8:14eb0fc65c62 draft

"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_fetch_genome_dbkeys_all_fasta commit b56485a8b386fc6f17219850e30e5656c159f231"

7:b1bc53e9bbc5

#!/usr/bin/env python
# Dan Blankenberg

import bz2
import gzip
import optparse
import os
import shutil
import sys
import tarfile
import tempfile
import zipfile
from ftplib import FTP
from json import dumps, loads

try:
    # For Python 3.0 and later
    from io import BytesIO as StringIO
    from io import UnsupportedOperation

    current_order = [_[1] for _ in sorted((_[1], _[0]) for _ in fasta_offsets.items())]
    return (unsorted_filename, fasta_offsets, current_order)


def _write_sorted_fasta(sorted_names, fasta_offsets, sorted_fasta_filename, unsorted_fasta_filename):
    unsorted_fh = open(unsorted_fasta_filename)
    sorted_fh = open(sorted_fasta_filename, 'wb+')

    for name in sorted_names:
        offset = fasta_offsets[name]
        unsorted_fh.seek(offset)
        sorted_fh.write(unsorted_fh.readline())
        while True:
            line = unsorted_fh.readline()
            if not line or line.startswith(">"):
                break
            sorted_fh.write(line)
    unsorted_fh.close()
    sorted_fh.close()


def _sort_fasta_as_is(fasta_filename, params):
    return

def download_from_url(params, tmp_dir, **kwds):
    """
    Download a file from a URL and return a list of filehandles from which to read the data.

    >>> url = 'https://github.com/mvdbeek/tools-devteam/raw/data_manager/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/test.tar'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> tmp_dir = tempfile.mkdtemp()
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('>FBtr0304171')
    >>> url = 'https://github.com/mvdbeek/tools-devteam/raw/data_manager/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/test.tar.bz2'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('>FBtr0304171')
    >>> url = 'https://github.com/mvdbeek/tools-devteam/raw/data_manager/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/test.tar.gz'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('>FBtr0304171')
    >>> url = 'https://github.com/mvdbeek/tools-devteam/raw/data_manager/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/test.zip'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('>FBtr0304171')
    >>> url = 'https://raw.githubusercontent.com/galaxyproject/tools-devteam/master/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/phiX174.fasta'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('>phiX174')
    """
    urls = filter(bool, [x.strip() for x in params['param_dict']['reference_source']['user_url'].split('\n')])
    return [get_stream_reader(urlopen(url), tmp_dir) for url in urls]

    # TODO: allow multiple FASTA input files
    input_filename = params['param_dict']['reference_source']['input_fasta']
    if isinstance(input_filename, list):
        fasta_readers = [get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename]
    else:
        fasta_readers = get_stream_reader(open(input_filename), tmp_dir)
    return fasta_readers


def copy_from_directory(data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir):
    input_filename = params['param_dict']['reference_source']['fasta_filename']

    parser.add_option('-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description')
    (options, args) = parser.parse_args()

    filename = args[0]

    params = loads(open(filename).read())
    target_directory = params['output_data'][0]['extra_files_path']
    os.mkdir(target_directory)
    data_manager_dict = {}

    dbkey, dbkey_name, sequence_id, sequence_name = get_dbkey_dbname_id_name(params, dbkey_description=options.dbkey_description)

                               params=params)

    finally:
        cleanup_before_exit(tmp_dir)
    # save info to json file
    open(filename, 'wb').write(dumps(data_manager_dict).encode())


if __name__ == "__main__":
    main()

8:14eb0fc65c62

#!/usr/bin/env python
# Dan Blankenberg

import bz2
import gzip
import json
import optparse
import os
import shutil
import sys
import tarfile
import tempfile
import zipfile
from ftplib import FTP

try:
    # For Python 3.0 and later
    from io import BytesIO as StringIO
    from io import UnsupportedOperation

    current_order = [_[1] for _ in sorted((_[1], _[0]) for _ in fasta_offsets.items())]
    return (unsorted_filename, fasta_offsets, current_order)


def _write_sorted_fasta(sorted_names, fasta_offsets, sorted_fasta_filename, unsorted_fasta_filename):
    with open(unsorted_fasta_filename, 'rb') as unsorted_fh, open(sorted_fasta_filename, 'wb+') as sorted_fh:
        for name in sorted_names:
            offset = fasta_offsets[name]
            unsorted_fh.seek(offset)
            sorted_fh.write(unsorted_fh.readline())
            while True:
                line = unsorted_fh.readline()
                if not line or line.startswith(b">"):
                    break
                sorted_fh.write(line)


def _sort_fasta_as_is(fasta_filename, params):
    return

def download_from_url(params, tmp_dir, **kwds):
    """
    Download a file from a URL and return a list of filehandles from which to read the data.

    >>> tmp_dir = tempfile.mkdtemp()
    >>> url = 'https://github.com/galaxyproject/tools-iuc/raw/master/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/test.tar.bz2'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('b>FBtr0304171')
    >>> url = 'https://github.com/galaxyproject/tools-iuc/raw/master/data_managers/data_manager_fetch_genome_dbkeys_all_fasta/test-data/phiX174.fasta'
    >>> params = {'param_dict': {'reference_source': {'user_url': url}}}
    >>> fh = download_from_url(params=params, tmp_dir=tmp_dir)[0][0]
    >>> assert fh.readline().startswith('b>phiX174')
    """
    urls = filter(bool, [x.strip() for x in params['param_dict']['reference_source']['user_url'].split('\n')])
    return [get_stream_reader(urlopen(url), tmp_dir) for url in urls]

    # TODO: allow multiple FASTA input files
    input_filename = params['param_dict']['reference_source']['input_fasta']
    if isinstance(input_filename, list):
        fasta_readers = [get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename]
    else:
        fasta_readers = get_stream_reader(open(input_filename, 'rb'), tmp_dir)
    return fasta_readers


def copy_from_directory(data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir):
    input_filename = params['param_dict']['reference_source']['fasta_filename']

    parser.add_option('-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description')
    (options, args) = parser.parse_args()

    filename = args[0]

    with open(filename) as fh:
        params = json.load(fh)
    target_directory = params['output_data'][0]['extra_files_path']
    os.mkdir(target_directory)
    data_manager_dict = {}

    dbkey, dbkey_name, sequence_id, sequence_name = get_dbkey_dbname_id_name(params, dbkey_description=options.dbkey_description)

                               params=params)

    finally:
        cleanup_before_exit(tmp_dir)
    # save info to json file
    with open(filename, 'w') as fh:
        json.dump(data_manager_dict, fh, sort_keys=True)


if __name__ == "__main__":
    main()