comparison data_manager/data_manager_fetch_genome_all_fasta.py @ 1:ac850912d386 draft

Uploaded

0:2ebc856bce29

import tarfile
import zipfile
import gzip
import bz2

from galaxy.util.json import from_json_string, to_json_string


CHUNK_SIZE = 2**20 #1mb

def cleanup_before_exit( tmp_dir ):

    ftp.retrlines( 'MLSD %s' % ( path ), path_contents.append )
    return [ line.split( ';' )[ -1 ].lstrip() for line in path_contents ]

def _get_stream_readers_for_tar( file_obj, tmp_dir ):
    fasta_tar = tarfile.open( fileobj=file_obj, mode='r:*' )
    return [ fasta_tar.extractfile( member ) for member in fasta_tar.getmembers() ]

def _get_stream_readers_for_zip( file_obj, tmp_dir ):
    fasta_zip = zipfile.ZipFile( file_obj, 'r' )
    rval = []
    for member in fasta_zip.namelist():

    else:
        _write_sorted_fasta( sorted_names, fasta_offsets, fasta_filename, unsorted_filename )

def download_from_ucsc( data_manager_dict, params, target_directory, dbkey, sequence_id, sequence_name ):
    UCSC_FTP_SERVER = 'hgdownload.cse.ucsc.edu'
    UCSC_CHROM_FA_FILENAME = 'chromFa'
    UCSC_DOWNLOAD_PATH = '/goldenPath/%s/bigZips/'
    COMPRESSED_EXTENSIONS = [ ( '.tar.gz', _get_stream_readers_for_tar ), ( '.tar.bz2', _get_stream_readers_for_tar ), ( '.zip', _get_stream_readers_for_zip ), ( '.fa.gz', _get_stream_readers_for_gzip ), ( '.fa.bz2', _get_stream_readers_for_bz2 ) ]
    
    email = params['param_dict']['__user_email__']
    if not email:
        email = '[e-mail]'

    ucsc_dbkey = params['param_dict']['reference_source']['requested_dbkey'] or dbkey
    ftp = FTP( UCSC_FTP_SERVER )
    ftp.login( 'anonymous', email )
    
    ucsc_path = UCSC_DOWNLOAD_PATH % ucsc_dbkey
    path_contents = _get_files_in_ftp_path( ftp, ucsc_path )
    
    ucsc_file_name = None
    get_stream_reader = None
    ext = None
    for ext, get_stream_reader in COMPRESSED_EXTENSIONS:
        if "%s%s" % ( UCSC_CHROM_FA_FILENAME, ext ) in path_contents:
            ucsc_file_name = "%s%s%s" % ( ucsc_path, UCSC_CHROM_FA_FILENAME, ext )
            break
    
    if not ucsc_file_name:
        raise Exception( 'Unable to determine filename for UCSC Genome for %s: %s' % ( ucsc_dbkey, path_contents ) )
    
    
    tmp_dir = tempfile.mkdtemp( prefix='tmp-data-manager-ucsc-' )
    ucsc_fasta_filename = os.path.join( tmp_dir, "%s%s" % ( UCSC_CHROM_FA_FILENAME, ext ) )
    
    fasta_base_filename = "%s.fa" % sequence_id
    fasta_filename = os.path.join( target_directory, fasta_base_filename )
    fasta_writer = open( fasta_filename, 'wb+' )
    

    parser.add_option( '-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description' )
    (options, args) = parser.parse_args()
    
    filename = args[0]
    
    params = from_json_string( open( filename ).read() )
    target_directory = params[ 'output_data' ][0]['extra_files_path']
    os.mkdir( target_directory )
    data_manager_dict = {}
    
    dbkey, sequence_id, sequence_name = get_dbkey_id_name( params, dbkey_description=options.dbkey_description ) 

    
    #Fetch the FASTA
    REFERENCE_SOURCE_TO_DOWNLOAD[ params['param_dict']['reference_source']['reference_source_selector'] ]( data_manager_dict, params, target_directory, dbkey, sequence_id, sequence_name )
    
    #save info to json file
    open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
        
if __name__ == "__main__": main()

1:ac850912d386

import tarfile
import zipfile
import gzip
import bz2

from json import loads, dumps


CHUNK_SIZE = 2**20 #1mb

def cleanup_before_exit( tmp_dir ):

    ftp.retrlines( 'MLSD %s' % ( path ), path_contents.append )
    return [ line.split( ';' )[ -1 ].lstrip() for line in path_contents ]

def _get_stream_readers_for_tar( file_obj, tmp_dir ):
    fasta_tar = tarfile.open( fileobj=file_obj, mode='r:*' )
    return filter( lambda x: x is not None, [ fasta_tar.extractfile( member ) for member in fasta_tar.getmembers() ] )

def _get_stream_readers_for_zip( file_obj, tmp_dir ):
    fasta_zip = zipfile.ZipFile( file_obj, 'r' )
    rval = []
    for member in fasta_zip.namelist():

    else:
        _write_sorted_fasta( sorted_names, fasta_offsets, fasta_filename, unsorted_filename )

def download_from_ucsc( data_manager_dict, params, target_directory, dbkey, sequence_id, sequence_name ):
    UCSC_FTP_SERVER = 'hgdownload.cse.ucsc.edu'
    UCSC_DOWNLOAD_PATH = '/goldenPath/%s/bigZips/'
    COMPRESSED_EXTENSIONS = [ ( '.tar.gz', _get_stream_readers_for_tar ), ( '.tar.bz2', _get_stream_readers_for_tar ), ( '.zip', _get_stream_readers_for_zip ), ( '.fa.gz', _get_stream_readers_for_gzip ), ( '.fa.bz2', _get_stream_readers_for_bz2 ) ]
    
    email = params['param_dict']['__user_email__']
    if not email:
        email = '[e-mail]'

    ucsc_dbkey = params['param_dict']['reference_source']['requested_dbkey'] or dbkey
    UCSC_CHROM_FA_FILENAMES = [ '%s.chromFa' % ucsc_dbkey, 'chromFa' ]
    
    ftp = FTP( UCSC_FTP_SERVER )
    ftp.login( 'anonymous', email )
    
    ucsc_path = UCSC_DOWNLOAD_PATH % ucsc_dbkey
    path_contents = _get_files_in_ftp_path( ftp, ucsc_path )
    
    ucsc_file_name = None
    get_stream_reader = None
    ext = None
    ucsc_chrom_fa_filename = None
    for ucsc_chrom_fa_filename in UCSC_CHROM_FA_FILENAMES:
        for ext, get_stream_reader in COMPRESSED_EXTENSIONS:
            if "%s%s" % ( ucsc_chrom_fa_filename, ext ) in path_contents:
                ucsc_file_name = "%s%s%s" % ( ucsc_path, ucsc_chrom_fa_filename, ext )
                break
        if ucsc_file_name:
            break
    
    if not ucsc_file_name:
        raise Exception( 'Unable to determine filename for UCSC Genome for %s: %s' % ( ucsc_dbkey, path_contents ) )
    
    
    tmp_dir = tempfile.mkdtemp( prefix='tmp-data-manager-ucsc-' )
    ucsc_fasta_filename = os.path.join( tmp_dir, "%s%s" % ( ucsc_chrom_fa_filename, ext ) )
    
    fasta_base_filename = "%s.fa" % sequence_id
    fasta_filename = os.path.join( target_directory, fasta_base_filename )
    fasta_writer = open( fasta_filename, 'wb+' )
    

    parser.add_option( '-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description' )
    (options, args) = parser.parse_args()
    
    filename = args[0]
    
    params = loads( open( filename ).read() )
    target_directory = params[ 'output_data' ][0]['extra_files_path']
    os.mkdir( target_directory )
    data_manager_dict = {}
    
    dbkey, sequence_id, sequence_name = get_dbkey_id_name( params, dbkey_description=options.dbkey_description ) 

    
    #Fetch the FASTA
    REFERENCE_SOURCE_TO_DOWNLOAD[ params['param_dict']['reference_source']['reference_source_selector'] ]( data_manager_dict, params, target_directory, dbkey, sequence_id, sequence_name )
    
    #save info to json file
    open( filename, 'wb' ).write( dumps( data_manager_dict ) )
        
if __name__ == "__main__": main()