Mercurial > repos > ieguinoa > data_manager_fetch_tx2gene

comparison data_manager/data_manager_fetch_tx2gene.py @ 3:d71f65b854de draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author ieguinoa
date Fri, 19 Oct 2018 07:36:02 -0400
parents a4d4da89aae1
children c380b7da5b65
comparison
equal deleted inserted replaced
2:7d3ffe28ff3f 3:d71f65b854de
9 from ftplib import FTP 9 from ftplib import FTP
10 import tarfile 10 import tarfile
11 import zipfile 11 import zipfile
12 import gzip 12 import gzip
13 import bz2 13 import bz2
14 import subprocess
14 try: 15 try:
15 # For Python 3.0 and later 16 # For Python 3.0 and later
16 from urllib.request import urlopen 17 from urllib.request import urlopen
17 from io import BytesIO as StringIO 18 from io import BytesIO as StringIO
18 from io import UnsupportedOperation 19 from io import UnsupportedOperation
91 92
92 def _get_stream_readers_for_bz2( fh, tmp_dir ): 93 def _get_stream_readers_for_bz2( fh, tmp_dir ):
93 return [ bz2.BZ2File( fh.name, 'rb') ] 94 return [ bz2.BZ2File( fh.name, 'rb') ]
94 95
95 96
96 def convert_tx2gene( fasta_filename, file_type, params ): 97 def convert_to_tx2gene( rscript_gff_to_tx2gene, fasta_filename, file_type, params ):
97 if file_type is 'tx2gene': 98 if file_type == 'tx2gene':
98 return #no need to extract tx2gene table 99 return #no need to extract tx2gene table
100 #print file_type
99 #If the file is actually a GFF/GTF file then extract the tx2gene 101 #If the file is actually a GFF/GTF file then extract the tx2gene
100 gff_temp_filename = tempfile.NamedTemporaryFile().name 102 gff_temp_filename = tempfile.NamedTemporaryFile().name
101 shutil.move(fasta_filename, gff_temp_filename) 103 shutil.move(fasta_filename, gff_temp_filename)
102 args= ['Rscript'] 104 args= ['Rscript']
103 args.append(RSCRIPT_GFF_TO_TX2GENE) 105 args.append(rscript_gff_to_tx2gene)
104 args.append(gff_temp_filename) 106 args.extend(['-x',gff_temp_filename])
105 args.append(fasta_filename) 107 args.extend(['-o',fasta_filename])
106 108 args.extend(['-t',file_type])
107 #assert sort_method in SORTING_METHODS, ValueError( "%s is not a valid sorting option." % sort_method ) 109 tmp_stderr = tempfile.NamedTemporaryFile( prefix = "tmp-stderr" )
108 #return SORTING_METHODS[ sort_method ]( fasta_filename, params ) 110 return_code = subprocess.call( args=args, shell=False, stderr=tmp_stderr.fileno() )
109 111 #return_code = subprocess.call( args=args, shell=False, stderr=None)
112 if return_code:
113 tmp_stderr.flush()
114 tmp_stderr.seek(0)
115 print >> sys.stderr, "Error in process call"
116 while True:
117 chunk = tmp_stderr.read( CHUNK_SIZE )
118 if not chunk:
119 break
120 sys.stderr.write( chunk )
121 sys.exit( return_code )
122 tmp_stderr.close()
123
124
125
110 def _download_file(start, fh): 126 def _download_file(start, fh):
111 tmp = tempfile.NamedTemporaryFile() 127 tmp = tempfile.NamedTemporaryFile()
112 tmp.write(start) 128 tmp.write(start)
113 tmp.write(fh.read()) 129 tmp.write(fh.read())
114 tmp.flush() 130 tmp.flush()
141 pass 157 pass
142 return fh 158 return fh
143 159
144 160
145 161
146 def add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params): 162 def add_fasta_to_table(rscript_gff_to_tx2gene, data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params):
147 for data_table_name, data_table_entry in _stream_fasta_to_file( fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params ): 163 for data_table_name, data_table_entry in _stream_fasta_to_file(rscript_gff_to_tx2gene, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params ):
148 if data_table_entry: 164 if data_table_entry:
149 _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name ) 165 _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name )
150 166
151 167
152 def download_from_url( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ): 168 def download_from_url(rscript_gff_to_tx2gene, data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ):
153 urls = filter( bool, map( lambda x: x.strip(), params['param_dict']['reference_source']['user_url'].split( '\n' ) ) ) 169 urls = filter( bool, map( lambda x: x.strip(), params['param_dict']['reference_source']['user_url'].split( '\n' ) ) )
154 fasta_readers = [ get_stream_reader(urlopen( url ), tmp_dir) for url in urls ] 170 fasta_readers = [ get_stream_reader(urlopen( url ), tmp_dir) for url in urls ]
155 add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id,sequence_name, params) 171 add_fasta_to_table(rscript_gff_to_tx2gene,data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id,sequence_name, params)
156 172
157 173
158 def download_from_history( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ): 174 def download_from_history(rscript_gff_to_tx2gene, data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ):
159 #TODO: allow multiple FASTA input files 175 #TODO: allow multiple FASTA input files
160 input_filename = params['param_dict']['reference_source']['input_fasta'] 176 input_filename = params['param_dict']['reference_source']['input_fasta']
161 if isinstance( input_filename, list ): 177 if isinstance( input_filename, list ):
162 fasta_readers = [ get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename ] 178 fasta_readers = [ get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename ]
163 else: 179 else:
164 fasta_readers = get_stream_reader(open(input_filename), tmp_dir) 180 fasta_readers = get_stream_reader(open(input_filename), tmp_dir)
165 add_fasta_to_table(data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params) 181 add_fasta_to_table(rscript_gff_to_tx2gene,data_manager_dict, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params)
166 182
167 183
168 def copy_from_directory( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ): 184 def copy_from_directory(rscript_gff_to_tx2gene, data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir ):
169 input_filename = params['param_dict']['reference_source']['fasta_filename'] 185 input_filename = params['param_dict']['reference_source']['fasta_filename']
170 create_symlink = params['param_dict']['reference_source']['create_symlink'] == 'create_symlink' 186 create_symlink = params['param_dict']['reference_source']['create_symlink'] == 'create_symlink'
171 if create_symlink: 187 if create_symlink:
172 data_table_entries = _create_symlink( input_filename, target_directory, dbkey, dbkey_name, sequence_id, sequence_name ) 188 data_table_entries = _create_symlink( input_filename, target_directory, dbkey, dbkey_name, sequence_id, sequence_name )
173 else: 189 else:
174 if isinstance( input_filename, list ): 190 if isinstance( input_filename, list ):
175 fasta_readers = [ get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename ] 191 fasta_readers = [ get_stream_reader(open(filename, 'rb'), tmp_dir) for filename in input_filename ]
176 else: 192 else:
177 fasta_readers = get_stream_reader(open(input_filename), tmp_dir) 193 fasta_readers = get_stream_reader(open(input_filename), tmp_dir)
178 data_table_entries = _stream_fasta_to_file( fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params ) 194 data_table_entries = _stream_fasta_to_file(rscript_gff_to_tx2gene, fasta_readers, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params )
179 for data_table_name, data_table_entry in data_table_entries: 195 for data_table_name, data_table_entry in data_table_entries:
180 if data_table_entry: 196 if data_table_entry:
181 _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name ) 197 _add_data_table_entry( data_manager_dict, data_table_entry, data_table_name )
182 198
183 199
186 data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get( DATA_TABLE_NAME, [] ) 202 data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get( DATA_TABLE_NAME, [] )
187 data_manager_dict['data_tables'][data_table_name].append( data_table_entry ) 203 data_manager_dict['data_tables'][data_table_name].append( data_table_entry )
188 return data_manager_dict 204 return data_manager_dict
189 205
190 206
191 def _stream_fasta_to_file( fasta_stream, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params, close_stream=True ): 207 def _stream_fasta_to_file( rscript_gff_to_tx2gene, fasta_stream, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, params, close_stream=True ):
192 fasta_base_filename = "%s_tx2gene.tab" % sequence_id 208 fasta_base_filename = "%s_tx2gene.tab" % sequence_id
193 fasta_filename = os.path.join( target_directory, fasta_base_filename ) 209 fasta_filename = os.path.join( target_directory, fasta_base_filename )
194 with open( fasta_filename, 'wb+' ) as fasta_writer: 210 with open( fasta_filename, 'wb+' ) as fasta_writer:
195 211
196 if isinstance( fasta_stream, list ) and len( fasta_stream ) == 1: 212 if isinstance( fasta_stream, list ) and len( fasta_stream ) == 1:
218 else: 234 else:
219 break 235 break
220 if close_stream: 236 if close_stream:
221 fasta_stream.close() 237 fasta_stream.close()
222 238
223 convert_to_tx2gene( fasta_filename, params['param_dict']['file_type'], params ) 239 convert_to_tx2gene( rscript_gff_to_tx2gene,fasta_filename, params['param_dict']['file_type'], params )
224 return [ ( DATA_TABLE_NAME, dict( value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_filename ) ) ] 240 return [ ( DATA_TABLE_NAME, dict( value=sequence_id, dbkey=dbkey, name=sequence_name, path=fasta_base_filename ) ) ]
225 241
226 242
227 def compute_fasta_length( fasta_file, out_file, keep_first_word=False ): 243 def compute_fasta_length( fasta_file, out_file, keep_first_word=False ):
228 244
269 285
270 def main(): 286 def main():
271 #Parse Command Line 287 #Parse Command Line
272 parser = optparse.OptionParser() 288 parser = optparse.OptionParser()
273 parser.add_option( '-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description' ) 289 parser.add_option( '-d', '--dbkey_description', dest='dbkey_description', action='store', type="string", default=None, help='dbkey_description' )
290 parser.add_option( '-b', '--base_dir', dest='base_dir', action='store', type='string', default=None, help='base_dir')
274 parser.add_option( '-t', '--type', dest='file_type', action='store', type='string', default=None, help='file_type') 291 parser.add_option( '-t', '--type', dest='file_type', action='store', type='string', default=None, help='file_type')
275 (options, args) = parser.parse_args() 292 (options, args) = parser.parse_args()
276 293
277 filename = args[0] 294 filename = args[0]
278 #global DATA_TABLE_NAME 295 #global DATA_TABLE_NAME
279 global RSCRIPT_GFF_TO_TX2GENE= os.path.join( options.base_dir, 'tximport.r') 296 rscript_gff_to_tx2gene=os.path.join( options.base_dir, 'get_tx2gene_table.R')
280 297
281 298 #input_type='gff_gtf'
282 if options.file_type == 'gff_gtf': 299 #if options.file_type != 'gff_gtf':
283 #DATA_TABLE_NAME= 'representative_gff' 300 # file_type='tx2gene'
284 else: #file_type='tx2gene'
285 301
286 params = loads( open( filename ).read() ) 302 params = loads( open( filename ).read() )
287 target_directory = params[ 'output_data' ][0]['extra_files_path'] 303 target_directory = params[ 'output_data' ][0]['extra_files_path']
288 os.mkdir( target_directory ) 304 os.mkdir( target_directory )
289 data_manager_dict = {} 305 data_manager_dict = {}
295 311
296 # Create a tmp_dir, in case a zip file needs to be uncompressed 312 # Create a tmp_dir, in case a zip file needs to be uncompressed
297 tmp_dir = tempfile.mkdtemp() 313 tmp_dir = tempfile.mkdtemp()
298 #Fetch the input file 314 #Fetch the input file
299 try: 315 try:
300 REFERENCE_SOURCE_TO_DOWNLOAD[ params['param_dict']['reference_source']['reference_source_selector'] ]( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir) 316 REFERENCE_SOURCE_TO_DOWNLOAD[ params['param_dict']['reference_source']['reference_source_selector'] ]( rscript_gff_to_tx2gene, data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name, tmp_dir)
301 finally: 317 finally:
302 cleanup_before_exit(tmp_dir) 318 cleanup_before_exit(tmp_dir)
303 #save info to json file 319 #save info to json file
304 open( filename, 'wb' ).write( dumps( data_manager_dict ).encode() ) 320 open( filename, 'wb' ).write( dumps( data_manager_dict ).encode() )
305 321