annotate extract_tables.py @ 1:9681a9180730 draft

Deleted selected files
author ieguinoa
date Wed, 02 Feb 2022 17:30:12 +0000
parents 5d59238cd3f4
children 2f7a70c0d3ab
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
1 import argparse
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
2 import json
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
3 import pathlib
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
4 from datetime import datetime
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
5
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
6 """
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
7 Parse the configfile generated by the Galaxy tool.
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
8 This file is JSON-formatted and should be converted to a set of tabular files.
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
9 """
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
10
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
11 FILE_FORMAT = 'fastq'
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
12
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
13 parser = argparse.ArgumentParser()
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
14 parser.add_argument('--studies', dest='studies_json_path', required=True)
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
15 parser.add_argument('--out_dir', dest='out_path', required=True)
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
16 parser.add_argument('--action', dest='action', required=True)
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
17 args = parser.parse_args()
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
18
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
19 with open(args.studies_json_path, 'r') as studies_json_file:
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
20 studies_dict = json.load(studies_json_file)
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
21 studies_table = open(pathlib.Path(args.out_path) / 'studies.tsv', 'w')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
22 studies_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'study_type',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
23 'study_abstract', 'pubmed_id', 'submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
24 samples_table = open(pathlib.Path(args.out_path) / 'samples.tsv', 'w')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
25 experiments_table = open(pathlib.Path(args.out_path) / 'experiments.tsv', 'w')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
26 experiments_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'study_alias',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
27 'sample_alias', 'design_description', 'library_name',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
28 'library_strategy', 'library_source', 'library_selection',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
29 'library_layout', 'insert_size',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
30 'library_construction_protocol', 'platform', 'instrument_model',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
31 'submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
32 runs_table = open(pathlib.Path(args.out_path) / 'runs.tsv', 'w')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
33 runs_table.write('\t'.join(['alias', 'status', 'accession', 'experiment_alias', 'file_name',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
34 'file_format', 'file_checksum', 'submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
35
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
36 action = args.action
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
37
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
38 dt_oobj = datetime.now(tz=None)
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
39 timestamp = dt_oobj.strftime("%Y%m%d_%H:%M:%S")
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
40 for study_index, study in enumerate(studies_dict):
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
41 study_alias = 'study_' + str(study_index) + '_' + timestamp
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
42 studies_table.write('\t'.join([study_alias, action, 'ENA_accession', study['title'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
43 study['type'], study['abstract'], study['pubmed_id'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
44 'ENA_submission_data']))
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
45 if "geo_location" in study['samples'][0].keys(): # sample belongs to a viral sample
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
46 samples_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'scientific_name',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
47 'taxon_id', 'sample_description', 'collection_date',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
48 'geographic_location', 'host_common_name', 'host_subject_id',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
49 'host_health_state', 'host_sex', 'host_scientific_name',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
50 'collector_name', 'collecting_institution', 'isolate',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
51 'submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
52 else:
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
53 samples_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'scientific_name',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
54 'taxon_id', 'sample_description', 'submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
55 for sample_index, sample in enumerate(study['samples']):
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
56 sample_alias = 'sample_' + str(sample_index) + '_' + timestamp
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
57 if "geo_location" in sample.keys(): # sample belongs to a viral sample
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
58 if sample['collector_name'] == '':
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
59 sample['collector_name'] = 'unknown'
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
60 samples_table.write('\t'.join([sample_alias, action, 'ena_accession', sample['title'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
61 sample['tax_name'], sample['tax_id'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
62 sample['description'], sample['collection_date'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
63 sample['geo_location'], sample['host_common_name'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
64 sample['host_subject_id'], sample['host_health_state'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
65 sample['host_sex'], sample['host_scientific_name'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
66 sample['collector_name'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
67 sample['collecting_institution'], sample['isolate'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
68 'ENA_submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
69 else:
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
70 samples_table.write('\t'.join([sample_alias, action, 'ena_accession', sample['title'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
71 sample['tax_name'], sample['tax_id'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
72 sample['description'], 'ENA_submission_date']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
73 for exp_index, exp in enumerate(sample['experiments']):
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
74 exp_alias = 'experiment_' + str(exp_index) + '.' + str(sample_index) + '_' + timestamp
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
75 lib_alias = 'library_' + str(exp_index) + '_' + str(sample_index)
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
76 experiments_table.write('\t'.join([exp_alias, action, 'accession_ena', exp['title'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
77 study_alias, sample_alias, exp['experiment_design'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
78 lib_alias, exp['library_strategy'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
79 exp['library_source'], exp['library_selection'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
80 exp['library_layout'].lower(), exp['insert_size'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
81 exp['library_construction_protocol'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
82 exp['platform'], exp['instrument_model'],
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
83 'submission_date_ENA']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
84 run_index = 0
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
85 # exp['runs'] is a list of lists
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
86 for (base_run, run_files) in exp['runs']:
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
87 run_index += 1
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
88 if base_run != '':
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
89 run_alias = base_run
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
90 else:
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
91 # no alias provided, generated a unique one
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
92 run_alias = '_'.join(['run_' + str(run_index), str(exp_index),
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
93 str(sample_index)]) + '_' + timestamp
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
94 for file_entry in run_files:
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
95 runs_table.write('\t'.join([run_alias, action, 'ena_run_accession', exp_alias,
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
96 file_entry, FILE_FORMAT, 'file_checksum',
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
97 'submission_date_ENA']) + '\n')
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
98
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
99 studies_table.close()
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
100 samples_table.close()
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
101 experiments_table.close()
5d59238cd3f4 Uploaded
ieguinoa
parents:
diff changeset
102 runs_table.close()