| Previous changeset 1:57251c760cab (2021-04-30) Next changeset 3:59bb6d34fca6 (2021-08-18) |
|
Commit message:
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/ena_upload commit 1eed23745846ce215e9bdc4a4934d6bc8f41b24e" |
|
modified:
ena_upload.xml process_xlsx.py samples_macros.xml |
|
added:
mappings.py test-data/metadata_test_viral_optional_columns.xlsx |
| b |
| diff -r 57251c760cab -r 9e2df763086c ena_upload.xml --- a/ena_upload.xml Fri Apr 30 12:09:25 2021 +0000 +++ b/ena_upload.xml Thu Jul 15 20:12:34 2021 +0000 |
| [ |
| @@ -1,6 +1,6 @@ -<tool id="ena_upload" name="ENA Upload tool" version="0.3.1" profile="20.01" license="MIT"> +<tool id="ena_upload" name="ENA Upload tool" version="0.3.2" profile="20.01" license="MIT"> <macros> - <token name="@VERSION@">0.2.7</token> + <token name="@VERSION@">0.3.1</token> <import>samples_macros.xml</import> </macros> <requirements> @@ -48,7 +48,6 @@ ln -s '$action_options.input_format_conditional.samples_users_table' $samples_table_path && #end if - #if $action_options.test_submit_parameters.dry_run == "false" and $action_options.test_submit == "False": webin_id=`grep 'username' $credentials`; if [ "\$webin_id" = "" ]; then @@ -292,7 +291,58 @@ <assert_contents> <has_n_lines n="5"/> <has_n_columns n="18"/> - <has_line_matching expression="alias\tstatus\taccession\ttitle\tscientific_name\ttaxon_id\tsample_description\tcollection_date\tgeographic_location\thost_common_name\thost_subject_id\thost_health_state\thost_sex\thost_scientific_name\tcollector_name\tcollecting_institution\tisolate\tsubmission_date"/> + <has_line_matching expression="alias\ttitle\tscientific_name\tsample_description\tstatus\taccession\ttaxon_id\tsubmission_date\tgeographic_location\thost_common_name\thost_subject_id\thost_health_state\thost_sex\thost_scientific_name\tcollector_name\tcollecting_institution\tisolate\tcollection_date"/> + </assert_contents> + </output> + <output name="runs_table_out"> + <assert_contents> + <has_n_lines n="5"/> + <has_n_columns n="8"/> + <has_line_matching expression="alias\tstatus\taccession\texperiment_alias\tfile_name\tfile_format\tfile_checksum\tsubmission_date"/> + <has_line_matching expression="r_(.*)_026\tadd\tena_run_accession\te_(.*)_026\tC026_exp5_clean.fastq.gz\tfastq\tfile_checksum\tsubmission_date_ENA"/> + </assert_contents> + </output> + </test> + <!--Test excel input of VIRAL samples with extended columns--> + <test> + <conditional name="action_options"> + <param name="action" value="add"/> + <section name="test_submit_parameters"> + <param name="submit_dev" value="false" /> + <param name="dry_run" value="true" /> + </section> + <conditional name="input_format_conditional"> + <param name="input_format" value="excel_tables"/> + <param name="viral_submission" value="True"/> + <param name="xlsx_file" value="metadata_test_viral_optional_columns.xlsx"/> + <conditional name="run_input_format_conditional"> + <param name="run_input_format" value="multiple_selection_list"/> + <param name="data" value="sample.fq"/> + </conditional> + </conditional> + </conditional> + <param name="center" value="Some research center"/> + <output name="experiments_table_out"> + <assert_contents> + <has_n_lines n="5"/> + <has_n_columns n="17"/> + <has_line_matching expression="alias\tstatus\taccession\ttitle\tstudy_alias\tsample_alias\tdesign_description\tlibrary_name\tlibrary_strategy\tlibrary_source\tlibrary_selection\tlibrary_layout\tinsert_size\tlibrary_construction_protocol\tplatform\tinstrument_model\tsubmission_date" /> + <has_line_matching expression="e_(.*)_026\tadd\taccession_ena\tNanopore sequencing\tSARS-CoV-2_genomes_01\ts_(.*)"/> + </assert_contents> + </output> + <output name="studies_table_out"> + <assert_contents> + <has_n_lines n="2"/> + <has_n_columns n="8"/> + <has_line_matching expression="alias\tstatus\taccession\ttitle\tstudy_type\tstudy_abstract\tpubmed_id\tsubmission_date"/> + <has_line_matching expression="SARS-CoV-2_genomes_01\tadd\tENA_accession\tWhole-genome sequencing of SARS-CoV-2 from Covid-19 patients\tWhole Genome Sequencing\tWhole-genome sequences of SARS-CoV-2 from oro-pharyngeal swabs obtained from Covid-19 patients(.*)"/> + </assert_contents> + </output> + <output name="samples_table_out"> + <assert_contents> + <has_n_lines n="5"/> + <has_n_columns n="42"/> + <has_line_matching expression="alias\ttitle\tscientific_name\tsample_description\tstatus\taccession\ttaxon_id\tsubmission_date\tgeographic_location\thost_common_name\thost_subject_id\thost_health_state\thost_sex\thost_scientific_name\tcollector_name\tcollecting_institution\tisolate\tcollection_date\tgeographic_location_latitude\tgeographic_location_longitude\tsample_capture_status\thost_disease_outcome\thost_age\tvirus_identifier\treceipt_date\tdefinition_for_seropositive_sample\tserotype\thost_habitat\tisolation_source_host_associated\thost_behaviour\tisolation_source_non_host_associated\tsubject_exposure\tsubject_exposure_duration\ttype_exposure\tpersonal_protective_equipment\thospitalisation\tillness_duration\tillness_symptoms\tsample_storage_conditions\tstrain\thost_description\tgravidity"/> </assert_contents> </output> <output name="runs_table_out"> @@ -341,7 +391,7 @@ <assert_contents> <has_n_lines n="5"/> <has_n_columns n="8"/> - <has_line_matching expression="alias\tstatus\taccession\ttitle\tscientific_name\ttaxon_id\tsample_description\tsubmission_date"/> + <has_line_matching expression="alias\ttitle\tscientific_name\tsample_description\tstatus\taccession\ttaxon_id\tsubmission_date"/> </assert_contents> </output> <output name="runs_table_out"> @@ -628,7 +678,7 @@ This is a wrapper for the ENA upload tool in https://github.com/usegalaxy-eu/ena-upload-cli The input metadata can be submitted following the tabular format of the templates in https://github.com/usegalaxy-eu/ena-upload-cli/tree/master/example_tables It is also possible to submit an excel file by following the template in https://drive.google.com/file/d/1ncC22--tW2v-EI-te_r86sAZujIPAjlX/view?usp=sharing - For viral submissions a larger set of metadata is required, you can find the template in https://drive.google.com/file/d/1U4VdcczsIecIXxseV8svE1zO_CBUadog/view?usp=sharing + For viral submissions a larger set of metadata is required, you can find the template in https://drive.google.com/file/d/1Gx78GKh58PmRjdmJ05DBbpObAL-3oUFX/view?usp=sharing ]]></help> <citations> </citations> |
| b |
| diff -r 57251c760cab -r 9e2df763086c mappings.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mappings.py Thu Jul 15 20:12:34 2021 +0000 |
| b |
| @@ -0,0 +1,29 @@ + +optional_samples_cols_mapping = { + "collection date": "collection_date", + "geographic location (latitude)": "geographic_location_latitude", + "geographic location (longitude)": "geographic_location_longitude", + "geographic location (region)": "geographic_location_region", + "sample capture status": "sample_capture_status", + "host disease outcome": "host_disease_outcome", + "host_age": "host_age", + "virus identifier": "virus_identifier", + "receipt date": "receipt_date", + "definition for seropositive sample": "definition_for_seropositive_sample", + "serotype (required for a seropositive sample)": "serotype", + "host habitat": "host_habitat", + "isolation source host-associated": "isolation_source_host_associated", + "host behaviour": "host_behaviour", + "isolation source non-host-associated": "isolation_source_non_host_associated", + "subject exposure": "subject_exposure", + "subject exposure duration": "subject_exposure_duration", + "type exposure": "type_exposure", + "personal protective equipment": "personal_protective_equipment", + "hospitalisation": "hospitalisation", + "illness duration": "illness_duration", + "illness symptoms": "illness_symptoms", + "sample storage conditions": "sample_storage_conditions", + "strain": "strain", + "host description": "host_description", + "gravidity": "gravidity" +} |
| b |
| diff -r 57251c760cab -r 9e2df763086c process_xlsx.py --- a/process_xlsx.py Fri Apr 30 12:09:25 2021 +0000 +++ b/process_xlsx.py Thu Jul 15 20:12:34 2021 +0000 |
| [ |
| b'@@ -4,23 +4,35 @@\n \n import xlrd\n import yaml\n+from mappings import optional_samples_cols_mapping\n \n FILE_FORMAT = \'fastq\'\n \n \n-def extract_data(xl_sheet, expected_columns):\n+def extract_data(xl_sheet, expected_columns, optional_cols=None):\n """\n 1. Check that the columns I expect are present in the sheet\n (any order and mixed with others, it\'s just a verification that\n the user filled the correct template)\n 2. Fill a dictionary with the rows data indexed by first column in list"""\n sheet_columns = {}\n+ if optional_cols is None:\n+ optional_cols = []\n+ optional_cols_loaded = []\n for sh_col in range(xl_sheet.ncols):\n- if xl_sheet.cell(0, sh_col).value in expected_columns:\n+ if (xl_sheet.cell(0, sh_col).value in expected_columns) \\\n+ or (xl_sheet.cell(0, sh_col).value in optional_cols):\n if xl_sheet.cell(0, sh_col).value in sheet_columns.keys():\n- sys.exit("Duplicated columns")\n+ sys.exit("Duplicated columns found")\n else:\n sheet_columns[xl_sheet.cell(0, sh_col).value] = sh_col\n+ if xl_sheet.cell(0, sh_col).value in optional_cols:\n+ # store the list of optional cols available\n+ optional_cols_loaded.append(xl_sheet.cell(0, sh_col).value)\n+ provided_cols = expected_columns + optional_cols_loaded\n+\n+ # check that the required columns are all present\n+ # TODO: revise this for optional columns\n for col in range(len(expected_columns)):\n assert expected_columns[col] in sheet_columns.keys(), \\\n "Expected column %s not found" % expected_columns[col]\n@@ -32,9 +44,9 @@\n # skip first 2 rows: column names + comments rows\n for row_id in range(2, xl_sheet.nrows):\n row_dict = {}\n- for col in range(1, len(expected_columns)):\n- sheet_col_index = sheet_columns[expected_columns[col]]\n- row_dict[expected_columns[col]] = xl_sheet.cell(row_id, sheet_col_index).value\n+ for col in range(1, len(provided_cols)):\n+ sheet_col_index = sheet_columns[provided_cols[col]]\n+ row_dict[provided_cols[col]] = xl_sheet.cell(row_id, sheet_col_index).value\n # should check for duplicate alias/ids?\n if xl_sheet.cell(row_id, index_col).value in data_dict.keys():\n tmp = data_dict[xl_sheet.cell(row_id, index_col).value]\n@@ -42,7 +54,7 @@\n data_dict[xl_sheet.cell(row_id, index_col).value].append(row_dict)\n else:\n data_dict[xl_sheet.cell(row_id, index_col).value] = row_dict\n- return data_dict\n+ return data_dict, optional_cols_loaded\n \n \n def paste_xls2yaml(xlsx_path):\n@@ -86,22 +98,25 @@\n raise ValueError(\'No entries found in studies sheet\')\n studies_dict = {}\n studies_col = [\'alias\', \'title\', \'study_type\', \'study_abstract\']\n-studies_dict = extract_data(xl_sheet, studies_col)\n+studies_dict, _ = extract_data(xl_sheet, studies_col)\n \n # PARSE SAMPLES\n #################\n xl_sheet = xl_workbook.sheet_by_name(\'ENA_sample\')\n if xl_sheet.nrows < 3:\n raise ValueError(\'No entries found in samples\')\n+\n+samples_cols_excel = [\'alias\', \'title\', \'scientific_name\', \'sample_description\']\n+# optional_samples_cols_mapping = {}\n if args.viral_submission:\n- samples_cols = [\'alias\', \'title\', \'scientific_name\', \'sample_description\',\n- \'geographic location (country and/or sea)\', \'host common name\',\n- \'host health state\', \'host sex\', \'host scientific name\', \'collector name\',\n- \'collection date\', \'collecting institution\', \'isolate\']\n-else:\n- samples_cols = [\'alias\', \'title\', \'scientific_name\', \'sample_description\']\n-samples_dict = extract_data(xl_sheet, samples_cols)\n+ # load columns names from the table\n+ samples_cols_excel = samples_cols_excel + [\'geographic location (country and/or sea)\',\n+ \'host common name\', \'host health '..b'e\', \'study_alias\',\n@@ -164,22 +183,44 @@\n \'ENA_submission_data\']) + \'\\n\') # assuming no pubmed_id\n for sample_alias, sample in samples_dict.items():\n # sample_alias = sample_alias + \'_\' + timestamp\n+ samples_row_values = [sample_alias, sample[\'title\'], sample[\'scientific_name\'],\n+ sample[\'sample_description\'], action, \'ena_accession\',\n+ \'tax_id_updated_by_ENA\', \'ENA_submission_date\']\n if args.viral_submission:\n+ # add the values that are unique for the viral samples\n if sample[\'collector name\'] == \'\':\n sample[\'collector name\'] = \'unknown\'\n- samples_table.write(\'\\t\'.join([sample_alias, action, \'ena_accession\', sample[\'title\'],\n- sample[\'scientific_name\'], \'tax_id_updated_by_ENA\',\n- sample[\'sample_description\'], sample[\'collection date\'],\n- sample[\'geographic location (country and/or sea)\'],\n- sample[\'host common name\'], \'host subject id\',\n- sample[\'host health state\'], sample[\'host sex\'],\n- sample[\'host scientific name\'], sample[\'collector name\'],\n- sample[\'collecting institution\'], sample[\'isolate\'],\n- \'ENA_submission_date\']) + \'\\n\')\n- else:\n- samples_table.write(\'\\t\'.join([sample_alias, action, \'ena_accession\', sample[\'title\'],\n- sample[\'scientific_name\'], \'tax_id_updated_by_ENA\',\n- sample[\'sample_description\']]) + \'\\n\')\n+ samples_row_values = samples_row_values + \\\n+ [sample[\'geographic location (country and/or sea)\'], sample[\'host common name\'],\n+ \'host subject id\', sample[\'host health state\'], sample[\'host sex\'],\n+ sample[\'host scientific name\'], sample[\'collector name\'],\n+ sample[\'collecting institution\'], sample[\'isolate\']]\n+ # add the (possible) optional columns values\n+ if len(samples_optional_cols_loaded) > 0:\n+ for optional_col in samples_optional_cols_loaded:\n+ # parse values stored as in excel date format (=float)\n+ if optional_col in (\'collection date\', \'receipt date\'):\n+ # check if excel stored it as date\n+ if isinstance(sample[optional_col], float):\n+ year, month, day, hour, minute, second = xlrd.xldate_as_tuple(\n+ sample[optional_col], xl_workbook.datemode)\n+ month = "{:02d}".format(month)\n+ day = "{:02d}".format(day)\n+ hour = "{:02d}".format(hour)\n+ minute = "{:02d}".format(minute)\n+ second = "{:02d}".format(second)\n+ # format it as 2008-01-23T19:23:10\n+ sample[optional_col] = str(year) + \'-\' + str(month) + \'-\' + str(day) + \\\n+ \'T\' + str(hour) + \':\' + str(minute) + \':\' + str(second)\n+ # excel stores everything as float so I need to check if\n+ # the value was actually an int and keep it as int\n+ if isinstance(sample[optional_col], float):\n+ if int(sample[optional_col]) == sample[optional_col]:\n+ # it is not really a float but an int\n+ sample[optional_col] = int(sample[optional_col])\n+ samples_row_values.append(str(sample[optional_col]))\n+ samples_table.write(\'\\t\'.join(samples_row_values) + \'\\n\')\n+\n for exp_alias, exp in experiments_dict.items():\n # should I check here if any experiment has a study or sample alias that is incorrect?\n # (not listed in the samples or study dict)\n' |
| b |
| diff -r 57251c760cab -r 9e2df763086c samples_macros.xml --- a/samples_macros.xml Fri Apr 30 12:09:25 2021 +0000 +++ b/samples_macros.xml Thu Jul 15 20:12:34 2021 +0000 |
| b |
| @@ -138,7 +138,7 @@ </options> </param> <repeat name="rep_runs" title="Runs executed within this experiment" min="1" > - <param name="run_base_name" type="text" optional="False" default="" label="Run alias" help="If an alias is not provided it will be generated combining the sample and experiment indexes"/> + <param name="run_base_name" type="text" optional="False" label="Run alias" help="If an alias is not provided it will be generated combining the sample and experiment indexes"/> <param name="upload_files" type="data" format="fastqsanger.gz,fastqsanger.bz2,fastq.gz,fastq.bz2" multiple="true" label="File(s) associated with this run"/> </repeat> </repeat> @@ -203,7 +203,7 @@ </options> </param> <repeat name="rep_runs" title="Runs executed within this experiment" min="1" > - <param name="run_base_name" type="text" optional="False" default="" label="Run alias" help="If an alias is not provided it will be generated combining the sample and experiment indexes"/> + <param name="run_base_name" type="text" optional="False" label="Run alias" help="If an alias is not provided it will be generated combining the sample and experiment indexes"/> <param name="upload_files" type="data" format="fastqsanger.gz,fastqsanger.bz2,fastq.gz,fastq.bz2" multiple="true" label="File(s) associated with this run"/> </repeat> </repeat> |
| b |
| diff -r 57251c760cab -r 9e2df763086c test-data/metadata_test_viral_optional_columns.xlsx |
| b |
| Binary file test-data/metadata_test_viral_optional_columns.xlsx has changed |