comparison ena_upload.xml @ 10:480d9e9d156b draft

planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/ena_upload commit dccd4c0668ec8ce1b1c8fd9cef4dcf3a99dc78b3

9:a62c4a11a67d

<tool id="ena_upload" name="ENA Upload tool" version="@VERSION@" profile="20.01" license="MIT">
    <macros>
        <token name="@VERSION@">0.6.1</token>
        <import>samples_macros.xml</import>
    </macros>
    <requirements>
        <requirement type="package" version="@VERSION@">ena-upload-cli</requirement>
    </requirements>
    <stdio>
        <regex match="Oops" source="stderr" level="fatal"/>
        <regex match="different file names between command line and RUN table" source="stderr" level="fatal"/>
    </stdio>

    ln -s '$action_options.input_format_conditional.samples_users_table' $samples_table_path &&
#end if
#if $action_options.input_format_conditional.input_format == "excel_tables":
    ln -s '$action_options.input_format_conditional.xlsx_file' ./xlsx_input.xlsx &&
#end if
#if $action_options.test_submit_parameters.dry_run == "false" and $action_options.test_submit == "False":
    webin_id=`grep 'username' $credentials`;
    if [ "\$webin_id" = "" ]; then
      ## No credentials in user defined preferences    
      ## Fallback to global defined credentials (if exist)   
      #import os
      #if os.path.isfile(os.environ.get('GALAXY_ENA_SECRETS', '')):
          credentials_path=\${GALAXY_ENA_SECRETS};     
          webin_id=`grep 'username' \$GALAXY_ENA_SECRETS`;
          if [ "\$webin_id" = "" ]; then
              echo "No global credentials defined. Check your GALAXY_ENA_SECRETS file or set your credentials via: User -> Preferences -> Manage Information";
              exit 1;
          fi;
      #else:
          echo "No ENA credentials defined. Set your credentials via: User -> Preferences -> Manage Information";
          exit 1;

      #for $sample in $study.rep_sample:
        #for $experiment in $sample.rep_experiment:
          #for $run in $experiment.rep_runs:
            #for $file in $run.upload_files:
                #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
                #if $action_options.input_format_conditional.add_extension == "true":
                    #set $extension = '.fastq'
                #else
                    #set $extension = ''
                #end if
                #if $file.is_of_type('fastq', 'fastqsanger'):
                    ## compression output is defined as safename_reads_file so no need to symlink
                    #set $safename_reads_file = $safename_reads_file + $extension + '.gz'
                    gzip -c '$file' > $safename_reads_file &&
                #else:
                    #if $action_options.input_format_conditional.add_extension == "true":
                        #if $file.is_of_type('fastq.gz', 'fastqsanger.gz'):
                            #set $compression = '.gz' 
                        #elif $file.is_of_type('fastqsanger.bz2', 'fastq.bz2'):
                            #set $compression = '.bz2' 
                        #end if

    #if $action_options.input_format_conditional.run_input_format_conditional.run_input_format == 'multiple_selection_list':
        #for $file in $action_options.input_format_conditional.run_input_format_conditional.data:
            #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
            #if $file.is_of_type('fastq', 'fastqsanger'):
                ## always compress add the gz extension
                #if $action_options.input_format_conditional.run_input_format_conditional.add_extension == "true":
                    #set $safename_reads_file = $safename_reads_file + '.fastq.gz'
                #else
                    #set $safename_reads_file = $safename_reads_file + '.gz'
                #end if   
                gzip -c '$file' > $safename_reads_file &&
            #else
                #if $action_options.input_format_conditional.run_input_format_conditional.add_extension == "true":
                    #if $file.is_of_type('fastq.gz', 'fastqsanger.gz'):
                        #set $extension = '.fastq.gz'
                    #elif $file.is_of_type('fastqsanger.bz2', 'fastq.bz2'):
                        #set $extension = '.fastq.bz2'
                    #end if

    #end if
#end if

#if $action_options.action == "add":
ena-upload-cli
    --tool 'ena-upload-cli v@VERSION@ @ Galaxy'
    --action '$action_options.action'
    --center '$action_options.center'
    --secret \${credentials_path}
    --data
    #for $dataset in $files_to_upload:
        '$dataset'
    #end for
#if $action_options.test_submit_parameters.dry_run == "true":
    --draft
#end if
#if $action_options.input_format_conditional.input_format != "excel_tables":
    --experiment '$experiments_table_path'
    --study '$studies_table_path'

--action add
#if $action_options.input_format_conditional.input_format == "user_generated_tables":
        --checklist $action_options.input_format_conditional.checklist_id
#else:
    #if $action_options.input_format_conditional.input_format == "build_tables":
        #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "true":
          --checklist ERC000033
        #end if
    #else:
          --checklist $action_options.input_format_conditional.checklist_id
    #end if
#end if
#if $action_options.idempotent == "true":
    --auto_action
#end if
#if $action_options.test_submit_parameters.submit_dev == "true":
    -d
#end if
    >> '$output';
#end if



#if $action_options.action == "modify":
    ena-upload-cli
    --tool 'ena-upload-cli v@VERSION@ @ Galaxy'
    --action '$action_options.action'
    --center '$action_options.center'
    --secret \${credentials_path}
    --data
    #for $dataset in $files_to_upload:
        '$dataset'
    #end for
#if $action_options.test_submit_parameters.dry_run == "true":
    --draft
#end if
#if $action_options.input_format_conditional.input_format != "excel_tables":
    --experiment '$experiments_table_path'
    --study '$studies_table_path'

--action 'modify'
#if $action_options.input_format_conditional.input_format == "user_generated_tables":
        --checklist $action_options.input_format_conditional.checklist_id
#else:
    #if $action_options.input_format_conditional.input_format == "build_tables":
        #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "true":
          --checklist ERC000033
        #end if
    #else:
          --checklist $action_options.input_format_conditional.checklist_id
    #end if
#end if
    >> '$output';
#end if

#if $action_options.test_submit_parameters.dry_run == "false":
    echo -e 'center_name\t$action_options.center' >> '$output';
    echo -e 'action_option\t$action_options.action' >> '$output';
#end if
        
## copy updated files

            $runs.append((str($run.run_base_name),$run_files))
        #end for
    
$experiments.append({'title':str($experiment.experiment_title),'experiment_design':str($experiment.experiment_design),'library_strategy':str($experiment.library_strategy),'library_source':str($experiment.library_source),'library_selection':str($experiment.library_selection),'library_layout':str($experiment.library_layout),'insert_size':str($experiment.insert_size),'library_construction_protocol':str($experiment.library_construction_protocol),'platform':str($experiment.platform),'instrument_model':str($experiment.instrument_model),'runs':$runs})
      #end for
      #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "true":

$samples.append({'title':str($sample.sample_title),'description':str($sample.sample_description),'tax_name':str($sample.scientific_name),'tax_id':str($sample.tax_id),'collection_date':str($sample.collection_date),'geo_location':str($sample.geo_location_country),'host_common_name':str($sample.host_common_name),'host_subject_id':str($sample.host_subject_id),'host_health_state':str($sample.host_health_state),'host_sex':str($sample.host_sex),'host_scientific_name':str($sample.host_scientific_name),'collector_name':str($sample.collector_name),'collecting_institution':str($sample.collecting_institution),'isolate':str($sample.isolate),'experiments':$experiments})
      #else:

$samples.append({'title':str($sample.sample_title),'description':str($sample.sample_description),'tax_name':str($sample.scientific_name),'tax_id':str($sample.tax_id),'experiments':$experiments})
      #end if
    #end for
    $studies.append({'title':str($study.study_title),'type':str($study.study_type),'abstract':str($study.study_abstract),'pubmed_id':str($study.study_pubmed_id),'samples':$samples})
  #end for
  #echo $json.dumps($studies)

            <param name="action" type="select" label="Action to execute">
                <option value="add" selected="True">Add new data</option>
                <option value="modify">Modify metadata</option>
            </param>
            <when value="add">
                <param type="boolean" name="idempotent" checked="False" label="Idempotent submissions: only add the elements that were not submitted before" help="NOTE: this feature is based on a beta parameter of the CLI that checks the remote ENA repository for entries using the alias. This remote detection can have false positives, i.e assumes that it's present in the repository but it's not"/>
                <expand macro="test_submit_section"/>    
                <param name="test_submit" type="hidden" value="False" />
                <expand macro="table_inputs_macro" />
            </when>
            <when value="modify">

            <filter> action_options['input_format_conditional']['input_format'] == "build_tables" or action_options['input_format_conditional']['input_format'] == "excel_tables"</filter>
        </data>
    </outputs>
    <tests>
       <!--Test 1:  excel input of VIRAL samples -->
        <test>
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="true" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000033"/>
                    <param name="xlsx_file" value="metadata_test_viral.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="add_extension" value="false"/>
                        <param name="run_input_format" value="multiple_selection_list"/>
                        <param name="data" value="C030_exp5_clean.fastq,C053_exp5_clean.fastq,C026_exp5_clean.fastq,C067_exp5_clean.fastq"/>
                    </conditional>
                </conditional>
            </conditional>

                    <has_line_matching expression="r_20201007_026\te_20201007_026\tC026_exp5_clean.fastq.gz\tFASTQ(.*)"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 2: excel input of VIRAL samples with extended columns-->
        <test>
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="true" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000033"/>
                    <param name="xlsx_file" value="metadata_test_viral_optional_columns.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="add_extension" value="false"/>
                        <param name="run_input_format" value="multiple_selection_list"/>
                        <param name="data" value="C030_exp5_clean.fastq,C053_exp5_clean.fastq,C026_exp5_clean.fastq,C067_exp5_clean.fastq"/>
                    </conditional>
                </conditional>
            </conditional>

                    <has_line_matching expression="ENA_run:"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 3: excel input of NON-VIRAL samples-->
        <test>
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="true" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000011"/>
                    <param name="xlsx_file" value="metadata_test_nonviral_1_run.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="add_extension" value="true"/>
                        <param name="run_input_format" value="multiple_selection_list"/>
                        <param name="data" value="sample_no_extension"/>
                    </conditional>
                </conditional>
            </conditional>

                </assert_contents>
            </output>
            <output name="samples_table_out">
                <assert_contents>
                    <has_n_lines n="2"/>
                    <has_n_columns n="8"/>
                    <has_line_matching expression="alias\ttitle\tscientific_name\tsample_description\taccession\tsubmission_date\tstatus\ttaxon_id"/>
                </assert_contents>
            </output>
            <output name="runs_table_out">
                <assert_contents>
                    <has_n_lines n="2"/>

                    <has_line_matching expression="r_20201007_026\te_20201007_026\tsample_no_extension.fastq.gz\tFASTQ(.*)"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 4: failure on excel input of NON-VIRAL samples with runs PAIRED collection -->
        <test expect_failure="true">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="false" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="true"/>
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000011"/>
                    <param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="run_input_format" value="paired_list"/>

                <has_text_matching expression="--data 'paired_run_name_1.fastq.gz' 'paired_run_name_2.fastq.gz'"/>
                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
            </assert_command>
        </test>
        <!--Test 4b: failure on excel input of NON-VIRAL samples with runs PAIRED collection + Idempotent submission -->
        <test expect_failure="true">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="false" />
                </section>
                <param name="idempotent" value="true"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="true"/>
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000011"/>
                    <param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="run_input_format" value="paired_list"/>

                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
                <has_text_matching expression="--auto_action"/>
            </assert_command>
        </test>
        <!--Test 5: build tables from user input fields NON-VIRAL samples-->
        <test>
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="true" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="build_tables"/>
                    <param name="add_extension" value="false"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="False"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>

                            <repeat name="rep_sample">
                                <param name="sample_title" value="Test Sample title"/>
                                <param name="sample_description" value="Test Sample description"/>
                                <param name="scientific_name" value="Test Sample scientific name"/>
                                <param name="tax_id" value="2697049"/>
                                <repeat name="rep_experiment">
                                    <param name="experiment_title" value="Test experiment title"/>
                                    <param name="experiment_design" value="Test experiment design description"/>
                                    <param name="library_strategy" value="CTS"/>
                                    <param name="library_source" value="GENOMIC"/>

                </assert_contents>
            </output>
            <output name="samples_table_out">
                <assert_contents>
                    <has_n_lines n="2"/>
                    <has_n_columns n="8"/>
                </assert_contents>
            </output>
            <output name="runs_table_out">
                <assert_contents>
                    <has_n_lines n="3"/>
                    <has_n_columns n="8"/>
                    <has_line_matching expression="alias\tstatus\texperiment_alias\tfile_name\tfile_format\taccession\tsubmission_date\tfile_checksum"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 6: RUN failing build tables from user input fields NON-VIRAL samples-->
        <test expect_failure="true">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="true" />
                    <param name="dry_run" value="false" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="build_tables"/>
                    <param name="add_extension" value="true"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="false"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>
                            <param name="study_type" value="Epigenetics"/>
                            <param name="study_pubmed_id" value="Test study pubmedID"/>
                            <repeat name="rep_sample">
                                <param name="sample_title" value="Test Sample title"/>
                                <param name="sample_description" value="Test Sample description"/>
                                <param name="scientific_name" value="Test Sample scientific name"/>
                                <param name="tax_id" value="2697049"/>
                                <repeat name="rep_experiment">
                                    <param name="experiment_title" value="Test experiment title"/>
                                    <param name="experiment_design" value="Test experiment design description"/>
                                    <param name="library_strategy" value="CTS"/>
                                    <param name="library_source" value="GENOMIC"/>
                                    <param name="library_selection" value="PCR"/>
                                    <param name="library_layout" value="SINGLE"/>
                                    <param name="insert_size" value="150"/>
                                    <param name="library_construction_protocol" value="Test library construction"/>
                                    <param name="platform" value="ILLUMINA"/>
                                    <param name="instrument_model" value="Illumina HiSeq 4000"/>
                                    <repeat name="rep_runs">
                                        <param name="upload_files" value="1.fastqsanger.gz,sample.fq" ftype="fastqsanger"/>
                                    </repeat>
                                </repeat>
                            </repeat>
                        </repeat>
                    </conditional>
                </conditional>
            </conditional>
            <param name="center" value="Some research center"/>
            <assert_stdout>
                <has_text_matching expression="No ENA credentials defined"/>
            </assert_stdout>
        </test>
        <!--Test 7: with submit_test to skip credentials checksRUN failing build tables from user input fields NON-VIRAL samples
            also tests compression of uncompressed inputs and adding the .gz suffix -->
        <test expect_failure="false">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="true" />
                    <param name="dry_run" value="true" />
                </section>
                <param name="test_submit" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="false"/>
                    <param name="input_format" value="build_tables"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="false"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>
                            <param name="study_type" value="Epigenetics"/>
                            <param name="study_pubmed_id" value="Test study pubmedID"/>
                            <repeat name="rep_sample">
                                <param name="sample_title" value="Test Sample title"/>
                                <param name="sample_description" value="Test Sample description"/>
                                <param name="scientific_name" value="Test Sample scientific name"/>
                                <param name="tax_id" value="2697049"/>
                                <repeat name="rep_experiment">
                                    <param name="experiment_title" value="Test experiment title"/>
                                    <param name="experiment_design" value="Test experiment design description"/>
                                    <param name="library_strategy" value="CTS"/>
                                    <param name="library_source" value="GENOMIC"/>

                <has_text_matching expression="--data '1.fastqsanger.gz' '2.fastqsanger.gz'"/>
                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
                <not_has_text text="modify" />
            </assert_command>
        </test>
        <!--Test 8: viral submission - User input metadata - Add extension = False-->
        <test expect_failure="false">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="true" />
                </section>
                <param name="test_submit" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="false"/>
                    <param name="input_format" value="build_tables"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="true"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>
                            <param name="study_type" value="Epigenetics"/>
                            <param name="study_pubmed_id" value="Test study pubmedID"/>

                <has_text_matching expression="--data '1.fastqsanger.gz' '2.fastqsanger.gz'"/>
                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
                <has_text_matching expression="--checklist ERC000033"/>
            </assert_command>
        </test>
        <!--Test 9: modify option and auto compression - viral submission - User input metadata-->
        <test expect_failure="false">
            <conditional name="action_options">
                <param name="action" value="modify"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="false" />
                    <param name="dry_run" value="true" />
                </section>
                <param name="test_submit" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="fasle"/>
                    <param name="input_format" value="build_tables"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="True"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>

                                    <param name="library_construction_protocol" value="Test library construction"/>
                                    <param name="platform" value="ILLUMINA"/>
                                    <param name="instrument_model" value="Illumina HiSeq 4000"/>
                                    <repeat name="rep_runs">
                                        <param name="run_base_name" value="run_from_hospital_X"/>
                                        <param name="upload_files" value="1.fastqsanger.gz" ftype="fastqsanger.gz"/>
                                    </repeat>
                                </repeat>
                            </repeat>
                        </repeat>
                    </conditional>
                </conditional>
            </conditional>
            <param name="center" value="Some research center"/>
            <assert_command>
                <has_text_matching expression="ena-upload-cli"/>
                <has_text_matching expression="--data '1.fastqsanger.gz'"/>
                <has_text_matching expression="--action 'modify' --center 'Some research center'"/>
                <has_text_matching expression="--checklist ERC000033"/>
                <not_has_text text="add" />
            </assert_command>
        </test>
    </tests>
    <help><![CDATA[
        This is a wrapper for the ENA upload tool in https://github.com/usegalaxy-eu/ena-upload-cli
        The input metadata can be submitted following the tabular format of the templates in https://github.com/usegalaxy-eu/ena-upload-cli/tree/master/example_tables
        It is also possible to submit an excel file by following the template in https://github.com/ELIXIR-Belgium/ENA-metadata-templates
    ]]></help>
    <citations>
    </citations>
</tool>

10:480d9e9d156b

<tool id="ena_upload" name="ENA Upload tool" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="21.05" license="MIT">
    <macros>
        <token name="@TOOL_VERSION@">0.6.3</token>
        <token name="@VERSION_SUFFIX@">0</token>
        <import>samples_macros.xml</import>
    </macros>
    <requirements>
        <requirement type="package" version="@TOOL_VERSION@">ena-upload-cli</requirement>
    </requirements>
    <stdio>
        <regex match="Oops" source="stderr" level="fatal"/>
        <regex match="different file names between command line and RUN table" source="stderr" level="fatal"/>
    </stdio>

    ln -s '$action_options.input_format_conditional.samples_users_table' $samples_table_path &&
#end if
#if $action_options.input_format_conditional.input_format == "excel_tables":
    ln -s '$action_options.input_format_conditional.xlsx_file' ./xlsx_input.xlsx &&
#end if
#if $action_options.test_submit_parameters.dry_run == "False" and $action_options.test_submit == "False":
    webin_id=`grep 'username' $credentials`;
    if [ "\$webin_id" = "" ]; then
      ## No credentials in user defined preferences    
      ## Fallback to global defined credentials (if exist)   
      #import os
      #if os.path.isfile(os.environ.get('GALAXY_ENA_SECRETS', '')):
          credentials_path=\${GALAXY_ENA_SECRETS};     
          webin_id=`grep 'username' \$GALAXY_ENA_SECRETS`;
          if [ "\$webin_id" = "" ]; then
                echo "No global credentials defined. Check your GALAXY_ENA_SECRETS file or set your credentials via: User -> Preferences -> Manage Information";
              exit 1;
          fi;
      #else:
          echo "No ENA credentials defined. Set your credentials via: User -> Preferences -> Manage Information";
          exit 1;

      #for $sample in $study.rep_sample:
        #for $experiment in $sample.rep_experiment:
          #for $run in $experiment.rep_runs:
            #for $file in $run.upload_files:
                #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
                #if $action_options.input_format_conditional.add_extension == "True":
                    #set $extension = '.fastq'
                #else
                    #set $extension = ''
                #end if
                #if $file.is_of_type('fastq', 'fastqsanger'):
                    ## compression output is defined as safename_reads_file so no need to symlink
                    #set $safename_reads_file = $safename_reads_file + $extension + '.gz'
                    gzip -c '$file' > $safename_reads_file &&
                #else:
                    #if $action_options.input_format_conditional.add_extension == "True":
                        #if $file.is_of_type('fastq.gz', 'fastqsanger.gz'):
                            #set $compression = '.gz' 
                        #elif $file.is_of_type('fastqsanger.bz2', 'fastq.bz2'):
                            #set $compression = '.bz2' 
                        #end if

    #if $action_options.input_format_conditional.run_input_format_conditional.run_input_format == 'multiple_selection_list':
        #for $file in $action_options.input_format_conditional.run_input_format_conditional.data:
            #set $safename_reads_file = re.sub('[^\w\-_\.]', '_', $file.element_identifier)
            #if $file.is_of_type('fastq', 'fastqsanger'):
                ## always compress add the gz extension
                #if $action_options.input_format_conditional.run_input_format_conditional.add_extension == "True":
                    #set $safename_reads_file = $safename_reads_file + '.fastq.gz'
                #else
                    #set $safename_reads_file = $safename_reads_file + '.gz'
                #end if
                gzip -c '$file' > $safename_reads_file &&
            #else
                #if $action_options.input_format_conditional.run_input_format_conditional.add_extension == "True":
                    #if $file.is_of_type('fastq.gz', 'fastqsanger.gz'):
                        #set $extension = '.fastq.gz'
                    #elif $file.is_of_type('fastqsanger.bz2', 'fastq.bz2'):
                        #set $extension = '.fastq.bz2'
                    #end if

    #end if
#end if

#if $action_options.action == "add":
ena-upload-cli
    --tool 'ena-upload-cli v@TOOL_VERSION@ @ Galaxy'
    --action '$action_options.action'
    --center '$action_options.center'
    --secret \${credentials_path}
    --data
    #for $dataset in $files_to_upload:
        '$dataset'
    #end for
#if $action_options.test_submit_parameters.dry_run == "True":
    --draft
#end if
#if $action_options.input_format_conditional.input_format != "excel_tables":
    --experiment '$experiments_table_path'
    --study '$studies_table_path'

--action add
#if $action_options.input_format_conditional.input_format == "user_generated_tables":
        --checklist $action_options.input_format_conditional.checklist_id
#else:
    #if $action_options.input_format_conditional.input_format == "build_tables":
        #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "True":
          --checklist ERC000033
        #end if
    #else:
          --checklist $action_options.input_format_conditional.checklist_id
    #end if
#end if
#if $action_options.idempotent == "True":
    --auto_action
#end if
#if $action_options.test_submit_parameters.submit_dev == "True":
    -d
#end if
    >> '$output';
#end if



#if $action_options.action == "modify":
    ena-upload-cli
    --tool 'ena-upload-cli v@TOOL_VERSION@ @ Galaxy'
    --action '$action_options.action'
    --center '$action_options.center'
    --secret \${credentials_path}
    --data
    #for $dataset in $files_to_upload:
        '$dataset'
    #end for
#if $action_options.test_submit_parameters.dry_run == "True":
    --draft
#end if
#if $action_options.input_format_conditional.input_format != "excel_tables":
    --experiment '$experiments_table_path'
    --study '$studies_table_path'

--action 'modify'
#if $action_options.input_format_conditional.input_format == "user_generated_tables":
        --checklist $action_options.input_format_conditional.checklist_id
#else:
    #if $action_options.input_format_conditional.input_format == "build_tables":
        #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "True":
          --checklist ERC000033
        #end if
    #else:
          --checklist $action_options.input_format_conditional.checklist_id
    #end if
#end if
    >> '$output';
#end if

#if $action_options.test_submit_parameters.dry_run == "False":
    echo -e 'center_name\t$action_options.center' >> '$output';
    echo -e 'action_option\t$action_options.action' >> '$output';
#end if
        
## copy updated files

            $runs.append((str($run.run_base_name),$run_files))
        #end for
    
$experiments.append({'title':str($experiment.experiment_title),'experiment_design':str($experiment.experiment_design),'library_strategy':str($experiment.library_strategy),'library_source':str($experiment.library_source),'library_selection':str($experiment.library_selection),'library_layout':str($experiment.library_layout),'insert_size':str($experiment.insert_size),'library_construction_protocol':str($experiment.library_construction_protocol),'platform':str($experiment.platform),'instrument_model':str($experiment.instrument_model),'runs':$runs})
      #end for
      #if $action_options.input_format_conditional.conditional_viral_metadata.viral_sample == "True":
$samples.append({'title':str($sample.sample_title),'description':str($sample.sample_description),'tax_name':str($sample.scientific_name),'tax_id':str($sample.tax_id),'collection_date':str($sample.collection_date),'geo_location':str($sample.geo_location_country),'host_common_name':str($sample.host_common_name),'host_subject_id':str($sample.host_subject_id),'host_health_state':str($sample.host_health_state),'host_sex':str($sample.host_sex),'host_scientific_name':str($sample.host_scientific_name),'collector_name':str($sample.collector_name),'collecting_institution':str($sample.collecting_institution),'isolate':str($sample.isolate),'experiments':$experiments})
      #else:
$samples.append({'title':str($sample.sample_title),'description':str($sample.sample_description),'tax_name':str($sample.scientific_name),'tax_id':str($sample.tax_id),'collection_date':str($sample.collection_date),'geo_location':str($sample.geo_location_country),'experiments':$experiments})
      #end if
    #end for
    $studies.append({'title':str($study.study_title),'type':str($study.study_type),'abstract':str($study.study_abstract),'pubmed_id':str($study.study_pubmed_id),'samples':$samples})
  #end for
  #echo $json.dumps($studies)

            <param name="action" type="select" label="Action to execute">
                <option value="add" selected="True">Add new data</option>
                <option value="modify">Modify metadata</option>
            </param>
            <when value="add">
                <param type="boolean" name="idempotent" checked="False" label="Idempotent submissions: only add the elements that were not submitted before" help="NOTE: this feature is based on a beta parameter of the CLI that checks the remote ENA repository for entries using the alias. This remote detection can have False positives, i.e assumes that it's present in the repository but it's not"/>
                <expand macro="test_submit_section"/>    
                <param name="test_submit" type="hidden" value="False" />
                <expand macro="table_inputs_macro" />
            </when>
            <when value="modify">

            <filter> action_options['input_format_conditional']['input_format'] == "build_tables" or action_options['input_format_conditional']['input_format'] == "excel_tables"</filter>
        </data>
    </outputs>
    <tests>
       <!--Test 1:  excel input of VIRAL samples -->
        <test expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="True" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000033"/>
                    <param name="xlsx_file" value="metadata_test_viral.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="add_extension" value="False"/>
                        <param name="run_input_format" value="multiple_selection_list"/>
                        <param name="data" value="C030_exp5_clean.fastq,C053_exp5_clean.fastq,C026_exp5_clean.fastq,C067_exp5_clean.fastq"/>
                    </conditional>
                </conditional>
            </conditional>

                    <has_line_matching expression="r_20201007_026\te_20201007_026\tC026_exp5_clean.fastq.gz\tFASTQ(.*)"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 2: excel input of VIRAL samples with extended columns-->
        <test expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="True" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000033"/>
                    <param name="xlsx_file" value="metadata_test_viral_optional_columns.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="add_extension" value="False"/>
                        <param name="run_input_format" value="multiple_selection_list"/>
                        <param name="data" value="C030_exp5_clean.fastq,C053_exp5_clean.fastq,C026_exp5_clean.fastq,C067_exp5_clean.fastq"/>
                    </conditional>
                </conditional>
            </conditional>

                    <has_line_matching expression="ENA_run:"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 3: excel input of NON-VIRAL samples-->
        <test expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="True" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000011"/>
                    <param name="xlsx_file" value="metadata_test_nonviral_1_run.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="add_extension" value="True"/>
                        <param name="run_input_format" value="multiple_selection_list"/>
                        <param name="data" value="sample_no_extension"/>
                    </conditional>
                </conditional>
            </conditional>

                </assert_contents>
            </output>
            <output name="samples_table_out">
                <assert_contents>
                    <has_n_lines n="2"/>
                    <has_n_columns n="10"/>
                    <has_line_matching expression="s_20201007_026\tC026\tSaccharomyces cerevisiae\tSample from beer\t2020-03-25\tAntarctica\t\t\tADDED\t"/>
                </assert_contents>
            </output>
            <output name="runs_table_out">
                <assert_contents>
                    <has_n_lines n="2"/>

                    <has_line_matching expression="r_20201007_026\te_20201007_026\tsample_no_extension.fastq.gz\tFASTQ(.*)"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 4: failure on excel input of NON-VIRAL samples with runs PAIRED collection -->
        <test expect_failure="True">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="False" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="True"/>
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000011"/>
                    <param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="run_input_format" value="paired_list"/>

                <has_text_matching expression="--data 'paired_run_name_1.fastq.gz' 'paired_run_name_2.fastq.gz'"/>
                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
            </assert_command>
        </test>
        <!--Test 4b: failure on excel input of NON-VIRAL samples with runs PAIRED collection + Idempotent submission -->
        <test expect_failure="True">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="False" />
                </section>
                <param name="idempotent" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="True"/>
                    <param name="input_format" value="excel_tables"/>
                    <param name="checklist_id" value="ERC000011"/>
                    <param name="xlsx_file" value="metadata_test_nonviral.xlsx"/>
                    <conditional name="run_input_format_conditional">
                        <param name="run_input_format" value="paired_list"/>

                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
                <has_text_matching expression="--auto_action"/>
            </assert_command>
        </test>
        <!--Test 5: build tables from user input fields NON-VIRAL samples-->
        <test expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="True" />
                </section>
                <conditional name="input_format_conditional">
                    <param name="input_format" value="build_tables"/>
                    <param name="add_extension" value="False"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="False"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>

                            <repeat name="rep_sample">
                                <param name="sample_title" value="Test Sample title"/>
                                <param name="sample_description" value="Test Sample description"/>
                                <param name="scientific_name" value="Test Sample scientific name"/>
                                <param name="tax_id" value="2697049"/>
                                <param name="collection_date" value="2020"/>
                                <param name="geo_location_country" value="Belgium"/>
                                <repeat name="rep_experiment">
                                    <param name="experiment_title" value="Test experiment title"/>
                                    <param name="experiment_design" value="Test experiment design description"/>
                                    <param name="library_strategy" value="CTS"/>
                                    <param name="library_source" value="GENOMIC"/>

                </assert_contents>
            </output>
            <output name="samples_table_out">
                <assert_contents>
                    <has_n_lines n="2"/>
                    <has_n_columns n="10"/>
                </assert_contents>
            </output>
            <output name="runs_table_out">
                <assert_contents>
                    <has_n_lines n="3"/>
                    <has_n_columns n="8"/>
                    <has_line_matching expression="alias\tstatus\texperiment_alias\tfile_name\tfile_format\taccession\tsubmission_date\tfile_checksum"/>
                </assert_contents>
            </output>
        </test>
        <!--Test 6: with submit_test to skip credentials checksRUN failing build tables from user input fields NON-VIRAL samples
            also tests compression of uncompressed inputs and adding the .gz suffix -->
        <test expect_failure="False" expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="True" />
                    <param name="dry_run" value="True" />
                </section>
                <param name="test_submit" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="False"/>
                    <param name="input_format" value="build_tables"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="False"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>
                            <param name="study_type" value="Epigenetics"/>
                            <param name="study_pubmed_id" value="Test study pubmedID"/>
                            <repeat name="rep_sample">
                                <param name="sample_title" value="Test Sample title"/>
                                <param name="sample_description" value="Test Sample description"/>
                                <param name="scientific_name" value="Test Sample scientific name"/>
                                <param name="tax_id" value="2697049"/>
                                <param name="collection_date" value="2020"/>
                                <param name="geo_location_country" value="Belgium"/>
                                <repeat name="rep_experiment">
                                    <param name="experiment_title" value="Test experiment title"/>
                                    <param name="experiment_design" value="Test experiment design description"/>
                                    <param name="library_strategy" value="CTS"/>
                                    <param name="library_source" value="GENOMIC"/>

                <has_text_matching expression="--data '1.fastqsanger.gz' '2.fastqsanger.gz'"/>
                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
                <not_has_text text="modify" />
            </assert_command>
        </test>
        <!--Test 7: viral submission - User input metadata - Add extension = False-->
        <test expect_failure="False" expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="add"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="True" />
                </section>
                <param name="test_submit" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="False"/>
                    <param name="input_format" value="build_tables"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="True"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>
                            <param name="study_abstract" value="Test study abstract"/>
                            <param name="study_type" value="Epigenetics"/>
                            <param name="study_pubmed_id" value="Test study pubmedID"/>

                <has_text_matching expression="--data '1.fastqsanger.gz' '2.fastqsanger.gz'"/>
                <has_text_matching expression="--action 'add' --center 'Some research center'"/>
                <has_text_matching expression="--checklist ERC000033"/>
            </assert_command>
        </test>
        <!--Test 8: modify option and auto compression - viral submission - User input metadata-->
        <test expect_failure="False" expect_num_outputs="5">
            <conditional name="action_options">
                <param name="action" value="modify"/>
                <section name="test_submit_parameters">
                    <param name="submit_dev" value="False" />
                    <param name="dry_run" value="True" />
                </section>
                <param name="test_submit" value="True"/>
                <conditional name="input_format_conditional">
                    <param name="add_extension" value="False"/>
                    <param name="input_format" value="build_tables"/>
                    <conditional name="conditional_viral_metadata">
                        <param name="viral_sample" value="True"/>
                        <repeat name="rep_study">
                            <param name="study_title" value="Test study title"/>

                                    <param name="library_construction_protocol" value="Test library construction"/>
                                    <param name="platform" value="ILLUMINA"/>
                                    <param name="instrument_model" value="Illumina HiSeq 4000"/>
                                    <repeat name="rep_runs">
                                        <param name="run_base_name" value="run_from_hospital_X"/>
                                        <param name="upload_files" value="1.fastqsanger.gz,2.fastqsanger.gz" ftype="fastqsanger.gz"/>
                                    </repeat>
                                </repeat>
                            </repeat>
                        </repeat>
                    </conditional>
                </conditional>
            </conditional>
            <param name="center" value="Some research center"/>
            <assert_command>
                <has_text_matching expression="ena-upload-cli"/>
                <has_text_matching expression="--data '1.fastqsanger.gz' '2.fastqsanger.gz'"/>
                <has_text_matching expression="--action 'modify' --center 'Some research center'"/>
                <has_text_matching expression="--checklist ERC000033"/>
                <not_has_text text="add" />
            </assert_command>
        </test>
    </tests>
    <help><![CDATA[
        This is a wrapper for the ENA upload tool in https://github.com/usegalaxy-eu/ena-upload-cli
        The input metadata can be submitted following the tabular format of the templates or their excel spreadsheet equivalent in https://github.com/ELIXIR-Belgium/ENA-metadata-templates. This template repo provides ready to use sheets for every ENA sample checklist and is automatically updated.
    ]]></help>
    <citations>
        <citation type="doi">doi:10.1093/bioinformatics/btab421</citation>
    </citations>
</tool>