Mercurial > repos > greg > data_manager_vsnp_excel

changeset 1:d1bf63483e33 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author greg
date Mon, 10 Feb 2020 08:07:05 -0500
parents 84a356e0950a
children 79fcb638533e
files data_manager/vsnp_excel_fetcher.py data_manager/vsnp_excel_fetcher.xml data_manager_conf.xml
diffstat 3 files changed, 22 insertions(+), 34 deletions(-) [+]
line wrap: on
line diff
--- a/data_manager/vsnp_excel_fetcher.py	Fri Feb 07 13:02:25 2020 -0500
+++ b/data_manager/vsnp_excel_fetcher.py	Mon Feb 10 08:07:05 2020 -0500
@@ -1,6 +1,7 @@
 import argparse
 import json
 import os
+import sys
 try:
     # For Python 3.0 and later
     from urllib.request import Request, urlopen
@@ -9,8 +10,8 @@
     from urllib2 import Request, urlopen
 
 
-def url_download(url, target_directory):
-    file_path = os.path.abspath(os.path.join(target_directory, os.path.basename(url)))
+def url_download(url, workdir):
+    file_path = os.path.abspath(os.path.join(workdir, os.path.basename(url)))
     src = None
     dst = None
     try:
@@ -23,29 +24,30 @@
                     dst.write(chunk)
                 else:
                     break
+    except Exception as e:
+        sys.exit(str(e))
     finally:
         if src:
             src.close()
     return file_path
 
 
-def download(dbkey, display_name, url, out_file):
+def download(dbkey, name, url, out_file):
 
     with open(out_file) as fh:
         params = json.loads(fh.read())
 
-    work_directory = params['output_data'][0]['extra_files_path']
-    target_directory = os.path.abspath(os.path.join(work_directory, "vsnp", dbkey))
-    os.mkdir(target_directory)
-    file_path = url_download(url, target_directory)
-    entry_name = "%s" % os.path.basename(file_path)
+    workdir = params['output_data'][0]['extra_files_path']
+    os.makedirs(workdir)
+    file_path = url_download(url, workdir)
+    entry_name = os.path.basename(file_path)
 
     data_manager_json = {"data_tables": {}}
     data_manager_entry = {}
     data_manager_entry['value'] = dbkey
     data_manager_entry['name'] = entry_name
     data_manager_entry['path'] = file_path
-    data_manager_entry['description'] = "Excel file for %s" % display_name
+    data_manager_entry['description'] = "Excel file for %s" % name
     data_manager_json["data_tables"]["vsnp_excel"] = data_manager_entry
 
     with open(out_file, 'w') as fh:
@@ -55,10 +57,10 @@
 parser = argparse.ArgumentParser()
 
 parser.add_argument('--dbkey', dest='dbkey', help='Genome reference dbkey')
-parser.add_argument('--display_name', dest='display_name', help='Reference display name')
+parser.add_argument('--name', dest='name', help='Reference display name')
 parser.add_argument('--url', dest='url', help='URL to download Excel file')
 parser.add_argument('--out_file', dest='out_file', help='JSON output file')
 
 args = parser.parse_args()
 
-download(args.dbkey, args.display_name, args.url, args.out_file)
+download(args.dbkey, args.name, args.url, args.out_file)
--- a/data_manager/vsnp_excel_fetcher.xml	Fri Feb 07 13:02:25 2020 -0500
+++ b/data_manager/vsnp_excel_fetcher.xml	Mon Feb 10 08:07:05 2020 -0500
@@ -7,15 +7,16 @@
     <command detect_errors="exit_code"><![CDATA[
     python '$__tool_directory__/vsnp_excel_fetcher.py'
     --dbkey '${all_fasta_source.fields.dbkey}'
-    --display_name '${all_fasta_source.fields.display_name}'
-    -excel_url '$excel_url'
+    --name '${all_fasta_source.fields.name}'
+    --out_file '$out_file'
+    --url '$url'
     ]]>
     </command>
     <inputs>
         <param name="all_fasta_source" type="select" label="FASTA reference">
             <options from_data_table="all_fasta"/>
         </param>
-        <param name="excel_url" type="text" value="" label="URL to download the Excel file associated with the selected FASTA reference" optional="False" />
+        <param name="url" type="text" value="" label="URL to download the Excel file associated with the selected FASTA reference" optional="False" />
     </inputs>
     <outputs>
         <data name="out_file" format="data_manager_json" />
@@ -25,7 +26,7 @@
         </test>
     </tests>
     <help><![CDATA[
-This tool fetches a vSNP Excel file associated with each supported genome reference to populate the vsnp_ecxel data table.  The dbkey and display_name fields
+This tool fetches a vSNP Excel file associated with each supported genome reference to populate the vsnp_ecxel data table.  The dbkey and name fields
 in the vsnp data table are inherited from the *all_fasta* data table, so no user entry is necessary.  These public vSNP Excel files are available in GitHub
 at https://github.com/USDA-VS/vSNP_reference_options.
 
@@ -33,100 +34,83 @@
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_AF2122/Mbovis_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_AF2122/NC_002945v4.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_AF2122/NC_002945v4.gff
 
  * **Brucella abortus bv. 1 str. 9-941**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus1/Bab1_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus1/NC_006932-NC_006933.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus1/NC_006932-NC_006933.gff
 
  * **Brucella abortus strain BER**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus3/Bab3_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus3/NZ_CP007682-NZ_CP007683.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_abortus3/NZ_CP007682-NZ_CP007683.gff
 
  * **Brucella canis ATCC 23365**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_canis/Bcanis_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_canis/NC_010103-NC_010104.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_canis/NC_010103-NC_010104.gff
 
  * **Brucella ceti TE10759-12**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ceti2/Bceti2_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ceti2/NC_022905-NC_022906.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ceti2/NC_022905-NC_022906.gff
 
  * **Brucella melitensis bv. 1 str. 16M**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1/Bmelbv1_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1/NC_003317-NC_003318.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1/NC_003317-NC_003318.gff
 
  * **Brucella melitensis bv. 3 str. Ether**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv3/Bmelbv3_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv3/NZ_CP007760-NZ_CP007761.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv3/NZ_CP007760-NZ_CP007761.gff
 
  * **Brucella melitensis BwIM_SOM_36b**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1b/Bmelbv1b_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1b/NZ_CP018508-NZ_CP018509.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv1b/NZ_CP018508-NZ_CP018509.gff
 
  * **Brucella melitensis ATCC 23457**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv2/Bmelbv2_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv2/NC_012441-NC_012442.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_melitensis-bv2/NC_012441-NC_012442.gff
 
  * **Brucella ovis ATCC 25840**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ovis/B_ovis_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ovis/NC_009505-NC_009504.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_ovis/NC_009505-NC_009504.gff
 
  * **Brucella suis 1330**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis1/Bsuis1_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis1/NC_017251-NC_017250.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis1/NC_017251-NC_017250.gff
 
  * **Mycobacterium tuberculosis H37Rv**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_H37/H37_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_H37/NC_000962.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Mycobacterium_H37/NC_000962.gff
 
  * **Mycobacterium avium subsp. paratuberculosis strain Telford**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/para-CP033688/CP033688_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/para-CP033688/CP033688.gbk
-   * **Gff file**      
 
  * **Mycobacterium avium subsp. paratuberculosis K-10**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/para-NC002944/NC_002944_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/para-NC002944/NC_002944.gbk
-   * **Gff file**      
 
  * **Brucella suis ATCC 23445**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis2/Bsuis2_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis2/NC_010169-NC_010167.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis2/NC_010169-NC_010167.gff
 
  * **Brucella suis bv. 3 str. 686**
 
    * **Excel file**    https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis3/Bsuis3_define_filter.xlsx
    * **Genbank file**  https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis3/NZ_CP007719-NZ_CP007718.gbk
-   * **Gff file**      https://github.com/USDA-VS/vSNP_reference_options/raw/master/Brucella_suis3/NZ_CP007719-NZ_CP007718.gff
 
     ]]></help>
     <citations>
     </citations>
 </tool>
-
--- a/data_manager_conf.xml	Fri Feb 07 13:02:25 2020 -0500
+++ b/data_manager_conf.xml	Mon Feb 10 08:07:05 2020 -0500
@@ -8,12 +8,14 @@
                 <column name="path" output_ref="out_file">
                     <move type="file" relativize_symlinks="True">
                         <source>${path}</source>
-                        <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">vsnp/${value}/excel</target>
+			<target base="${GALAXY_DATA_MANAGER_DATA_PATH}">vsnp/${value}/excel/${name}</target>
                     </move>
-                    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/vsnp/${value}/excel</value_translation>
+		    <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/vsnp/${value}/excel/${name}</value_translation>
                     <value_translation type="function">abspath</value_translation>
                 </column>
+                <column name="description" />
             </output>
         </data_table>
     </data_manager>
 </data_managers>
+