|
Commit message:
Uploaded |
|
added:
data_manager/fetch_mlst_data.py data_manager/fetch_mlst_data.xml data_manager_conf.xml tool-data/mlst_data.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml |
| b |
| diff -r 000000000000 -r 25d4d9f313a0 data_manager/fetch_mlst_data.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/fetch_mlst_data.py Wed Jul 13 05:50:48 2016 -0400 |
| [ |
| b'@@ -0,0 +1,293 @@\n+#!/usr/bin/env python\n+\n+\'\'\'\n+Download MLST datasets from this site: http://pubmlst.org/data/ by\n+parsing an xml file (http://pubmlst.org/data/dbases.xml).\n+\n+Data is downloaded for a species determined by the user:\n+- profiles (maps STs to allele numbers)\n+- numbered sequences for each locus in the scheme\n+\n+In addition, the alleles are concatenated together for use with SRST2.\n+\n+A log file is also generated in the working directory, detailing the\n+time, date and location of all files downloaded, as well as the <retrieved>\n+tag which tells us when the XML entry was last updated.\n+\n+If the species name input by the user matches multiple <species> in the\n+xml file, the script simply reports the possible matches so the user can\n+try again.\n+\'\'\'\n+\n+"""\n+- Remove empty line at the end of profiles.txt file.\n+- Ensure the allele names at the profiles.txt file don\'t contain "_".\n+\n+"""\n+from argparse import ArgumentParser\n+import xml.dom.minidom as xml\n+import urllib2 as url\n+import re\n+import os\n+import sys\n+import glob\n+import csv\n+import shutil\n+from urlparse import urlparse\n+import time\n+import subprocess\n+from json import dumps\n+from json import loads\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+def parse_args():\n+ parser = ArgumentParser(description=\'Download MLST datasets by species\'\n+ \'from pubmlst.org.\')\n+\n+ parser.add_argument(\'--repository_url\',\n+ metavar = \'URL\',\n+ default = \'http://pubmlst.org/data/dbases.xml\',\n+ help = \'URL for MLST repository XML index\')\n+\n+ parser.add_argument(\'--species\',\n+ metavar = \'NAME\',\n+ required = True,\n+ help = \'The name of the species that you want to download (e.g. "Escherichia coli")\')\n+\n+ parser.add_argument(\'--outfile\',\n+ metavar = \'FILE\',\n+ required = True,\n+ help = \'The name of the Json file to write that galaxy stuff to.\')\n+\n+ parser.add_argument(\'--reference\',\n+ metavar = \'ACCESSION\',\n+ required = True,\n+ help = \'NCBI accession number of the reference genome to use for flanking regions.\')\n+\n+ return parser.parse_args()\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+def main():\n+\n+ """\n+ <species>\n+ Achromobacter spp.\n+ <mlst>\n+ <database>\n+ <url>http://pubmlst.org/achromobacter</url>\n+ <retrieved>2015-08-11</retrieved>\n+ <profiles>\n+ <count>272</count>\n+ <url>http://pubmlst.org/data/profiles/achromobacter.txt</url>\n+ </profiles>\n+ <loci>\n+ <locus>\n+ nusA\n+ <url>\n+ http://pubmlst.org/data/alleles/achromobacter/nusA.tfa\n+ </url>\n+ </locus>\n+ <locus>\n+ rpoB\n+ <url>\n+ http://pubmlst.org/data/alleles/achromobacter/rpoB.tfa\n+ </url>\n+ </locus>\n+ """\n+\n+ args = parse_args()\n+ docFile = url.urlopen(args.repository_url) # url address #args.repository_url =http://pubmlst.org/data/dbases.xml\n+\n+ doc = xml.parse(docFile)\n+ root = doc.childNodes[0]\n+ found_species = []\n+\n+ if args.species == "Escherichia coli":\n+ args.species = "Escherichia coli#1"\n+ elif args.species == "Acinetobacter baumannii":\n+ args.species = "Acinetobacter baumannii#1"\n+ elif args.species == "Pasteurella multocida":\n+ args.species = "Pasteurella multocida#1"\n+ else:\n+ pass\n+\n+ for species_node in root.getElementsByTagName(\'species\'):\n+ info = getSpeciesInfo(species_node, args.species)\n+ if info != None:\n+ found_species.append(info)\n+\n+ if len(found_species) == 0:\n+ sys.stderr.write("No species matched your query.\\n")\n+ exit(1)\n+\n+ if len(found_species) > 1:\n+ sys.stderr.writ'..b'--------------------------------------------------------------------------------\n+\n+# test if a node is an Element and that it has a specific tag name\n+def testElementTag(node, name):\n+ return node.nodeType == node.ELEMENT_NODE and node.localName == name\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+# Get the text from an element node with a text node child\n+def getText(element):\n+ result = \'\'\n+ for node in element.childNodes:\n+ if node.nodeType == node.TEXT_NODE:\n+ result += node.data\n+ return normaliseText(result)\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+# remove unwanted whitespace including linebreaks etc.\n+def normaliseText(str):\n+ return \' \'.join(str.split())\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+# A collection of interesting information about a taxa\n+class SpeciesInfo(object):\n+ def __init__(self):\n+ self.name = None # String name of species\n+ self.database_url = None # URL as string\n+ self.retrieved = None # date as string\n+ self.profiles_url = None # URL as string\n+ self.profiles_count = None # positive integer\n+ self.loci = [] # list of loci\n+\n+ def __str__(self):\n+ s = "Name: %s\\n" % self.name\n+ s += "Database URL: %s\\n" % self.database_url\n+ s += "Retrieved: %s\\n" % self.retrieved\n+ s += "Profiles URL: %s\\n" % self.profiles_url\n+ s += "Profiles count: %s\\n" % self.profiles_count\n+ s += "Loci: %s\\n" % (\',\'.join([str(x) for x in self.loci]))\n+ return s\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+class LocusInfo(object):\n+ def __init__(self):\n+ self.url = None\n+ self.name = None\n+ def __str__(self):\n+ return "Locus: name:%s,url:%s" % (self.name, self.url)\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+# retrieve the interesting information for a given sample element\n+def getSpeciesInfo(species_node, species):\n+ this_name = getText(species_node)\n+ print this_name\n+ if this_name.startswith(species):\n+ info = SpeciesInfo()\n+ info.name = this_name\n+ for mlst_node in species_node.getElementsByTagName(\'mlst\'):\n+ for database_node in mlst_node.getElementsByTagName(\'database\'):\n+ for database_child_node in database_node.childNodes:\n+ if testElementTag(database_child_node, \'url\'):\n+ info.database_url = getText(database_child_node)\n+ elif testElementTag(database_child_node, \'retrieved\'):\n+ info.retrieved = getText(database_child_node)\n+ elif testElementTag(database_child_node, \'profiles\'):\n+ for profile_count in database_child_node.getElementsByTagName(\'count\'):\n+ info.profiles_count = getText(profile_count)\n+ for profile_url in database_child_node.getElementsByTagName(\'url\'):\n+ info.profiles_url = getText(profile_url)\n+ elif testElementTag(database_child_node, \'loci\'):\n+ for locus_node in database_child_node.getElementsByTagName(\'locus\'):\n+ locus_info = LocusInfo()\n+ locus_info.name = getText(locus_node)\n+ for locus_url in locus_node.getElementsByTagName(\'url\'):\n+ locus_info.url = getText(locus_url)\n+ info.loci.append(locus_info)\n+\n+ return info\n+ else:\n+ return None\n+\n+# --------------------------------------------------------------------------------------------------\n+\n+if __name__ == \'__main__\':\n+ main()\n' |
| b |
| diff -r 000000000000 -r 25d4d9f313a0 data_manager/fetch_mlst_data.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/fetch_mlst_data.xml Wed Jul 13 05:50:48 2016 -0400 |
| b |
| b'@@ -0,0 +1,157 @@\n+<tool id="fetch_mlst_data" name="Fetch MLST Data" version="0.0.1" tool_type="manage_data">\n+ <description>fetching</description>\n+ <requirements>\n+ <requirement type="package" version="2.7.10">python</requirement>\n+ </requirements>\n+ <command interpreter="python">fetch_mlst_data.py --outfile "${out_file}" --species "$species" --reference $reference</command>\n+ <inputs>\n+ <param name="species" type="select" label="Select the organism">\n+ <option value="Achromobacter spp.">Achromobacter spp.</option>\n+ <option value="Acinetobacter baumannii">Acinetobacter baumannii</option>\n+ <option value="Aeromonas spp.">Aeromonas spp.</option>\n+ <option value="Anaplasma phagocytophilum">Anaplasma phagocytophilum</option>\n+ <option value="Arcobacter spp.">Arcobacter spp.</option>\n+ <option value="Aspergillus fumigatus">Aspergillus fumigatus</option>\n+ <option value="Bacillus cereus">Bacillus cereus</option>\n+ <option value="Bacillus licheniformis">Bacillus licheniformis</option>\n+ <option value="Bacillus subtilis">Bacillus subtilis</option>\n+ <option value="Bartonella henselae">Bartonella henselae</option>\n+ <option value="Bordetella spp.">Bordetella spp.</option>\n+ <option value="Borrelia spp.">Borrelia spp.</option>\n+ <option value="Brachyspira hampsonii">Brachyspira hampsonii</option>\n+ <option value="Brachyspira hyodysenteriae">Brachyspira hyodysenteriae</option>\n+ <option value="Brachyspira intermedia">Brachyspira intermedia</option>\n+ <option value="Brachyspira pilosicoli">Brachyspira pilosicoli</option>\n+ <option value="Brachyspira spp.">Brachyspira spp.</option>\n+ <option value="Burkholderia cepacia complex">Burkholderia cepacia complex</option>\n+ <option value="Burkholderia pseudomallei">Burkholderia pseudomallei</option>\n+ <option value="Campylobacter concisus/curvus">Campylobacter concisus/curvus</option>\n+ <option value="Campylobacter fetus">Campylobacter fetus</option>\n+ <option value="Campylobacter helveticus">Campylobacter helveticus</option>\n+ <option value="Campylobacter hyointestinalis">Campylobacter hyointestinalis</option>\n+ <option value="Campylobacter insulaenigrae">Campylobacter insulaenigrae</option>\n+ <option value="Campylobacter jejuni">Campylobacter jejuni</option>\n+ <option value="Campylobacter lanienae">Campylobacter lanienae</option>\n+ <option value="Campylobacter lari">Campylobacter lari</option>\n+ <option value="Campylobacter sputorum">Campylobacter sputorum</option>\n+ <option value="Campylobacter upsaliensis">Campylobacter upsaliensis</option>\n+ <option value="Candida albicans">Candida albicans</option>\n+ <option value="Candida glabrata">Candida glabrata</option>\n+ <option value="Candida krusei">Candida krusei</option>\n+ <option value="Candida tropicalis">Candida tropicalis</option>\n+ <option value="Carnobacterium maltaromaticum">Carnobacterium maltaromaticum</option>\n+ <option value="Chlamydiales spp.">Chlamydiales spp.</option>\n+ <option value="Citrobacter freundii">Citrobacter freundii</option>\n+ <option value="Clonorchis sinensis">Clonorchis sinensis</option>\n+ <option value="Clostridium botulinum">Clostridium botulinum</option>\n+ <option value="Clostridium difficile">Clostridium difficile</option>\n+ <option value="Clostridium septicum">Clostridium septicum</option>\n+ <option value="Corynebacterium diphtheriae">Corynebacterium diphtheriae</option>\n+ <option value="Cronobacter spp.">Cronobacter spp.</option>\n+ <option value="Enterobacter cloacae">Enterobacter cloacae</option>\n+ <option value="Enterococcus faecalis">Enterococcus faecalis</option>\n+ <option value="Enterococcus faecium">Enterococcus faecium</option>\n+ <option value="Escherichia col'..b' enterica</option>\n+ <option value="Sinorhizobium spp.">Sinorhizobium spp.</option>\n+ <option value="Staphylococcus aureus">Staphylococcus aureus</option>\n+ <option value="Staphylococcus epidermidis">Staphylococcus epidermidis</option>\n+ <option value="Staphylococcus haemolyticus">Staphylococcus haemolyticus</option>\n+ <option value="Staphylococcus lugdunensis">Staphylococcus lugdunensis</option>\n+ <option value="Staphylococcus pseudintermedius">Staphylococcus pseudintermedius</option>\n+ <option value="Stapylococcus hominis">Stapylococcus hominis</option>\n+ <option value="Stenotrophomonas maltophilia">Stenotrophomonas maltophilia</option>\n+ <option value="Streptococcus agalactiae">Streptococcus agalactiae</option>\n+ <option value="Streptococcus canis">Streptococcus canis</option>\n+ <option value="Streptococcus dysgalactiae equisimilis">Streptococcus dysgalactiae equisimilis</option>\n+ <option value="Streptococcus gallolyticus">Streptococcus gallolyticus</option>\n+ <option value="Streptococcus oralis">Streptococcus oralis</option>\n+ <option value="Streptococcus pneumoniae">Streptococcus pneumoniae</option>\n+ <option value="Streptococcus pyogenes">Streptococcus pyogenes</option>\n+ <option value="Streptococcus suis">Streptococcus suis</option>\n+ <option value="Streptococcus thermophilus">Streptococcus thermophilus</option>\n+ <option value="Streptococcus uberis">Streptococcus uberis</option>\n+ <option value="Streptococcus zooepidemicus">Streptococcus zooepidemicus</option>\n+ <option value="Streptomyces spp">Streptomyces spp</option>\n+ <option value="Taylorella spp.">Taylorella spp.</option>\n+ <option value="Tenacibaculum spp.">Tenacibaculum spp.</option>\n+ <option value="Trichomonas vaginalis">Trichomonas vaginalis</option>\n+ <option value="Vibrio cholerae">Vibrio cholerae</option>\n+ <option value="Vibrio parahaemolyticus">Vibrio parahaemolyticus</option>\n+ <option value="Vibrio spp.">Vibrio spp.</option>\n+ <option value="Vibrio tapetis"> Vibrio tapetis</option>\n+ <option value="Vibrio vulnificus">Vibrio vulnificus</option>\n+ <option value="Wolbachia">Wolbachia</option>\n+ <option value="Xylella fastidiosa">Xylella fastidiosa</option>\n+ <option value="Yersinia pseudotuberculosis">Yersinia pseudotuberculosis</option>\n+ <option value="Yersinia ruckeri">Yersinia ruckeri</option>\n+ <option value="Yersinia spp.">Yersinia spp.</option>\n+\t </param>\n+\n+ <param name="reference" type="text" value="NC_xxxxxx" label="Please provide a NCBI accession number for the reference genome to use." help="A reference genome is used for extracting the flanking regions to tying genes. Please see help below." />\n+\n+ </inputs>\n+ <outputs>\n+ <data name="out_file" format="data_manager_json"/>\n+ </outputs>\n+\n+ <help>\n+**What it does**\n+\n+Fetches MLST loci and profiles for a given organism from pubmlst.org and populates the "mlst" data table.\n+\n+A reference must be provided each time. Use an NBCI accession number (e.g. NC_003210) so it can be automatically downloaded.\n+\n+Here are some valid accession number for the species we use most frequently:\n+\n+\n+======================== =====================\n+Organism recommended accession\n+======================== =====================\n+Campylobacter jejuni AL111168\n+Escherichia coli CP002797\n+Listeria monocytogenes NC_003210\n+Salmonella enterica NC_003197\n+Staphylococcus aureus NC_002952\n+Streptococcus pneumoniae NC_017769\n+Streptococcus pyogenes AE014074\n+======================== =====================\n+\n+------\n+\n+If your organism of choice is not on this list you need to look up the correct accession number for a valid reference genome on the NCBI website (http://www.ncbi.nlm.nih.gov/nuccore/).\n+\n+ </help>\n+</tool>\n' |
| b |
| diff -r 000000000000 -r 25d4d9f313a0 data_manager_conf.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Wed Jul 13 05:50:48 2016 -0400 |
| b |
| @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/fetch_mlst_data.xml" id="fetch_mlst_data" version="1.0.0"> + <data_table name="mlst_data"> + <output> + <column name="value" /> + <column name="dbkey" /> + <column name="name" /> + <column name="time_stamp" /> + <column name="file_path" output_ref="out_file" > + <move type="directory"> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">mlst_data</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/mlst_data/${dbkey}/${time_stamp}</value_translation> <!-- Store this value in the final Data Table --> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers> |
| b |
| diff -r 000000000000 -r 25d4d9f313a0 tool-data/mlst_data.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/mlst_data.loc.sample Wed Jul 13 05:50:48 2016 -0400 |
| b |
| @@ -0,0 +1,7 @@ +#This file contains entries that point to available MLST data +#It has 5 columns separated by a tab character +#value species name +#dbkey species name +#name Species name + time stamp to be shown in table, eg. Klebsiella pneumoniae-20160707142859 +#time_stamp when the data was downloaded +#file_path absolute path to there allele fastas, profile.txt and reference are stored |
| b |
| diff -r 000000000000 -r 25d4d9f313a0 tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Wed Jul 13 05:50:48 2016 -0400 |
| b |
| @@ -0,0 +1,7 @@ +<?xml version="1.0"?> +<tables> + <table name="mlst_data" comment_char="#"> + <columns>value, dbkey, name, time_stamp, file_path</columns> + <file path="tool-data/mlst_data.loc" /> + </table> +</tables> |
| b |
| diff -r 000000000000 -r 25d4d9f313a0 tool_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Wed Jul 13 05:50:48 2016 -0400 |
| b |
| @@ -0,0 +1,6 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="python" version="2.7.10"> + <repository changeset_revision="0339c4a9b87b" name="package_python_2_7_10" owner="iuc" prior_installation_required="True" toolshed="https://toolshed.g2.bx.psu.edu" /> + </package> +</tool_dependency> |