| Previous changeset 1:5a0d0bee4f8d (2019-12-03) Next changeset 3:431198ad8a09 (2021-12-07) |
|
Commit message:
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_diamond_database_builder commit 75abf7d4b23ed7ae8abce80609d81b20bc882863" |
|
modified:
data_manager/data_manager_diamond_database_builder.py data_manager/data_manager_diamond_database_builder.xml data_manager_conf.xml tool_data_table_conf.xml.sample tool_data_table_conf.xml.test |
|
added:
test-data/diamond_data_manager_ncbi.json test-data/ncbi_accession2taxid.loc test-data/taxonomy/names.dmp test-data/taxonomy/nodes.dmp test-data/taxonomy/prot.accession2taxid |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 data_manager/data_manager_diamond_database_builder.py --- a/data_manager/data_manager_diamond_database_builder.py Tue Dec 03 17:39:48 2019 -0500 +++ b/data_manager/data_manager_diamond_database_builder.py Sun Mar 21 12:07:34 2021 +0000 |
| [ |
| b'@@ -1,17 +1,19 @@\n #!/usr/bin/env python\n+import bz2\n+import gzip\n import json\n-import sys\n-import os\n-import tempfile\n-import shutil\n import optparse\n-import urllib2\n+import os\n+import shutil\n import subprocess\n-from ftplib import FTP\n+import sys\n import tarfile\n+import tempfile\n+import urllib.error\n+import urllib.parse\n+import urllib.request\n import zipfile\n-import gzip\n-import bz2\n+from ftplib import FTP\n \n CHUNK_SIZE = 2**20 # 1mb\n \n@@ -21,11 +23,6 @@\n shutil.rmtree(tmp_dir)\n \n \n-def stop_err(msg):\n- sys.stderr.write(msg)\n- sys.exit(1)\n-\n-\n def _get_files_in_ftp_path(ftp, path):\n path_contents = []\n ftp.retrlines(\'MLSD %s\' % (path), path_contents.append)\n@@ -54,12 +51,17 @@\n return [bz2.BZ2File(file_obj.name, \'rb\')]\n \n \n-def download_from_ncbi(data_manager_dict, params, target_directory, database_id, database_name):\n+def download_from_ncbi(data_manager_dict, params, target_directory,\n+ database_id, database_name):\n NCBI_FTP_SERVER = \'ftp.ncbi.nlm.nih.gov\'\n NCBI_DOWNLOAD_PATH = \'/blast/db/FASTA/\'\n- COMPRESSED_EXTENSIONS = [(\'.tar.gz\', _get_stream_readers_for_tar), (\'.tar.bz2\', _get_stream_readers_for_tar), (\'.zip\', _get_stream_readers_for_zip), (\'.gz\', _get_stream_readers_for_gzip), (\'.bz2\', _get_stream_readers_for_bz2)]\n+ COMPRESSED_EXTENSIONS = [(\'.tar.gz\', _get_stream_readers_for_tar),\n+ (\'.tar.bz2\', _get_stream_readers_for_tar),\n+ (\'.zip\', _get_stream_readers_for_zip),\n+ (\'.gz\', _get_stream_readers_for_gzip),\n+ (\'.bz2\', _get_stream_readers_for_bz2)]\n \n- ncbi_identifier = params[\'param_dict\'][\'reference_source\'][\'requested_identifier\']\n+ ncbi_identifier = params[\'reference_source\'][\'requested_identifier\']\n ftp = FTP(NCBI_FTP_SERVER)\n ftp.login()\n \n@@ -79,9 +81,9 @@\n tmp_dir = tempfile.mkdtemp(prefix=\'tmp-data-manager-ncbi-\')\n ncbi_fasta_filename = os.path.join(tmp_dir, "%s%s" % (ncbi_identifier, ext))\n \n- fasta_base_filename = "%s.fa" % database_id\n- fasta_filename = os.path.join(target_directory, fasta_base_filename)\n- fasta_writer = open(fasta_filename, \'wb+\')\n+ # fasta_base_filename = "%s.fa" % database_id\n+ # fasta_filename = os.path.join(target_directory, fasta_base_filename)\n+ # fasta_writer = open(fasta_filename, \'wb+\')\n \n tmp_extract_dir = os.path.join(tmp_dir, \'extracted_fasta\')\n os.mkdir(tmp_extract_dir)\n@@ -106,8 +108,8 @@\n \n def download_from_url(data_manager_dict, params, target_directory, database_id, database_name):\n # TODO: we should automatically do decompression here\n- urls = filter(bool, map(lambda x: x.strip(), params[\'param_dict\'][\'reference_source\'][\'user_url\'].split(\'\\n\')))\n- fasta_reader = [urllib2.urlopen(url) for url in urls]\n+ urls = list(filter(bool, [x.strip() for x in params[\'reference_source\'][\'user_url\'].split(\'\\n\')]))\n+ fasta_reader = [urllib.request.urlopen(url) for url in urls]\n \n data_table_entry = _stream_fasta_to_file(fasta_reader, target_directory, database_id, database_name, params)\n _add_data_table_entry(data_manager_dict, data_table_entry)\n@@ -115,19 +117,19 @@\n \n def download_from_history(data_manager_dict, params, target_directory, database_id, database_name):\n # TODO: allow multiple FASTA input files\n- input_filename = params[\'param_dict\'][\'reference_source\'][\'input_fasta\']\n+ input_filename = params[\'reference_source\'][\'input_fasta\']\n if isinstance(input_filename, list):\n fasta_reader = [open(filename, \'rb\') for filename in input_filename]\n else:\n- fasta_reader = open(input_filename)\n+ fasta_reader = open(input_filename, \'rb\')\n \n data_table_entry = _stream_fasta_to_file(fasta_reader, target_directory, database_id, database_name, params)\n _add_data_table_entry(data_manager_dict, data_table_entry)\n \n \n def copy_from_directory(data_manager_dict, params, target_directory, database_id, datab'..b'[\'--taxonmap\',\n+ os.path.join(params[\'tax_cond\'][\'ncbi_tax\'], \'prot.accession2taxid.gz\')])\n+ elif os.path.isfile(os.path.join(params[\'tax_cond\'][\'ncbi_tax\'], \'prot.accession2taxid\')):\n+ args.extend([\'--taxonmap\',\n+ os.path.join(params[\'tax_cond\'][\'ncbi_tax\'], \'prot.accession2taxid\')])\n+ else:\n+ raise Exception(\'Unable to find prot.accession2taxid file in %s\' % (params[\'tax_cond\'][\'ncbi_tax\']))\n+\n+ args.extend([\'--taxonnodes\',\n+ os.path.join(params[\'tax_cond\'][\'ncbi_tax\'], \'nodes.dmp\')])\n+ args.extend([\'--taxonnames\',\n+ os.path.join(params[\'tax_cond\'][\'ncbi_tax\'], \'names.dmp\')])\n \n tmp_stderr = tempfile.NamedTemporaryFile(prefix="tmp-data-manager-diamond-database-builder-stderr")\n- proc = subprocess.Popen(args=args, shell=False, cwd=target_directory, stderr=tmp_stderr.fileno())\n+ proc = subprocess.Popen(args=args, shell=False, cwd=target_directory,\n+ stderr=tmp_stderr.fileno())\n return_code = proc.wait()\n if return_code:\n tmp_stderr.flush()\n tmp_stderr.seek(0)\n- print >> sys.stderr, "Error building diamond database:"\n+ print("Error building diamond database:", file=sys.stderr)\n while True:\n chunk = tmp_stderr.read(CHUNK_SIZE)\n if not chunk:\n break\n- sys.stderr.write(chunk)\n+ sys.stderr.write(chunk.decode(\'utf-8\'))\n sys.exit(return_code)\n tmp_stderr.close()\n os.remove(temp_fasta.name)\n- return dict(value=database_id, name=database_name, db_path="%s.dmnd" % fasta_base_filename)\n+ return dict(value=database_id, name=database_name,\n+ db_path="%s.dmnd" % fasta_base_filename)\n \n \n def _create_symlink(input_filename, target_directory, database_id, database_name):\n@@ -210,27 +230,36 @@\n return dict(value=database_id, name=database_name, db_path=fasta_base_filename)\n \n \n-REFERENCE_SOURCE_TO_DOWNLOAD = dict(ncbi=download_from_ncbi, url=download_from_url, history=download_from_history, directory=copy_from_directory)\n+REFERENCE_SOURCE_TO_DOWNLOAD = dict(ncbi=download_from_ncbi,\n+ url=download_from_url,\n+ history=download_from_history,\n+ directory=copy_from_directory)\n \n \n def main():\n # Parse Command Line\n parser = optparse.OptionParser()\n- parser.add_option(\'-d\', \'--dbkey_description\', dest=\'dbkey_description\', action=\'store\', type="string", default=None, help=\'dbkey_description\')\n+ parser.add_option(\'-d\', \'--dbkey_description\', dest=\'dbkey_description\',\n+ action=\'store\', type="string", default=None,\n+ help=\'dbkey_description\')\n (options, args) = parser.parse_args()\n \n filename = args[0]\n \n- params = json.loads(open(filename).read())\n+ with open(filename) as fp:\n+ params = json.load(fp)\n target_directory = params[\'output_data\'][0][\'extra_files_path\']\n os.mkdir(target_directory)\n data_manager_dict = {}\n \n- database_id = params[\'param_dict\'][\'database_id\']\n- database_name = params[\'param_dict\'][\'database_name\']\n+ param_dict = params[\'param_dict\']\n+ database_id = param_dict[\'database_id\']\n+ database_name = param_dict[\'database_name\']\n+ if param_dict[\'tax_cond\'][\'tax_select\'] == "ncbi":\n+ param_dict[\'tax_cond\'][\'ncbi_tax\'] = args[1]\n \n # Fetch the FASTA\n- REFERENCE_SOURCE_TO_DOWNLOAD[params[\'param_dict\'][\'reference_source\'][\'reference_source_selector\']](data_manager_dict, params, target_directory, database_id, database_name)\n+ REFERENCE_SOURCE_TO_DOWNLOAD[param_dict[\'reference_source\'][\'reference_source_selector\']](data_manager_dict, param_dict, target_directory, database_id, database_name)\n \n # save info to json file\n open(filename, \'w\').write(json.dumps(data_manager_dict, sort_keys=True))\n' |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 data_manager/data_manager_diamond_database_builder.xml --- a/data_manager/data_manager_diamond_database_builder.xml Tue Dec 03 17:39:48 2019 -0500 +++ b/data_manager/data_manager_diamond_database_builder.xml Sun Mar 21 12:07:34 2021 +0000 |
| [ |
| @@ -1,36 +1,60 @@ -<tool id="diamond_database_builder" name="Diamond" tool_type="manage_data" version="0.0.2"> +<tool id="diamond_database_builder" name="Diamond" tool_type="manage_data" version="0.0.3" profile="19.05"> <description> Database builder</description> <requirements> - <requirement type="package" version="0.9.29">diamond</requirement> - <requirement type="package" version="2.7">python</requirement> + <requirement type="package" version="2.0.8">diamond</requirement> + <requirement type="package" version="3.7">python</requirement> </requirements> - <command> + <command><![CDATA[ python '$__tool_directory__/data_manager_diamond_database_builder.py' '${out_file}' - </command> + #if $tax_cond.tax_select == "ncbi" + '$tax_cond.ncbi_tax.fields.path' + #end if + ]]></command> <inputs> <param type="text" name="database_name" value="" label="Database name or description" help="This will be shown to the user in the database selectbox."/> <param type="text" name="database_id" value="" label="Database ID" help="This needs to be unique. No whitespaces allowed. Use underscores." /> <conditional name="reference_source"> - <param name="reference_source_selector" type="select" label="Choose the source for the reference genome"> - <option value="ncbi">NCBI</option> - <option value="url">URL</option> - <option value="history" selected="True" help="Recommended to archive the original FASTA file as well.">History</option> - <option value="directory">Directory on Server</option> - </param> - <when value="ncbi"> - <param type="text" name="requested_identifier" value="" label="NCBI identifier" help="e.g. nr" optional="False" /> - </when> - <when value="url"> - <param type="text" area="True" name="user_url" value="http://" label="URLs" optional="False" /> - </when> - <when value="history"> - <param name="input_fasta" type="data" format="fasta" label="FASTA File" multiple="False" optional="False" /> - </when> - <when value="directory"> - <param type="text" name="fasta_filename" value="" label="Full path to FASTA File on disk" optional="False" /> - <param type="boolean" name="create_symlink" truevalue="create_symlink" falsevalue="copy_file" label="Create symlink to orignal data instead of copying" checked="False" /> - </when> + <param name="reference_source_selector" type="select" label="Choose the source for the reference genome"> + <option value="ncbi">NCBI</option> + <option value="url">URL</option> + <option value="history" selected="true" help="Recommended to archive the original FASTA file as well.">History</option> + <option value="directory">Directory on Server</option> + </param> + <when value="ncbi"> + <param type="text" name="requested_identifier" value="" label="NCBI identifier" help="e.g. nr" optional="false" /> + </when> + <when value="url"> + <param type="text" name="user_url" value="http://" label="URLs" optional="false" /> + </when> + <when value="history"> + <param name="input_fasta" type="data" format="fasta" label="FASTA File" multiple="false" optional="false" /> + </when> + <when value="directory"> + <param type="text" name="fasta_filename" value="" label="Full path to FASTA File on disk" optional="false" /> + <param type="boolean" name="create_symlink" truevalue="create_symlink" falsevalue="copy_file" label="Create symlink to orignal data instead of copying" checked="false" /> + </when> + </conditional> + <conditional name="tax_cond"> + <param name="tax_select" type="select" label="Add taxonomic information"> + <option value="no">Do not add taxonomic information</option> + <option value="ncbi">Use cached NCBI taxonomy</option> + <option value="history">Use data from history</option> + </param> + <when value="no"/> + <when value="ncbi"> + <param name="ncbi_tax" type="select" label="Select NCBI taxonomy"> + <options from_data_table="ncbi_accession2taxid"> + <filter type="sort_by" column="2"/> + <validator type="no_options" message="No taxonomy available"/> + </options> + </param> + </when> + <when value="history"> + <param argument="--taxonmap" type="data" format="tabular" label="protein accession to taxid mapping file" help="" /> + <param argument="--taxonnodes" type="data" format="tabular" label="taxonomy nodes.dmp from NCBI" help="" /> + <param argument="--taxonnames" type="data" format="tabular" label="taxonomy names.dmp from NCBI" help="" /> + </when> </conditional> </inputs> <outputs> @@ -44,16 +68,41 @@ <param name="reference_source_selector" value="history"/> <param name="input_fasta" value="phiX174.fasta"/> </conditional> + <conditional name="tax_cond"> + <param name="tax_select" value="history"/> + <param name="taxonmap" ftype="tabular" value="taxonomy/prot.accession2taxid" /> + <param name="taxonnodes" ftype="tabular" value="taxonomy/nodes.dmp" /> + <param name="taxonnames" ftype="tabular" value="taxonomy/names.dmp" /> + </conditional> <output name="out_file" value="diamond_data_manager.json"/> </test> + <test> + <param name="database_name" value="pdbaa"/> + <param name="database_id" value="pdbaa"/> + <conditional name="reference_source"> + <param name="reference_source_selector" value="ncbi"/> + <param name="requested_identifier" value="pdbaa"/> + </conditional> + <conditional name="tax_cond"> + <param name="tax_select" value="ncbi"/> + <param name="ncbi_tax" value="2021-03-19"/> + </conditional> + <output name="out_file" value="diamond_data_manager_ncbi.json"/> + </test> </tests> <help> .. class:: infomark -NCBI databases can be downded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/ +NCBI databases can be downloaded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/ For example the NR database can be downloaded from ftp://ftp.ncbi.nlm.nih.gov/blast/db/FASTA/nr.gz + +- taxonmap: Path to mapping file that maps NCBI protein accession numbers to taxon ids (gzip compressed). This parameter is optional and needs to be supplied in order to provide taxonomy features. The file can be downloaded from NCBI: ftp.ncbi.nlm.nih.gov/pub/taxonomy/accession2taxid/prot.accession2taxid.FULL.gz + +- taxonnames: Path to the names.dmp file from the NCBI taxonomy. This parameter is optional and needs to be supplied in order to provide taxonomy features. The file is contained within this archive downloadable at NCBI: ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdmp.zip + +- taxonnodes: Path to the nodes.dmp file from the NCBI taxonomy. This parameter is optional and needs to be supplied in order to provide taxonomy features. The file is contained within this archive downloadable at NCBI: ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdmp.zip </help> <citations> <citation type="doi">10.1038/nmeth.3176</citation> |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 data_manager_conf.xml --- a/data_manager_conf.xml Tue Dec 03 17:39:48 2019 -0500 +++ b/data_manager_conf.xml Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -1,13 +1,12 @@ <?xml version="1.0"?> <data_managers> - <data_manager tool_file="data_manager/data_manager_diamond_database_builder.xml" id="diamond_database_builder" version="0.0.1"> + <data_manager tool_file="data_manager/data_manager_diamond_database_builder.xml" id="diamond_database_builder" version="0.0.3"> <data_table name="diamond_database"> <output> <column name="value" /> <column name="name" /> <column name="db_path" output_ref="out_file" > <move type="directory" relativize_symlinks="True"> - <!-- <source>${db_path}</source>--> <!-- out_file.extra_files_path is used as base by default --> <!-- if no source, eg for type=directory, then refers to base --> <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">diamond_database/${value}</target> </move> <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/diamond_database/${value}/${db_path}</value_translation> |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 test-data/diamond_data_manager_ncbi.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/diamond_data_manager_ncbi.json Sun Mar 21 12:07:34 2021 +0000 |
| [ |
| @@ -0,0 +1,1 @@ +{"data_tables": {"diamond_database": [{"db_path": "pdbaa.fa.dmnd", "name": "pdbaa", "value": "pdbaa"}]}} \ No newline at end of file |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 test-data/ncbi_accession2taxid.loc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/ncbi_accession2taxid.loc Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -0,0 +1,6 @@ +# Tab separated fields where +# value is unique key +# name is descriptive name +# path is path to directory containing accession2taxid files +#value name path +2021-03-19 2021-03-19 ${__HERE__}/taxonomy |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 test-data/taxonomy/names.dmp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/taxonomy/names.dmp Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -0,0 +1,11 @@ +1 | all | | synonym | +1 | root | | scientific name | +2 | Bacteria | Bacteria <bacteria> | scientific name | +2 | bacteria | | blast name | +2 | eubacteria | | genbank common name | +2 | Monera | Monera <bacteria> | in-part | +3 | Procaryotae | Procaryotae <bacteria> | in-part | +3 | Prokaryotae | Prokaryotae <bacteria> | in-part | +3 | Prokaryota | Prokaryota <bacteria> | in-part | +3 | prokaryote | prokaryote <bacteria> | in-part | +3 | prokaryotes | prokaryotes <bacteria> | in-part | |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 test-data/taxonomy/nodes.dmp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/taxonomy/nodes.dmp Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -0,0 +1,3 @@ +1 | 1 | no rank | | 8 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | | +2 | 1 | species | AC | 0 | 1 | 11 | 1 | 0 | 1 | 1 | 0 | | +3 | 1 | species | AC | 0 | 1 | 11 | 1 | 0 | 1 | 1 | 0 | | |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 test-data/taxonomy/prot.accession2taxid --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/taxonomy/prot.accession2taxid Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -0,0 +1,4 @@ +accession accession.version taxid gi +AAD44166 AAD44166.1 2 5524211 +AAD44167 AAD44167.1 3 5524212 + |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 tool_data_table_conf.xml.sample --- a/tool_data_table_conf.xml.sample Tue Dec 03 17:39:48 2019 -0500 +++ b/tool_data_table_conf.xml.sample Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -4,4 +4,9 @@ <columns>value, name, db_path</columns> <file path="tool-data/diamond_database.loc" /> </table> + <!-- Locations of taxonomy data downloaded from NCBI --> + <table name="ncbi_accession2taxid" comment_char="#"> + <columns>value, name, path</columns> + <file path="tool-data/ncbi_accession2taxid.loc" /> + </table> </tables> |
| b |
| diff -r 5a0d0bee4f8d -r 5558f74bd296 tool_data_table_conf.xml.test --- a/tool_data_table_conf.xml.test Tue Dec 03 17:39:48 2019 -0500 +++ b/tool_data_table_conf.xml.test Sun Mar 21 12:07:34 2021 +0000 |
| b |
| @@ -4,4 +4,8 @@ <columns>value, name, db_path</columns> <file path="${__HERE__}/test-data/diamond_database.loc" /> </table> + <table name="ncbi_accession2taxid" comment_char="#"> + <columns>value, name, path</columns> + <file path="${__HERE__}/test-data/ncbi_accession2taxid.loc" /> + </table> </tables> |