Mercurial > repos > iuc > data_manager_gtdbtk_database_installer
changeset 0:629464b96c2e draft
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_gtdbtk_database_installer commit 69da0d34ea6e72c0427ec2d3e882c86cbb90833f
| author | iuc |
|---|---|
| date | Thu, 08 Dec 2022 19:26:49 +0000 |
| parents | |
| children | 2814c058a087 |
| files | data_manager/gtdbtk_database_installer.py data_manager/gtdbtk_database_installer.xml data_manager_conf.xml test-data/gtdbtk_database.loc tool-data/gtdbtk_database.loc.sample tool_data_table_conf.xml.sample tool_data_table_conf.xml.test |
| diffstat | 7 files changed, 214 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/gtdbtk_database_installer.py Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +import argparse +import json +import os +import shutil +import sys +import tarfile +from urllib.parse import urlparse +from urllib.request import Request +from urllib.request import urlopen + + +def url_download(url, target_directory): + url_parts = urlparse(url) + tarball = os.path.abspath(os.path.join(target_directory, os.path.basename(url_parts.path))) + src = None + dst = None + try: + req = Request(url) + src = urlopen(req) + with open(tarball, 'wb') as dst: + while True: + chunk = src.read(2**10) + if chunk: + dst.write(chunk) + else: + break + except Exception as e: + sys.exit(str(e)) + finally: + if src is not None: + src.close() + if tarfile.is_tarfile(tarball): + fh = tarfile.open(tarball, 'r:*') + else: + return tarball + fh.extractall(target_directory) + fh.close() + os.remove(tarball) + # The tarball extraction will create a directory named + # something like release202 in the target_directory, so + # we need to move the items in that directory to the + # target directory. + subdir = next(os.walk(target_directory))[1][0] + subdir_path = os.path.join(target_directory, subdir) + items = os.listdir(subdir_path) + for item in items: + item_path = os.path.join(subdir_path, item) + shutil.move(item_path, target_directory) + os.rmdir(subdir_path) + return target_directory + + +def download(database_id, database_name, url, out_file): + + with open(out_file) as fh: + params = json.load(fh) + + target_directory = params['output_data'][0]['extra_files_path'] + os.makedirs(target_directory) + file_path = url_download(url, target_directory) + + data_manager_json = {"data_tables": {}} + data_manager_entry = {} + data_manager_entry['value'] = database_id + data_manager_entry['name'] = database_name + data_manager_entry['path'] = file_path + data_manager_json["data_tables"]["gtdbtk_database"] = data_manager_entry + + with open(out_file, 'w') as fh: + json.dump(data_manager_json, fh, sort_keys=True) + + +parser = argparse.ArgumentParser() + +parser.add_argument('--database_name', dest='database_name', help='GTDB-Tk database display name') +parser.add_argument('--database_id', dest='database_id', help='Unique GTDB-Tk database id') +parser.add_argument('--url', dest='url', help='URL to download GTDB-Tk databse version') +parser.add_argument('--out_file', dest='out_file', help='JSON output file') + +args = parser.parse_args() + +download(args.database_id, args.database_name, args.url, args.out_file)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/gtdbtk_database_installer.xml Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,48 @@ +<tool id="gtdbtk_database_installer" name="GTDB-Tk Database Installer" tool_type="manage_data" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="@PROFILE@"> + <description></description> + <macros> + <token name="@TOOL_VERSION@">202</token> + <token name="@VERSION_SUFFIX@">0</token> + <token name="@PROFILE@">20.09</token> + </macros> + <requirements> + <requirement type="package" version="3.10.2">python</requirement> + </requirements> + <command> + <![CDATA[ + python '$__tool_directory__/gtdbtk_database_installer.py' + --database_id '$database_id' + --database_name '$database_name' + --url '$url' + --out_file '$out_file' + ]]> + </command> + <inputs> + <param name="database_name" type="text" value="" label="Database name or description" help="This value will be displayed in the GTDB-Tk Database select list"/> + <param name="database_id" type="text" value="" label="Database id" help="This value must be unique with nNo whitespace allowed-use underscores"/> + <param name="url" type="text" value="https://data.gtdb.ecogenomic.org/releases/latest/auxillary_files/gtdbtk_data.tar.gz" label="URL for downloading the selected version of the GTDB-Tk database"/> + </inputs> + <outputs> + <data name="out_file" format="data_manager_json"/> + </outputs> + <tests> + <test> + <!-- Not actually installing a huge GTDB-Tk database --> + <param name="database_id" value="release202"/> + <param name="database_name" value="GTDB-Tk database release 202"/> + <param name="url" value="https://data.gtdb.ecogenomic.org/releases/release202/202.0/VERSION"/> + <output name="out_file"> + <assert_contents> + <has_text text="GTDB-Tk database release 202"/> + <has_text text="release202"/> + </assert_contents> + </output> + </test> + </tests> + <help> + </help> + <citations> + <citation type="doi">doi.org/10.1038/s41587-020-0501-8</citation> + <citation type="doi">dx.doi.org/10.1038/nbt.4229</citation> + </citations> +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,17 @@ +<data_managers> + <data_manager tool_file="data_manager/gtdbtk_database_installer.xml" id="gtdbtk_database_installer"> + <data_table name="gtdbtk_database"> + <output> + <column name="value"/> + <column name="name"/> + <column name="path" output_ref="out_file"> + <move type="directory" relativize_symlinks="True"> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">gtdbtk_database/${value}</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/gtdbtk_database/${value}</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> +</data_managers>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/gtdbtk_database.loc Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,26 @@ +# This is a sample file distributed with Galaxy that enables tools +# to use a directory of GTDB-Tk databases. The gtdbtk_databases.loc +# file has this format (longer white space characters are TAB characters): +# +# <unique_build_id> <display_name> <directory_path> +# +# So, for example, if you have the gtdbtk 202 stored in +# /depot/data2/galaxy/gtdbtk/202/, +# then the gtdbtk_databases.loc entry would look like this: +# +# release202 gtdbtk database release 202 /depot/data2/galaxy/gtdbtk/release202 +# +# and your /depot/data2/galaxy/gtdbtk/release202 directory +# would contain GTDB-Tk database files for release 202, sommething like this: +# +#drwxr-sr-x 3 gvk G-824019 4096 Apr 20 2021 fastani/ +#-rw-r--r-- 1 gvk G-824019 4810764 Apr 22 2021 manifest.tsv +#drwxr-sr-x 4 gvk G-824019 4096 Apr 21 2021 markers/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 masks/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 metadata/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 21 2021 mrca_red/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 msa/ +#drwxr-sr-x 4 gvk G-824019 4096 Apr 21 2021 pplacer/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 radii/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 taxonomy/ +release202 GTDB-Tk database release 202 /depot/data2/galaxy/tool-data/gtdbtk_database/release202
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/gtdbtk_database.loc.sample Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,25 @@ +# This is a sample file distributed with Galaxy that enables tools +# to use a directory of GTDB-Tk databases. The gtdbtk_databases.loc +# file has this format (longer white space characters are TAB characters): +# +# <unique_build_id> <display_name> <directory_path> +# +# So, for example, if you have the gtdbtk 202 stored in +# /depot/data2/galaxy/gtdbtk/202/, +# then the gtdbtk_databases.loc entry would look like this: +# +# release202 gtdbtk database release 202 /depot/data2/galaxy/gtdbtk/release202 +# +# and your /depot/data2/galaxy/gtdbtk/release202 directory +# would contain GTDB-Tk database files for release 202, sommething like this: +# +#drwxr-sr-x 3 gvk G-824019 4096 Apr 20 2021 fastani/ +#-rw-r--r-- 1 gvk G-824019 4810764 Apr 22 2021 manifest.tsv +#drwxr-sr-x 4 gvk G-824019 4096 Apr 21 2021 markers/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 masks/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 metadata/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 21 2021 mrca_red/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 msa/ +#drwxr-sr-x 4 gvk G-824019 4096 Apr 21 2021 pplacer/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 radii/ +#drwxr-sr-x 2 gvk G-824019 4096 Apr 20 2021 taxonomy/
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,7 @@ +<tables> + <!-- Locations of GTDB-Tk database versions 202 and higher --> + <table name="gtdbtk_database" comment_char="#"> + <columns>value, name, path</columns> + <file path="tool-data/gtdbtk_database.loc" /> + </table> +</tables>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.test Thu Dec 08 19:26:49 2022 +0000 @@ -0,0 +1,7 @@ +<tables> + <!-- Location of databases for gtdbtk version 202 and higher --> + <table name="gtdbtk_database" comment_char="#"> + <columns>value, name, path</columns> + <file path="${__HERE__}/test-data/gtdbtk_database.loc" /> + </table> +</tables>