|
Commit message:
Uploaded |
|
added:
data_manager/add_ctat_centrifuge_index.py data_manager/add_ctat_centrifuge_index.xml data_manager_conf.xml tool-data/ctat_centrifuge_indexes.loc.sample tool_data_table_conf.xml.sample |
| b |
| diff -r 000000000000 -r b4d4f0d76e94 data_manager/add_ctat_centrifuge_index.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/add_ctat_centrifuge_index.py Mon Jul 16 20:27:06 2018 -0400 |
| [ |
| b'@@ -0,0 +1,354 @@\n+#!/usr/bin/env python\n+# ref: https://galaxyproject.org/admin/tools/data-managers/how-to/define/\n+\n+# Rewritten by H.E. Cicada Brokaw Dennis from a source downloaded from the toolshed and\n+# other example code on the web.\n+# This allows downloading of a centrifuge index, or specification of its disk location.\n+# This index is one of the input paramters needed by the ctat_metagenomics tool.\n+# At the moment only one index is supported by the ctat_metagenomics tool:\n+# ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed+h+v.tar.gz\n+\n+import argparse\n+import os\n+#import tarfile\n+#import urllib\n+import subprocess\n+\n+# The following is used to generate a unique_id value\n+from datetime import *\n+\n+# Remove the following line when testing without galaxy package:\n+from galaxy.util.json import to_json_string\n+# Am not using the following:\n+# from galaxy.util.json import from_json_string\n+\n+# The FileListParser is used by get_ctat_genome_filenames(),\n+# which is called by the Data Manager interface (.xml file) to get\n+# the filenames that are available online at broadinstitute.org\n+# Not sure best way to do it. \n+# This object uses HTMLParser to look through the html \n+# searching for the filenames within anchor tags.\n+import urllib2\n+from HTMLParser import HTMLParser\n+\n+_CTAT_CentrifugeIndexPage_URL = \'https://ccb.jhu.edu/software/centrifuge/\'\n+_CTAT_CentrifugeDownload_URL = \'ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed+h+v.tar.gz\'\n+_CTAT_CentrifugeIndexTableName = \'ctat_centrifuge_indexes\'\n+_CTAT_CentrifugeDir_Name = \'p_compressed+h+v\'\n+_CTAT_Centrifuge_DisplayNamePrefix = \'CTAT_CentrifugeIndex_\'\n+_CentrifugeIndexFileExtension = \'cf\'\n+_NumBytesNeededForIndex = 7400130287 # 6.9 GB\n+#_DownloadFileSize = 5790678746 # 5.4 Gigabytes.\n+_Download_TestFile = \'write_testfile.txt\'\n+_DownloadSuccessFile = \'download_succeeded.txt\'\n+\n+class FileListParser(HTMLParser):\n+ def __init__(self):\n+ # Have to use direct call to super class rather than using super():\n+ # super(FileListParser, self).__init__()\n+ # because HTMLParser is an "old style" class and its inheritance chain does not include object.\n+ HTMLParser.__init__(self)\n+ self.filenames = set()\n+ def handle_starttag(self, tag, attrs):\n+ # Look for filename references in anchor tags and add them to filenames.\n+ if tag == "a":\n+ # The tag is an anchor tag.\n+ for attribute in attrs:\n+ # print "Checking: {:s}".format(str(attribute))\n+ if attribute[0] == "href":\n+ # Does the href have a tar.gz in it?\n+ if ("tar.gz" in attribute[1]) and ("md5" not in attribute[1]):\n+ # Add the value to filenames.\n+ self.filenames.add(attribute[1]) \n+# End of class FileListParser\n+\n+def get_ctat_centrifuge_index_locations():\n+ # For dynamic options need to return an interable with contents that are tuples with 3 items.\n+ # Item one is a string that is the display name put into the option list.\n+ # Item two is the value that is put into the parameter associated with the option list.\n+ # Item three is a True or False value, indicating whether the item is selected.\n+ options = []\n+ # open the url and retrieve the filenames of the files in the directory.\n+ resource = urllib2.urlopen(_CTAT_CentrifugeIndexPage_URL)\n+ theHTML = resource.read()\n+ filelist_parser = FileListParser()\n+ filelist_parser.feed(theHTML)\n+ # This is what was returned on 2018-04-23\n+ # ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed_2018_4_15.tar.gz\n+ # ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/nt_2018_3_3.tar.gz\n+ # ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed+h+v.tar.gz\n+ # ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p+h+v.tar.gz\n+ # Which could be hard coded:\n+ # options.append(("p_compressed+h+v", "'..b'tory does not exist:\\n\\t{:s}".format(index_directory))\n+ # If args.destination_path is a directory containing \n+ # a subdirectory that contains the index files,\n+ # then we need to set the index_directory to be that subdirectory.\n+ files_in_destination_path = os.listdir(cannonical_destination)\n+ if (len(files_in_destination_path) == 1):\n+ path_to_file = "{:s}/{:s}".format(cannonical_destination, files_in_destination_path[0])\n+ if os.path.isdir(path_to_file):\n+ index_directory = path_to_file\n+ else:\n+ index_directory = cannonical_destination\n+ else:\n+ index_directory = cannonical_destination\n+ # Get the root_index_dirname of the index from the index_directory name.\n+ root_index_dirname = index_directory.split("/")[-1].split(".")[0]\n+\n+ # Check if there is an actual Centrifuge Index file in the index_directory.\n+ print "\\nThe location of the Centrifuge Index is {:s}.\\n".format(index_directory)\n+ files_in_index_directory = set(os.listdir(index_directory))\n+ index_file_found = False\n+ index_file_path = index_directory\n+ for filename in files_in_index_directory:\n+ # The current index is split into 3 files.\n+ # filenames are in the form: index_root_name.#.cf,\n+ # where # is a numeral (1, 2, or 3)\n+ # indicating the order of the files.\n+ if filename.split(".")[-1] == _CentrifugeIndexFileExtension:\n+ index_file_found = True\n+ # The centrifuge program wants the root name of the files to be final part of the path.\n+ index_file_path = "{:s}/{:s}".format(index_directory, filename.split(".")[0])\n+ if not index_file_found:\n+ raise ValueError("Cannot find any Centrifuge Index files.\\n" + \\\n+ "The contents of the directory {:s} are:\\n\\t".format(index_directory) + \\\n+ "\\n\\t".join(files_in_index_directory))\n+\n+ # Set the display_name\n+ if (args.display_name is None) or (args.display_name == ""):\n+ # Use the root_index_dirname.\n+ if (root_index_dirname != None) and (root_index_dirname != ""):\n+ display_name = _CTAT_Centrifuge_DisplayNamePrefix + root_index_dirname\n+ else:\n+ display_name = _CTAT_Centrifuge_DisplayNamePrefix + _CTAT_CentrifugeDir_Name\n+ print "WARNING: Did not set the display name. Using the default: {:s}".format(display_name_value)\n+ else:\n+ display_name = _CTAT_Centrifuge_DisplayNamePrefix + args.display_name\n+ display_name = display_name.replace(" ","_")\n+\n+ # Set the unique_id\n+ datetime_stamp = datetime.now().strftime("_%Y_%m_%d_%H_%M_%S_%f")\n+ if (root_index_dirname != None) and (root_index_dirname != ""):\n+ unique_id = root_index_dirname + datetime_stamp\n+ else:\n+ unique_id = _CTAT_CentrifugeDir_Name + datetime_stamp\n+\n+ print "The Index\'s display_name will be set to: {:s}\\n".format(display_name)\n+ print "Its unique_id will be set to: {:s}\\n".format(unique_id)\n+ print "Its dir_path will be set to: {:s}\\n".format(index_file_path)\n+\n+ data_manager_dict = {}\n+ data_manager_dict[\'data_tables\'] = {}\n+ data_manager_dict[\'data_tables\'][_CTAT_CentrifugeIndexTableName] = []\n+ data_table_entry = dict(value=unique_id, name=display_name, path=index_file_path)\n+ data_manager_dict[\'data_tables\'][_CTAT_CentrifugeIndexTableName].append(data_table_entry)\n+\n+ # Temporarily the output file\'s dictionary is written for debugging:\n+ print "The dictionary for the output file is:\\n\\t{:s}".format(str(data_manager_dict))\n+ # Save info to json file. This is used to transfer data from the DataManager tool, to the data manager,\n+ # which then puts it into the correct .loc file (I think).\n+ # Remove the following line when testing without galaxy package.\n+ open(args.output_filename, \'wb\').write(to_json_string(data_manager_dict))\n+\n+if __name__ == "__main__":\n+ main()\n' |
| b |
| diff -r 000000000000 -r b4d4f0d76e94 data_manager/add_ctat_centrifuge_index.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/add_ctat_centrifuge_index.xml Mon Jul 16 20:27:06 2018 -0400 |
| [ |
| @@ -0,0 +1,89 @@ +<tool id="ctat_centrifuge_indexes_data_manager" + name="CTAT Centrifuge Indexes Data Manager" + version="1.0.0" tool_type="manage_data"> + <description>Retrieve, and/or specify the location of, a CTAT Centrifuge Index. + </description> + <requirements> + <requirement type="package" version="2.7">python</requirement> + </requirements> + <command detect_errors="default"> + <![CDATA[ + python $__tool_directory__/add_ctat_centrifuge_index.py + --display_name "${display_name}" + --destination_path "${destination}" + --output_filename "${out_file}" + #if str( $download_question.download ) == "true": + --download_location "\"${download_question.filename}\"" + #if str( $download_question.force_download ) == "true": + --force_download + #end if + #end if + ]]> + </command> + <inputs> + <!-- The following are left in here, just as examples of various ways of doing options. + <param name="force_download" type="boolean" checked="false" + truevalue="- -force_download" falsevalue="" label="Force New Download? (yes/no)" /> + <param name="download" type="select" label="Need to Download?"> + <option value="single" selected="true">Single Dataset</option> + <option value="paired_collection">Paired Collection</option> + <when value="paired_collection"> + <param name="fastq_input" format="fastqsanger" type="data_collection" collection_type="paired" label="Select dataset pair" help="Specify paired dataset collection containing paired reads"/> + </when> + --> + <conditional name="download_question"> + <param name="download" type="boolean" checked="false" label="Need to Download?" /> + <when value="true"> + <!-- The use of a code block to get dynamic options is now deprecated and discouraged. + I am still using it here. The only other way I can think of to do this is to + create another data_manager that gets the list of files and puts them into a + data_table, that is then used to get the filenames. That would require the admin + to first run the data_manager that builds the filename data_table before running + this data_manager. + However, I have not been able to figure out how to send information back correctly + from the function and there is no documentation that I have found showing how to do it. + <param name="filename" type="select" label="Select File" display="radio" + dynamic_options="get_ctat_centrifuge_index_locations()" + help="Select a Centrifuge Index to Download." /> + Hard coded version. + <param name="filename" type="text" value="ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed+h+v.tar.gz"> + <option value="ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed+h+v.tar.gz"> + p_compressed+h+v + </option> + </param> + --> + <param name="filename" type="select" label="Select File" + dynamic_options="get_ctat_centrifuge_index_locations()" + help="Select a Centrifuge Index to Download." /> + <param name="force_download" type="boolean" checked="false" label="Force New Download?" /> + </when> + </conditional> + + <param name="display_name" type="text" label="Centrifuge Index Display Name" /> + <param name="destination" type="text" label="Local Destination (full path)" /> + </inputs> + <outputs> + <data name="out_file" format="data_manager_json" /> + </outputs> + <help> +.. class:: infomark + +Retrieve, and/or specify the location of, a CTAT Centrifuge Index. +When download is true, Centrifuge index on this FTP link_ will be downloaded. + +Currently that is the only supported index. + +.. class:: infomark + +Specify the Full Path of the destination where the CTAT Reference Library should be placed. +If you already have the index, specify the full path of the location where it exists and leave the download box unchecked. + +.. class:: infomark + +The display_name may be left empty if downloading. +The display_name will be used as the selector text of the entry in the data table. + +.. _link: ftp://ftp.ccb.jhu.edu/pub/infphilo/centrifuge/data/p_compressed+h+v.tar.gz + </help> + <code file="add_ctat_centrifuge_index.py" /> +</tool> |
| b |
| diff -r 000000000000 -r b4d4f0d76e94 data_manager_conf.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Mon Jul 16 20:27:06 2018 -0400 |
| b |
| @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<data_managers> + <data_manager tool_file="data_manager/add_ctat_centrifuge_index.xml" id="ctat_centrifuge_index_data_manager"> + <data_table name="ctat_centrifuge_indexes"> + <output> + <column name="value" /> + <!-- value is used to uniquely identify this entry in the table. + --> + <column name="name" /> + <!-- name is used as the selector in the pull down lists for items in this table. + --> + <column name="path" /> + <!-- path is the absolute path of the directory containing the centrifuge index files. + --> + <!-- Same as the Genome Reference Library, we create the Centrifuge Index files where we want them. + --> + </output> + </data_table> + </data_manager> +</data_managers> |
| b |
| diff -r 000000000000 -r b4d4f0d76e94 tool-data/ctat_centrifuge_indexes.loc.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool-data/ctat_centrifuge_indexes.loc.sample Mon Jul 16 20:27:06 2018 -0400 |
| b |
| @@ -0,0 +1,15 @@ +# This file lists the locations of CTAT Centrifuge Indexes +# Usually there will only be one index, but it is concievable +# that there could be multiple indexes. +# This file format is as follows +# (white space characters are TAB characters): +# +#<value> <name> <path> +# value is a unique id +# name is the display name +# path is the directory where the index files are stored +# +#ctat_centrifuge_indexes.loc could look like: +# +#p_compressed+h+v CTAT_CentrifugeIndex_p_compressed+h+v /path/to/centrifuge/index/p_compressed+h+v +# |
| b |
| diff -r 000000000000 -r b4d4f0d76e94 tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Mon Jul 16 20:27:06 2018 -0400 |
| b |
| @@ -0,0 +1,6 @@ +<tables> + <table name="ctat_centrifuge_indexes" comment_char="#" allow_duplicate_entries="False"> + <columns>value, name, path</columns> + <file path="tool-data/ctat_centrifuge_indexes.loc" /> + </table> +</tables> |