| Previous changeset 38:d27c122c2424 (2016-06-24) Next changeset 40:87a921902f5e (2016-07-08) |
|
Commit message:
planemo upload for repository https://scottx611x@toolshed.g2.bx.psu.edu/repos/scottx611x/data_manager_fetch_gene_annotation |
|
added:
data_manager/data_manager.py data_manager/gene_annotation_fetcher.xml data_manager_conf.xml test-data/gene_annotation_out.json tool-data/gene_annotation.loc.sample tool_data_table_conf.xml.sample tool_dependencies.xml |
|
removed:
data_manager_gene_annotation/data_manager/data_manager.py data_manager_gene_annotation/data_manager/gene_annotation_fetcher.xml data_manager_gene_annotation/data_manager_conf.xml data_manager_gene_annotation/test-data/gene_annotation_out.json data_manager_gene_annotation/tool-data/gene_annotation.loc.sample data_manager_gene_annotation/tool_data_table_conf.xml.sample data_manager_gene_annotation/tool_dependencies.xml |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager/data_manager.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/data_manager.py Fri Jul 08 11:17:50 2016 -0400 |
| [ |
| @@ -0,0 +1,73 @@ +import os +import sys +import uuid +import json +import argparse +import requests +from requests.exceptions import ContentDecodingError + + +def url_download(url): + """Attempt to download gene annotation file from a given url + :param url: full url to gene annotation file + :type url: str. + :returns: name of downloaded gene annotation file + :raises: ContentDecodingError, IOError + """ + response = requests.get(url=url, stream=True) + + # Generate file_name + file_name = response.url.split("/")[-1] + + block_size = 10 * 1024 * 1024 # 10MB chunked download + with open(file_name, 'w+') as f: + try: + # Good to note here that requests' iter_content() will + # automatically handle decoding "gzip" and "deflate" encoding + # formats + for buf in response.iter_content(block_size): + f.write(buf) + except (ContentDecodingError, IOError) as e: + sys.stderr.write("Error occured downloading reference file: %s" + % e) + os.remove(file_name) + + return file_name + + +def main(): + + # Generate and parse command line args + parser = argparse.ArgumentParser(description='Create data manager JSON.') + parser.add_argument('--out', dest='output', action='store', + help='JSON filename') + parser.add_argument('--name', dest='name', action='store', + default=uuid.uuid4(), help='Data table entry unique ID' + ) + parser.add_argument('--url', dest='url', action='store', + help='Url to download gtf file from') + + args = parser.parse_args() + + work_dir = os.getcwd() + + # Attempt to download gene annotation file from given url + gene_annotation_file_name = url_download(args.url) + + # Update Data Manager JSON and write to file + data_manager_entry = { + 'data_tables': { + 'gene_annotation': { + 'value': str(args.name), + 'dbkey': str(args.name), + 'name': gene_annotation_file_name, + 'path': os.path.join(work_dir, gene_annotation_file_name) + } + } + } + + with open(os.path.join(args.output), "w+") as f: + f.write(json.dumps(data_manager_entry)) + +if __name__ == '__main__': + main() |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager/gene_annotation_fetcher.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager/gene_annotation_fetcher.xml Fri Jul 08 11:17:50 2016 -0400 |
| [ |
| @@ -0,0 +1,35 @@ +<?xml version="1.0"?> +<tool id="gene_annotation_fetcher_data_manager" name="Gene Annotation Fetch" tool_type="manage_data" version="1.0.0"> + <description>gene annotation fetcher</description> + <stdio> + <exit_code description="Error" level="fatal" range="1:" /> + </stdio> + <command interpreter="python"> + <![CDATA[ + data_manager.py --out "${out_file}" + #if $gene_annotation_url: + --url "${gene_annotation_url}" + #end if + #if $database_name: + --dbkey "${database_name}" + #end if + ]]> + </command> + <inputs> + <param + help="Enter a unique identifier for the Data Table column to be created, or leave blank for an auto-generated uuid" label="Identifier for this DataTable entry" name="database_name" type="text" optional="True" /> + <param + label="Enter URL pointing to a remote .gtf file" name="gene_annotation_url" type="text" optional="False"/> + </inputs> + <outputs> + <data format="data_manager_json" name="out_file" /> + </outputs> + <tests> + <test> + <param name="database_name" value="cool_name"/> + <param name="gene_annotation_url" value="http://www.scott-ouellette.com/gene_annotations/chr1-hg19_genes.gtf"/> + <output name="out_file" file="gene_annotation_out.json" compare="contains"/> + <!--<extra_files type="file" name="chr1-hg19_genes.gtf" value="chr1-hg19_genes.gtf"/>--> + </test> + </tests> +</tool> |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_conf.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/data_manager_conf.xml Fri Jul 08 11:17:50 2016 -0400 |
| b |
| @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<data_managers> + + <data_manager tool_file="data_manager/gene_annotation_fetcher.xml" id="gene_annotation_fetcher" version="1.0.0"> + <data_table name="gene_annotation"> + <output> + <column name="value" /> + <column name="dbkey" /> + <column name="name" /> + <column name="path" output_ref="out_file"> + <move type="file"> + <source>${path}</source> + <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">gene_annotations/${name}</target> + </move> + <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/gene_annotations/${name}</value_translation> + <value_translation type="function">abspath</value_translation> + </column> + </output> + </data_table> + </data_manager> + +</data_managers> |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_gene_annotation/data_manager/data_manager.py --- a/data_manager_gene_annotation/data_manager/data_manager.py Fri Jun 24 09:25:53 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,74 +0,0 @@ -import os -import sys -import uuid -import json -import argparse -import datetime -import requests -from requests.exceptions import ContentDecodingError - - -def url_download(url): - """Attempt to download gene annotation file from a given url - :param url: NodeSet UUID. - :type url: str. - :returns: name of downloaded gene annotation file - :raises: ContentDecodingError, IOError - """ - response = requests.get(url=url, stream=True) - - # Generate file_name - file_name = response.url.split("/")[-1] - - block_size = 10 * 1024 * 1024 # 10MB chunked download - with open(file_name, 'w+') as f: - try: - # Good to note here that requests' iter_content() will - # automatically handle decoding "gzip" and "deflate" encoding - # formats - for buf in response.iter_content(block_size): - f.write(buf) - except (ContentDecodingError, IOError) as e: - sys.stderr.write("Error occured downloading reference file: %s" - % e) - os.remove(file_name) - - return file_name - - -def main(): - - # Generate and parse command line args - parser = argparse.ArgumentParser(description='Create data manager JSON.') - parser.add_argument('--out', dest='output', action='store', - help='JSON filename') - parser.add_argument('--name', dest='name', action='store', - default=uuid.uuid4(), help='Data table entry unique ID' - ) - parser.add_argument('--url', dest='url', action='store', - help='Url to download gtf file from') - - args = parser.parse_args() - - work_dir = os.getcwd() - - # Attempt to download gene annotation file from given url - gene_annotation_file_name = url_download(args.url) - - # Update Data Manager JSON and write to file - data_manager_entry = { - 'data_tables': { - 'gene_annotation': { - 'value': str(datetime.datetime.now()), - 'dbkey': str(args.name), - 'name': gene_annotation_file_name, - 'path': os.path.join(work_dir, gene_annotation_file_name) - } - } - } - - with open(os.path.join(args.output), "w+") as f: - f.write(json.dumps(data_manager_entry)) - -if __name__ == '__main__': - main() |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_gene_annotation/data_manager/gene_annotation_fetcher.xml --- a/data_manager_gene_annotation/data_manager/gene_annotation_fetcher.xml Fri Jun 24 09:25:53 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| [ |
| @@ -1,33 +0,0 @@ -<?xml version="1.0"?> -<tool id="gene_annotation_fetcher_data_manager" name="Gene Annotation Fetch" tool_type="manage_data" version="1.0.0"> - <description>gene annotation fetcher</description> - <stdio> - <exit_code description="Error" level="fatal" range="1:" /> - </stdio> - <command interpreter="python"> - <![CDATA[ - data_manager.py --out "${out_file}" - #if $gene_annotation_url: - --url "${gene_annotation_url}" - #end if - #if $database_name: - --name "${database_name}" - #end if - ]]> - </command> - <inputs> - <param help="Enter a unique identifier, or leave blank for an auto-generated uuid" label="Name for this database" name="database_name" type="text" optional="True" /> - <param label="Enter URL for gene annotation files" name="gene_annotation_url" type="text" optional="False"/> - </inputs> - <outputs> - <data format="data_manager_json" name="out_file" /> - </outputs> - <tests> - <test> - <param name="database_name" value="cool_name"/> - <param name="gene_annotation_url" value="http://www.scott-ouellette.com/gene_annotations/chr1-hg19_genes.gtf"/> - <output name="out_file" file="../test-data/gene_annotation_out.json" compare="contains"/> - <extra_files type="file" name="chr1-hg19_genes.gtf" value="chr1-hg19_genes.gtf"/> - </test> - </tests> -</tool> |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_gene_annotation/data_manager_conf.xml --- a/data_manager_gene_annotation/data_manager_conf.xml Fri Jun 24 09:25:53 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,22 +0,0 @@ -<?xml version="1.0"?> -<data_managers> - - <data_manager tool_file="data_manager/gene_annotation_fetcher.xml" id="gene_annotation_fetcher" version="1.0.0"> - <data_table name="gene_annotation"> - <output> - <column name="value" /> - <column name="dbkey" /> - <column name="name" /> - <column name="path" output_ref="out_file"> - <move type="file"> - <source>${path}</source> - <target base="${GALAXY_DATA_MANAGER_DATA_PATH}">gene_annotations/${name}</target> - </move> - <value_translation>${GALAXY_DATA_MANAGER_DATA_PATH}/gene_annotations/${name}</value_translation> - <value_translation type="function">abspath</value_translation> - </column> - </output> - </data_table> - </data_manager> - -</data_managers> |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_gene_annotation/test-data/gene_annotation_out.json --- a/data_manager_gene_annotation/test-data/gene_annotation_out.json Fri Jun 24 09:25:53 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,1 +0,0 @@ -{"data_tables": {"gene_annotation": {"path": "/Users/scott/PyCharmProjects/Galaxy/galaxy/database/job_working_directory/003/3702/chr1-hg19_genes.gtf", "dbkey": "cool_name", "value": "2016-06-23 17:19:39.412249", "name": "chr1-hg19_genes.gtf"}}} \ No newline at end of file |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_gene_annotation/tool_data_table_conf.xml.sample --- a/data_manager_gene_annotation/tool_data_table_conf.xml.sample Fri Jun 24 09:25:53 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,8 +0,0 @@ -<?xml version="1.0"?> -<tables> - <!-- Locations of gene annotation data --> - <table name="gene_annotation" comment_char="#"> - <columns>value, dbkey, name, path</columns> - <file path="tool-data/gene_annotation.loc" /> - </table> -</tables> \ No newline at end of file |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df data_manager_gene_annotation/tool_dependencies.xml --- a/data_manager_gene_annotation/tool_dependencies.xml Fri Jun 24 09:25:53 2016 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,3 +0,0 @@ -<?xml version="1.0"?> -<tool_dependency> -</tool_dependency> |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df test-data/gene_annotation_out.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/gene_annotation_out.json Fri Jul 08 11:17:50 2016 -0400 |
| b |
| @@ -0,0 +1,1 @@ +{"data_tables": {"gene_annotation": {"path": "/Users/scott/PyCharmProjects/Galaxy/galaxy/database/job_working_directory/003/3702/chr1-hg19_genes.gtf", "dbkey": "cool_name", "value": "2016-06-23 17:19:39.412249", "name": "chr1-hg19_genes.gtf"}}} \ No newline at end of file |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df tool_data_table_conf.xml.sample --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.sample Fri Jul 08 11:17:50 2016 -0400 |
| b |
| @@ -0,0 +1,8 @@ +<?xml version="1.0"?> +<tables> + <!-- Locations of gene annotation data --> + <table name="gene_annotation" comment_char="#"> + <columns>value, dbkey, name, path</columns> + <file path="tool-data/gene_annotation.loc" /> + </table> +</tables> \ No newline at end of file |
| b |
| diff -r d27c122c2424 -r 2ab076cf69df tool_dependencies.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Fri Jul 08 11:17:50 2016 -0400 |
| b |
| @@ -0,0 +1,3 @@ +<?xml version="1.0"?> +<tool_dependency> +</tool_dependency> |