| Previous changeset 1:8ff92bd7e2a3 (2017-04-04) |
|
Commit message:
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 57f71aa633a43ab02bbf05acd0c6d7f406e01f1e" |
|
modified:
data_manager/data_manager_rsync.py data_manager/data_manager_rsync.xml test-data/sacCer2_rsync_all_fasta.data_manager_json |
|
added:
test-data/all_fasta.loc tool_data_table_conf.xml.test |
| b |
| diff -r 8ff92bd7e2a3 -r e0329ab30f6d data_manager/data_manager_rsync.py --- a/data_manager/data_manager_rsync.py Tue Apr 04 18:13:26 2017 -0400 +++ b/data_manager/data_manager_rsync.py Thu Nov 28 15:47:47 2019 -0500 |
| [ |
| b'@@ -2,6 +2,7 @@\n # Dan Blankenberg\n from __future__ import print_function\n \n+import base64\n import datetime\n import logging\n import optparse\n@@ -25,7 +26,7 @@\n _log_name = __name__\n if _log_name == \'__builtin__\':\n _log_name = \'toolshed.installed.g2.rsync.data.manager\'\n-log = logging.getLogger( _log_name )\n+log = logging.getLogger(_log_name)\n \n # Get the Data from the Galaxy Project rsync server\n RSYNC_CMD = \'rsync\'\n@@ -35,16 +36,16 @@\n \n # Pull the Tool Data Table files from github\n # FIXME: These files should be accessible from the rsync server directly.\n-TOOL_DATA_TABLE_CONF_XML_URLS = {\'main\': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/usegalaxy.org/config/tool_data_table_conf.xml",\n- \'test\': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/test.galaxyproject.org/config/tool_data_table_conf.xml" }\n+TOOL_DATA_TABLE_CONF_XML_URLS = {\'main\': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/env/main/files/galaxy/config/tool_data_table_conf.xml",\n+ \'test\': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/env/test/files/galaxy/config/tool_data_table_conf.xml"}\n \n # Replace data table source entries with local temporary location\n GALAXY_DATA_CANONICAL_PATH = "/galaxy/data/"\n-TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE = \'<file path="%slocation/\' % ( GALAXY_DATA_CANONICAL_PATH )\n+TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE = \'<file path="%slocation/\' % (GALAXY_DATA_CANONICAL_PATH)\n TOOL_DATA_TABLE_CONF_XML_REPLACE_TARGET = \'<file path="%s/\'\n \n # Some basic Caching, so we don\'t have to reload and download everything every time\n-CACHE_TIME = datetime.timedelta( minutes=10 )\n+CACHE_TIME = datetime.timedelta(minutes=10)\n TOOL_DATA_TABLES_LOADED_BY_URL = {}\n \n # Entries will not be selected by default\n@@ -60,179 +61,179 @@\n \n \n # --- These methods are called by/within the Galaxy Application\n-def exec_before_job( app, inp_data, out_data, param_dict, tool=None, **kwd ):\n+def exec_before_job(app, inp_data, out_data, param_dict, tool=None, **kwd):\n # Look for any data tables that haven\'t been defined for this data manager before and dynamically add them to Galaxy\n- param_dict = dict( **param_dict )\n- param_dict[\'data_table_entries\'] = param_dict.get( \'data_table_entries\', [] )\n- if not isinstance( param_dict[\'data_table_entries\'], list ):\n+ param_dict = dict(**param_dict)\n+ param_dict[\'data_table_entries\'] = param_dict.get(\'data_table_entries\', [])\n+ if not isinstance(param_dict[\'data_table_entries\'], list):\n param_dict[\'data_table_entries\'] = [param_dict[\'data_table_entries\']]\n- param_dict[\'data_table_entries\'] = ",".join( param_dict[\'data_table_entries\'] )\n+ param_dict[\'data_table_entries\'] = ",".join(param_dict[\'data_table_entries\'])\n if tool:\n tool_shed_repository = tool.tool_shed_repository\n else:\n tool_shed_repository = None\n tdtm = None\n- data_manager = app.data_managers.get_manager( tool.data_manager_id, None )\n- data_table_entries = get_data_table_entries( param_dict )\n- data_tables = load_data_tables_from_url( data_table_class=app.tool_data_tables.__class__ ).get( \'data_tables\' )\n+ data_manager = app.data_managers.get_manager(tool.data_manager_id, None)\n+ data_table_entries = get_data_table_entries(param_dict)\n+ data_tables = load_data_tables_from_url(data_table_class=app.tool_data_tables.__class__).get(\'data_tables\')\n for data_table_name, entries in data_table_entries.items():\n # get data table managed by this data Manager\n- has_data_table = app.tool_data_tables.get_tables().get( data_table_name )\n+ has_data_table = app.tool_data_tables.get_tables().get(str(data_table_name))\n if has_data_table:\n- has_data_table = bool( has_data_table.get_filename_for_source( data_manager, None ) )\n+ has_data_table = b'..b'ONICAL_PATH ):\n- data_table_entry[col_name] = get_data_for_path( value, data_root_dir )\n+ if value.startswith(GALAXY_DATA_CANONICAL_PATH):\n+ data_table_entry[col_name] = get_data_for_path(value, data_root_dir)\n else:\n print(\'unable to determine location of rsync data for\', data_table_name, data_table_entry)\n return data_table_entry\n \n \n-def fulfill_data_table_entries( data_table_entries, data_manager_dict, data_root_dir ):\n+def fulfill_data_table_entries(data_table_entries, data_manager_dict, data_root_dir):\n for data_table_name, entries in data_table_entries.items():\n for entry in entries:\n- entry = get_data_and_munge_path( data_table_name, entry, data_root_dir )\n- _add_data_table_entry( data_manager_dict, data_table_name, entry )\n+ entry = get_data_and_munge_path(data_table_name, entry, data_root_dir)\n+ _add_data_table_entry(data_manager_dict, data_table_name, entry)\n return data_manager_dict\n \n \n-def _add_data_table_entry( data_manager_dict, data_table_name, data_table_entry ):\n- data_manager_dict[\'data_tables\'] = data_manager_dict.get( \'data_tables\', {} )\n- data_manager_dict[\'data_tables\'][data_table_name] = data_manager_dict[\'data_tables\'].get( data_table_name, [] )\n- data_manager_dict[\'data_tables\'][data_table_name].append( data_table_entry )\n+def _add_data_table_entry(data_manager_dict, data_table_name, data_table_entry):\n+ data_manager_dict[\'data_tables\'] = data_manager_dict.get(\'data_tables\', {})\n+ data_manager_dict[\'data_tables\'][data_table_name] = data_manager_dict[\'data_tables\'].get(data_table_name, [])\n+ data_manager_dict[\'data_tables\'][data_table_name].append(data_table_entry)\n return data_manager_dict\n \n \n-def cleanup_before_exit( tmp_dir ):\n- if tmp_dir and os.path.exists( tmp_dir ):\n- shutil.rmtree( tmp_dir )\n+def cleanup_before_exit(tmp_dir):\n+ if tmp_dir and os.path.exists(tmp_dir):\n+ shutil.rmtree(tmp_dir)\n \n \n-def get_data_table_entries( params ):\n+def get_data_table_entries(params):\n rval = {}\n- data_table_entries = params.get( \'data_table_entries\', None )\n+ data_table_entries = params.get(\'data_table_entries\', None)\n if data_table_entries:\n- for entry_text in data_table_entries.split( \',\' ):\n- entry_text = entry_text.strip().decode( \'base64\' )\n- entry_dict = loads( entry_text )\n+ for entry_text in data_table_entries.split(\',\'):\n+ entry_text = base64.b64decode(entry_text.strip().encode(\'utf-8\'))\n+ entry_dict = loads(entry_text)\n data_table_name = entry_dict[\'name\']\n data_table_entry = entry_dict[\'entry\']\n- rval[ data_table_name ] = rval.get( data_table_name, [] )\n- rval[ data_table_name ].append( data_table_entry )\n+ rval[data_table_name] = rval.get(data_table_name, [])\n+ rval[data_table_name].append(data_table_entry)\n return rval\n \n \n@@ -393,18 +394,18 @@\n \n filename = args[0]\n \n- params = loads( open( filename ).read() )\n- target_directory = params[ \'output_data\' ][0][\'extra_files_path\']\n- os.mkdir( target_directory )\n+ params = loads(open(filename).read())\n+ target_directory = params[\'output_data\'][0][\'extra_files_path\']\n+ os.mkdir(target_directory)\n data_manager_dict = {}\n \n- data_table_entries = get_data_table_entries( params[\'param_dict\'] )\n+ data_table_entries = get_data_table_entries(params[\'param_dict\'])\n \n # Populate the data Tables\n- data_manager_dict = fulfill_data_table_entries( data_table_entries, data_manager_dict, target_directory )\n+ data_manager_dict = fulfill_data_table_entries(data_table_entries, data_manager_dict, target_directory)\n \n # save info to json file\n- open( filename, \'wb\' ).write( dumps( data_manager_dict ) )\n+ open(filename, \'w\').write(dumps(data_manager_dict, sort_keys=True))\n \n \n if __name__ == "__main__":\n' |
| b |
| diff -r 8ff92bd7e2a3 -r e0329ab30f6d data_manager/data_manager_rsync.xml --- a/data_manager/data_manager_rsync.xml Tue Apr 04 18:13:26 2017 -0400 +++ b/data_manager/data_manager_rsync.xml Thu Nov 28 15:47:47 2019 -0500 |
| [ |
| @@ -1,17 +1,21 @@ -<tool id="data_manager_rsync_g2" name="Rsync with g2" version="0.0.1" tool_type="manage_data"> +<tool id="data_manager_rsync_g2" name="Rsync with g2" version="0.0.3" tool_type="manage_data" profile="19.05"> <options sanitize="False" /> <description>fetching</description> + <requirements> + <requirement type="package" version="3.7">python</requirement> + <requirement type="package" version="3.1.3">rsync</requirement> + </requirements> <command detect_errors="exit_code"><![CDATA[ python '$__tool_directory__/data_manager_rsync.py' '${out_file}' ]]></command> <inputs> - <param name="dbkey" type="genomebuild" label="dbkey to search for Reference Data" help="Specify ? to show all"/> + <param name="dbkey" type="genomebuild" label="dbkey to search for Reference Data" help="Specify ? to show all" refresh_on_change="True"/> - <param name="data_table_names" type="select" display="checkboxes" multiple="True" optional="True" refresh_on_change="dbkey" + <param name="data_table_names" type="select" display="checkboxes" multiple="True" optional="True" refresh_on_change="True" label="Choose Desired Data Tables" dynamic_options="galaxy_code_get_available_data_tables( __trans__ )" /> - <param name="data_table_entries" type="select" display="checkboxes" multiple="true" optional="false" refresh_on_change="dbkey" + <param name="data_table_entries" type="select" display="checkboxes" multiple="true" optional="false" label="Choose Desired Data Tables Entries" dynamic_options="galaxy_code_get_available_data_tables_entries( __trans__, dbkey, data_table_names )" /> </inputs> <outputs> @@ -21,7 +25,7 @@ <test> <param name="dbkey" value="sacCer2"/> <param name="data_table_names" value="all_fasta"/> - <param name="data_table_entries" value="eyJlbnRyeSI6IHsicGF0aCI6ICIvZ2FsYXh5L2RhdGEvc2FjQ2VyMi9zZXEvc2FjQ2VyMi5mYSIs ICJkYmtleSI6ICJzYWNDZXIyIiwgInZhbHVlIjogInNhY0NlcjIiLCAibmFtZSI6ICJZZWFzdCAo U2FjY2hhcm9teWNlcyBjZXJldmlzaWFlKTogc2FjQ2VyMiJ9LCAibmFtZSI6ICJhbGxfZmFzdGEi fQ=="/> + <param name="data_table_entries" value="eyJlbnRyeSI6IHsiZGJrZXkiOiAic2FjQ2VyMiIsICJuYW1lIjogIlllYXN0IChTYWNjaGFyb215Y2VzIGNlcmV2aXNpYWUpOiBzYWNDZXIyIiwgInBhdGgiOiAiL2N2bWZzL2RhdGEuZ2FsYXh5cHJvamVjdC5vcmcvYnloYW5kL3NhY0NlcjIvc2VxL3NhY0NlcjIuZmEiLCAidmFsdWUiOiAic2FjQ2VyMiJ9LCAibmFtZSI6ICJhbGxfZmFzdGEifQ=="/> <output name="out_file" file="sacCer2_rsync_all_fasta.data_manager_json"/> </test> </tests> |
| b |
| diff -r 8ff92bd7e2a3 -r e0329ab30f6d test-data/sacCer2_rsync_all_fasta.data_manager_json --- a/test-data/sacCer2_rsync_all_fasta.data_manager_json Tue Apr 04 18:13:26 2017 -0400 +++ b/test-data/sacCer2_rsync_all_fasta.data_manager_json Thu Nov 28 15:47:47 2019 -0500 |
| [ |
| @@ -1,1 +1,1 @@ -{"data_tables": {"all_fasta": [{"path": "sacCer2/seq/sacCer2.fa", "value": "sacCer2", "dbkey": "sacCer2", "name": "Yeast (Saccharomyces cerevisiae): sacCer2"}]}} \ No newline at end of file +{"data_tables": {"all_fasta": [{"dbkey": "sacCer2", "name": "Yeast (Saccharomyces cerevisiae): sacCer2", "path": "/cvmfs/data.galaxyproject.org/byhand/sacCer2/seq/sacCer2.fa", "value": "sacCer2"}]}} \ No newline at end of file |
| b |
| diff -r 8ff92bd7e2a3 -r e0329ab30f6d tool_data_table_conf.xml.test --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_data_table_conf.xml.test Thu Nov 28 15:47:47 2019 -0500 |
| b |
| @@ -0,0 +1,8 @@ +<!-- Use the file tool_data_table_conf.xml.oldlocstyle if you don't want to update your loc files as changed in revision 4550:535d276c92bc--> +<tables> + <!-- Locations of all fasta files under genome directory --> + <table name="all_fasta" comment_char="#"> + <columns>value, dbkey, name, path</columns> + <file path="${__HERE__}/test-data/all_fasta.loc" /> + </table> +</tables> |