| Previous changeset 0:0a3a6f862104 (2015-10-14) Next changeset 2:e0329ab30f6d (2019-11-28) |
|
Commit message:
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f |
|
modified:
data_manager/data_manager_rsync.py data_manager/data_manager_rsync.xml |
| b |
| diff -r 0a3a6f862104 -r 8ff92bd7e2a3 data_manager/data_manager_rsync.py --- a/data_manager/data_manager_rsync.py Wed Oct 14 13:48:12 2015 -0400 +++ b/data_manager/data_manager_rsync.py Tue Apr 04 18:13:26 2017 -0400 |
| [ |
| b'@@ -1,19 +1,27 @@\n #!/usr/bin/env python\n-#Dan Blankenberg\n+# Dan Blankenberg\n+from __future__ import print_function\n \n-import sys\n-import os\n-import tempfile\n-import shutil\n+import datetime\n+import logging\n import optparse\n-import urllib2\n+import os\n+import shutil\n import subprocess\n-import datetime\n+import tempfile\n+from json import (\n+ dumps,\n+ loads\n+)\n from os.path import basename\n-from json import loads, dumps\n from xml.etree.ElementTree import tostring\n+try:\n+ # For Python 3.0 and later\n+ from urllib.request import urlopen\n+except ImportError:\n+ # Fall back to Python 2 imports\n+ from urllib2 import urlopen\n \n-import logging\n _log_name = __name__\n if _log_name == \'__builtin__\':\n _log_name = \'toolshed.installed.g2.rsync.data.manager\'\n@@ -27,8 +35,8 @@\n \n # Pull the Tool Data Table files from github\n # FIXME: These files should be accessible from the rsync server directly.\n-TOOL_DATA_TABLE_CONF_XML_URLS = { \'main\':"https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/usegalaxy.org/config/tool_data_table_conf.xml",\n- \'test\':"https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/test.galaxyproject.org/config/tool_data_table_conf.xml" }\n+TOOL_DATA_TABLE_CONF_XML_URLS = {\'main\': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/usegalaxy.org/config/tool_data_table_conf.xml",\n+ \'test\': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/test.galaxyproject.org/config/tool_data_table_conf.xml" }\n \n # Replace data table source entries with local temporary location\n GALAXY_DATA_CANONICAL_PATH = "/galaxy/data/"\n@@ -50,8 +58,8 @@\n # e.g. mafs. Although this maf table is goofy and doesn\'t have path defined in <table> def,\n # but it does exit in the .loc.\n \n+\n # --- These methods are called by/within the Galaxy Application\n-\n def exec_before_job( app, inp_data, out_data, param_dict, tool=None, **kwd ):\n # Look for any data tables that haven\'t been defined for this data manager before and dynamically add them to Galaxy\n param_dict = dict( **param_dict )\n@@ -67,8 +75,8 @@\n data_manager = app.data_managers.get_manager( tool.data_manager_id, None )\n data_table_entries = get_data_table_entries( param_dict )\n data_tables = load_data_tables_from_url( data_table_class=app.tool_data_tables.__class__ ).get( \'data_tables\' )\n- for data_table_name, entries in data_table_entries.iteritems():\n- #get data table managed by this data Manager\n+ for data_table_name, entries in data_table_entries.items():\n+ # get data table managed by this data Manager\n has_data_table = app.tool_data_tables.get_tables().get( data_table_name )\n if has_data_table:\n has_data_table = bool( has_data_table.get_filename_for_source( data_manager, None ) )\n@@ -77,7 +85,7 @@\n from tool_shed.tools import data_table_manager\n tdtm = data_table_manager.ToolDataTableManager( app )\n target_dir, tool_path, relative_target_dir = tdtm.get_target_install_dir( tool_shed_repository )\n- #Dynamically add this data table\n+ # Dynamically add this data table\n log.debug( "Attempting to dynamically create a missing Tool Data Table named %s." % data_table_name )\n data_table = data_tables[data_table_name]\n repo_info = tdtm.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None )\n@@ -89,21 +97,23 @@\n app.tool_data_tables.add_new_entries_from_config_file( tmp_file.name, None, app.config.shed_tool_data_table_config, persist=True )\n tmp_file.close()\n \n+\n def galaxy_code_get_available_data_tables( trans ):\n- #list of data tables\n+ # list of data tables\n found_tables = get_available_tables( trans )\n- rval = map( lambda x: ( ( x, x, DEFAULT_SELECTE'..b'PATH ):]\n@@ -301,14 +318,14 @@\n rsync_source = rsync_source[:-1]\n try:\n dir_list = rsync_list_dir( rsync_source + "/" )\n- except Exception, e:\n+ except Exception:\n dir_list = None\n while not dir_list or \'.\' not in dir_list:\n head, tail = os.path.split( make_path )\n if not head:\n head = tail\n make_path = head\n- rsync_source = rsync_urljoin( rsync_urljoin( RSYNC_SERVER, INDEX_DIR ), head ) #if we error here, likely due to a connection issue\n+ rsync_source = rsync_urljoin( rsync_urljoin( RSYNC_SERVER, INDEX_DIR ), head ) # if we error here, likely due to a connection issue\n if rsync_source.endswith( \'/\' ):\n rsync_source = rsync_source[:-1]\n dir_list = rsync_list_dir( rsync_source + "/" )\n@@ -321,43 +338,45 @@\n rsync_sync_to_dir( rsync_source, target_path )\n return path\n \n+\n def get_data_and_munge_path( data_table_name, data_table_entry, data_root_dir ):\n path_cols = []\n- for key, value in data_table_entry.iteritems():\n+ for key, value in data_table_entry.items():\n if key in PATH_COLUMN_NAMES:\n path_cols.append( ( key, value ) )\n- found_data = False\n if path_cols:\n for col_name, value in path_cols:\n- #GALAXY_DATA_CANONICAL_PATH\n if value.startswith( GALAXY_DATA_CANONICAL_PATH ):\n data_table_entry[col_name] = get_data_for_path( value, data_root_dir )\n- found_data = True\n else:\n- print \'unable to determine location of rsync data for\', data_table_name, data_table_entry\n+ print(\'unable to determine location of rsync data for\', data_table_name, data_table_entry)\n return data_table_entry\n \n+\n def fulfill_data_table_entries( data_table_entries, data_manager_dict, data_root_dir ):\n- for data_table_name, entries in data_table_entries.iteritems():\n+ for data_table_name, entries in data_table_entries.items():\n for entry in entries:\n entry = get_data_and_munge_path( data_table_name, entry, data_root_dir )\n _add_data_table_entry( data_manager_dict, data_table_name, entry )\n return data_manager_dict\n \n+\n def _add_data_table_entry( data_manager_dict, data_table_name, data_table_entry ):\n data_manager_dict[\'data_tables\'] = data_manager_dict.get( \'data_tables\', {} )\n data_manager_dict[\'data_tables\'][data_table_name] = data_manager_dict[\'data_tables\'].get( data_table_name, [] )\n data_manager_dict[\'data_tables\'][data_table_name].append( data_table_entry )\n return data_manager_dict\n \n+\n def cleanup_before_exit( tmp_dir ):\n if tmp_dir and os.path.exists( tmp_dir ):\n shutil.rmtree( tmp_dir )\n \n+\n def get_data_table_entries( params ):\n rval = {}\n data_table_entries = params.get( \'data_table_entries\', None )\n- if data_table_entries :\n+ if data_table_entries:\n for entry_text in data_table_entries.split( \',\' ):\n entry_text = entry_text.strip().decode( \'base64\' )\n entry_dict = loads( entry_text )\n@@ -367,24 +386,26 @@\n rval[ data_table_name ].append( data_table_entry )\n return rval\n \n+\n def main():\n- #Parse Command Line\n parser = optparse.OptionParser()\n (options, args) = parser.parse_args()\n- \n+\n filename = args[0]\n- \n+\n params = loads( open( filename ).read() )\n target_directory = params[ \'output_data\' ][0][\'extra_files_path\']\n os.mkdir( target_directory )\n data_manager_dict = {}\n- \n+\n data_table_entries = get_data_table_entries( params[\'param_dict\'] )\n- \n+\n # Populate the data Tables\n data_manager_dict = fulfill_data_table_entries( data_table_entries, data_manager_dict, target_directory )\n- \n- #save info to json file\n+\n+ # save info to json file\n open( filename, \'wb\' ).write( dumps( data_manager_dict ) )\n- \n-if __name__ == "__main__": main()\n+\n+\n+if __name__ == "__main__":\n+ main()\n' |
| b |
| diff -r 0a3a6f862104 -r 8ff92bd7e2a3 data_manager/data_manager_rsync.xml --- a/data_manager/data_manager_rsync.xml Wed Oct 14 13:48:12 2015 -0400 +++ b/data_manager/data_manager_rsync.xml Tue Apr 04 18:13:26 2017 -0400 |
| [ |
| @@ -1,23 +1,18 @@ <tool id="data_manager_rsync_g2" name="Rsync with g2" version="0.0.1" tool_type="manage_data"> <options sanitize="False" /> <description>fetching</description> - <command interpreter="python">data_manager_rsync.py "${out_file}"</command> - <stdio> - <exit_code range="1:" err_level="fatal" /> - <exit_code range=":-1" err_level="fatal" /> - </stdio> + <command detect_errors="exit_code"><![CDATA[ + python '$__tool_directory__/data_manager_rsync.py' + '${out_file}' + ]]></command> <inputs> - <param name="dbkey" type="genomebuild" label="dbkey to search for Reference Data" help="Specify ? to show all"/> - <param name="data_table_names" type="select" display="checkboxes" multiple="True" optional="True" - label="Choose Desired Data Tables" dynamic_options="galaxy_code_get_available_data_tables( __trans__ )" - refresh_on_change="dbkey"/> - - - <param name="data_table_entries" type="select" display="checkboxes" multiple="True" optional="False" - label="Choose Desired Data Tables Entries" dynamic_options="galaxy_code_get_available_data_tables_entries( __trans__, dbkey, data_table_names )" - refresh_on_change="dbkey"/> + <param name="data_table_names" type="select" display="checkboxes" multiple="True" optional="True" refresh_on_change="dbkey" + label="Choose Desired Data Tables" dynamic_options="galaxy_code_get_available_data_tables( __trans__ )" /> + + <param name="data_table_entries" type="select" display="checkboxes" multiple="true" optional="false" refresh_on_change="dbkey" + label="Choose Desired Data Tables Entries" dynamic_options="galaxy_code_get_available_data_tables_entries( __trans__, dbkey, data_table_names )" /> </inputs> <outputs> <data name="out_file" format="data_manager_json" dbkey="dbkey"/> @@ -37,12 +32,9 @@ ------ - - .. class:: infomark -**Notice:** If you do not have a particular data table defined, then it will be created and persisted dynamically. - +**Notice:** If you do not have a particular data table defined, then it will be created and persisted dynamically. </help> <code file="data_manager_rsync.py" /> </tool> |