changeset 1:8ff92bd7e2a3 draft

planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_rsync_g2 commit 8652f36a3a3838dca989426961561e81432acf4f
author iuc
date Tue, 04 Apr 2017 18:13:26 -0400
parents 0a3a6f862104
children e0329ab30f6d
files data_manager/data_manager_rsync.py data_manager/data_manager_rsync.xml
diffstat 2 files changed, 91 insertions(+), 78 deletions(-) [+]
line wrap: on
line diff
--- a/data_manager/data_manager_rsync.py	Wed Oct 14 13:48:12 2015 -0400
+++ b/data_manager/data_manager_rsync.py	Tue Apr 04 18:13:26 2017 -0400
@@ -1,19 +1,27 @@
 #!/usr/bin/env python
-#Dan Blankenberg
+# Dan Blankenberg
+from __future__ import print_function
 
-import sys
-import os
-import tempfile
-import shutil
+import datetime
+import logging
 import optparse
-import urllib2
+import os
+import shutil
 import subprocess
-import datetime
+import tempfile
+from json import (
+    dumps,
+    loads
+)
 from os.path import basename
-from json import loads, dumps
 from xml.etree.ElementTree import tostring
+try:
+    # For Python 3.0 and later
+    from urllib.request import urlopen
+except ImportError:
+    # Fall back to Python 2 imports
+    from urllib2 import urlopen
 
-import logging
 _log_name = __name__
 if _log_name == '__builtin__':
     _log_name = 'toolshed.installed.g2.rsync.data.manager'
@@ -27,8 +35,8 @@
 
 # Pull the Tool Data Table files from github
 # FIXME: These files should be accessible from the rsync server directly.
-TOOL_DATA_TABLE_CONF_XML_URLS = { 'main':"https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/usegalaxy.org/config/tool_data_table_conf.xml",
-                                  'test':"https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/test.galaxyproject.org/config/tool_data_table_conf.xml" }
+TOOL_DATA_TABLE_CONF_XML_URLS = {'main': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/usegalaxy.org/config/tool_data_table_conf.xml",
+                                 'test': "https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/master/files/galaxy/test.galaxyproject.org/config/tool_data_table_conf.xml" }
 
 # Replace data table source entries with local temporary location
 GALAXY_DATA_CANONICAL_PATH = "/galaxy/data/"
@@ -50,8 +58,8 @@
 # e.g. mafs. Although this maf table is goofy and doesn't have path defined in <table> def,
 # but it does exit in the .loc.
 
+
 # --- These methods are called by/within the Galaxy Application
-
 def exec_before_job( app, inp_data, out_data, param_dict, tool=None, **kwd ):
     # Look for any data tables that haven't been defined for this data manager before and dynamically add them to Galaxy
     param_dict = dict( **param_dict )
@@ -67,8 +75,8 @@
     data_manager = app.data_managers.get_manager( tool.data_manager_id, None )
     data_table_entries = get_data_table_entries( param_dict )
     data_tables = load_data_tables_from_url( data_table_class=app.tool_data_tables.__class__ ).get( 'data_tables' )
-    for data_table_name, entries in data_table_entries.iteritems():
-        #get data table managed by this data Manager
+    for data_table_name, entries in data_table_entries.items():
+        # get data table managed by this data Manager
         has_data_table = app.tool_data_tables.get_tables().get( data_table_name )
         if has_data_table:
             has_data_table = bool( has_data_table.get_filename_for_source( data_manager, None ) )
@@ -77,7 +85,7 @@
                 from tool_shed.tools import data_table_manager
                 tdtm = data_table_manager.ToolDataTableManager( app )
                 target_dir, tool_path, relative_target_dir = tdtm.get_target_install_dir( tool_shed_repository )
-            #Dynamically add this data table
+            # Dynamically add this data table
             log.debug( "Attempting to dynamically create a missing Tool Data Table named %s." % data_table_name )
             data_table = data_tables[data_table_name]
             repo_info = tdtm.generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None )
@@ -89,21 +97,23 @@
             app.tool_data_tables.add_new_entries_from_config_file( tmp_file.name, None, app.config.shed_tool_data_table_config, persist=True )
             tmp_file.close()
 
+
 def galaxy_code_get_available_data_tables( trans ):
-    #list of data tables
+    # list of data tables
     found_tables = get_available_tables( trans )
-    rval = map( lambda x: ( ( x, x, DEFAULT_SELECTED ) ), found_tables )
+    rval = [ ( x, x, DEFAULT_SELECTED ) for x in found_tables]
     return rval
 
+
 def galaxy_code_get_available_data_tables_entries( trans, dbkey, data_table_names ):
-    #available entries, optionally filtered by dbkey and table names
+    # available entries, optionally filtered by dbkey and table names
     if dbkey in [ None, '', '?' ]:
         dbkey = None
     if data_table_names in [ None, '', '?' ]:
         data_table_names = None
     found_tables = get_available_tables_for_dbkey( trans, dbkey, data_table_names )
     dbkey_text = '(%s) ' % ( dbkey ) if dbkey else ''
-    rval = map( lambda x: ( "%s%s" % ( dbkey_text, x[0] ), dumps( dict( name=x[0].split( ': ' )[0], entry=x[1] ) ).encode( 'base64' ).rstrip(), DEFAULT_SELECTED ), found_tables.items() )
+    rval = [( "%s%s" % ( dbkey_text, x[0] ), dumps( dict( name=x[0].split( ': ' )[0], entry=x[1] ) ).encode( 'base64' ).rstrip(), DEFAULT_SELECTED ) for x in found_tables.items()]
     return rval
 
 # --- End Galaxy called Methods ---
@@ -116,8 +126,9 @@
     url = url.lstrip( '/' )
     return "%s/%s" % ( base, url )
 
+
 def rsync_list_dir( server, dir=None, skip_names=[] ):
-    #drwxr-xr-x          50 2014/05/16 20:58:11 .
+    # drwxr-xr-x          50 2014/05/16 20:58:11 .
     if dir:
         dir = rsync_urljoin( server, dir )
     else:
@@ -153,6 +164,7 @@
     rsync_response.close()
     return rval
 
+
 def rsync_sync_to_dir( source, target ):
     rsync_response = tempfile.NamedTemporaryFile()
     rsync_stderr = tempfile.NamedTemporaryFile()
@@ -176,31 +188,31 @@
     if cached_data_table is None:
         return True, {}
     if datetime.datetime.now() - cached_data_table.get( 'time_loaded' ) > CACHE_TIME:
-        data_table_text = urllib2.urlopen( url ).read()
+        data_table_text = urlopen( url ).read()
         if cached_data_table.get( 'data_table_text', None ) != data_table_text:
-            return True, {'data_table_text':data_table_text}
+            return True, {'data_table_text': data_table_text}
         loc_file_attrs = rsync_list_dir( RSYNC_SERVER, LOCATION_DIR )
-        if cached_data_table.get( 'loc_file_attrs', None ) !=  loc_file_attrs:
-            return True, {'loc_file_attrs':loc_file_attrs}
+        if cached_data_table.get( 'loc_file_attrs', None ) != loc_file_attrs:
+            return True, {'loc_file_attrs': loc_file_attrs}
     return False, {}
 
+
 def load_data_tables_from_url( url=None, site='main', data_table_class=None  ):
     if not url:
         url = TOOL_DATA_TABLE_CONF_XML_URLS.get( site, None )
     assert url, ValueError( 'You must provide either a URL or a site=name.' )
-    
+
     cached_data_table = TOOL_DATA_TABLES_LOADED_BY_URL.get( url, None )
     refresh, attribs = data_table_needs_refresh( cached_data_table, url )
     if refresh:
-        data_table_text = attribs.get( 'data_table_text' )or urllib2.urlopen( url ).read()
+        data_table_text = attribs.get( 'data_table_text' )or urlopen( url ).read()
         loc_file_attrs = attribs.get( 'loc_file_attrs' ) or rsync_list_dir( RSYNC_SERVER, LOCATION_DIR )
-        
+
         tmp_dir = tempfile.mkdtemp( prefix='rsync_g2_' )
         tmp_loc_dir = os.path.join( tmp_dir, 'location' )
         os.mkdir( tmp_loc_dir )
         rsync_sync_to_dir( rsync_urljoin( RSYNC_SERVER, LOCATION_DIR ), os.path.abspath( tmp_loc_dir ) )
-        
-        
+
         new_data_table_text = data_table_text.replace( TOOL_DATA_TABLE_CONF_XML_REPLACE_SOURCE, TOOL_DATA_TABLE_CONF_XML_REPLACE_TARGET % ( tmp_loc_dir ) )
         data_table_fh = tempfile.NamedTemporaryFile( dir=tmp_dir, prefix='rysnc_data_manager_data_table_conf_' )
         data_table_fh.write( new_data_table_text )
@@ -208,17 +220,18 @@
         tmp_data_dir = os.path.join( tmp_dir, 'tool-data' )
         os.mkdir( tmp_data_dir )
         data_tables = data_table_class( tmp_data_dir, config_filename=data_table_fh.name )
-        for name, data_table in data_tables.data_tables.items():
+        for name, data_table in list(data_tables.data_tables.items()):
             if name in EXCLUDE_DATA_TABLES or not data_table_has_path_column( data_table ):
                 log.debug( 'Removing data table "%s" because it is excluded by name or does not have a defined "path" column.', name )
                 del data_tables.data_tables[name]
         cached_data_table = { 'data_tables': data_tables, 'tmp_dir': tmp_dir, 'data_table_text': data_table_text, 'tmp_loc_dir': tmp_loc_dir, 'loc_file_attrs': loc_file_attrs, 'time_loaded': datetime.datetime.now() }
         TOOL_DATA_TABLES_LOADED_BY_URL[ url ] = cached_data_table
-        #delete the files
+        # delete the files
         data_table_fh.close()
         cleanup_before_exit( tmp_dir )
     return cached_data_table
 
+
 def data_table_has_path_column( data_table ):
     col_names = data_table.get_column_name_list()
     for name in PATH_COLUMN_NAMES:
@@ -226,10 +239,12 @@
             return True
     return False
 
+
 def get_available_tables( trans ):
-    #list of data tables
+    # list of data tables
     data_tables = load_data_tables_from_url( data_table_class=trans.app.tool_data_tables.__class__ )
-    return data_tables.get( 'data_tables' ).get_tables().keys()
+    return list(data_tables.get( 'data_tables' ).get_tables().keys())
+
 
 def get_new_xml_definition( app, data_table, data_manager, repo_info=None, location_file_dir=None ):
     sub_dict = { 'table_name': data_table.name, 'comment_char': '', 'columns': '', 'file_path': '' }
@@ -242,7 +257,7 @@
     location_file_dir = location_file_dir or app.config.galaxy_data_manager_data_path
     for filename in data_table.filenames.keys():
         sub_dict['file_path'] = basename( filename )
-        sub_dict['file_path'] = os.path.join( location_file_dir, sub_dict['file_path'] ) #os.path.abspath?
+        sub_dict['file_path'] = os.path.join( location_file_dir, sub_dict['file_path'] )  # os.path.abspath?
         if not os.path.exists( sub_dict['file_path'] ):
             # Create empty file
             open( sub_dict['file_path'], 'wb+' ).close()
@@ -256,22 +271,23 @@
             </table></tables>
            """ % sub_dict
 
+
 def get_available_tables_for_dbkey( trans, dbkey, data_table_names ):
-    my_data_tables = trans.app.tool_data_tables.get_tables()
     data_tables = load_data_tables_from_url( data_table_class=trans.app.tool_data_tables.__class__ )
     rval = {}
-    for name, data_table in data_tables.get( 'data_tables' ).get_tables().iteritems():
-        if ( not data_table_names or name in data_table_names ): #name in my_data_tables.keys() and 
-            #TODO: check that columns are similiar
+    for name, data_table in data_tables.get( 'data_tables' ).get_tables().items():
+        if ( not data_table_names or name in data_table_names ):
+            # TODO: check that columns are similiar
             if not dbkey:
                 entry_getter = data_table.get_named_fields_list()
             else:
                 entry_getter = data_table.get_entries( 'dbkey', dbkey, None, default=[] )
             for entry in entry_getter:
-                 name = "%s: %s" % ( data_table.name, dumps( entry ) )
-                 rval[name] = entry
+                name = "%s: %s" % ( data_table.name, dumps( entry ) )
+                rval[name] = entry
     return rval
 
+
 def split_path_all( path ):
     rval = []
     path = path.rstrip( '/' )
@@ -286,12 +302,13 @@
         else:
             break
     rval.reverse()
-    return rval 
+    return rval
+
 
 def get_data_for_path( path, data_root_dir ):
     # We list dir with a /, but copy data without
     # listing with / gives a . entry when its a dir
-    # cloning without the / will copy that whole directory into the target, 
+    # cloning without the / will copy that whole directory into the target,
     # instead of just that target's contents
     if path.startswith( GALAXY_DATA_CANONICAL_PATH ):
         path = path[ len( GALAXY_DATA_CANONICAL_PATH ):]
@@ -301,14 +318,14 @@
         rsync_source = rsync_source[:-1]
     try:
         dir_list = rsync_list_dir( rsync_source + "/" )
-    except Exception, e:
+    except Exception:
         dir_list = None
     while not dir_list or '.' not in dir_list:
         head, tail = os.path.split( make_path )
         if not head:
             head = tail
         make_path = head
-        rsync_source = rsync_urljoin( rsync_urljoin( RSYNC_SERVER, INDEX_DIR ), head ) #if we error here, likely due to a connection issue
+        rsync_source = rsync_urljoin( rsync_urljoin( RSYNC_SERVER, INDEX_DIR ), head )  # if we error here, likely due to a connection issue
         if rsync_source.endswith( '/' ):
             rsync_source = rsync_source[:-1]
         dir_list = rsync_list_dir( rsync_source + "/" )
@@ -321,43 +338,45 @@
     rsync_sync_to_dir( rsync_source, target_path )
     return path
 
+
 def get_data_and_munge_path( data_table_name, data_table_entry, data_root_dir ):
     path_cols = []
-    for key, value in data_table_entry.iteritems():
+    for key, value in data_table_entry.items():
         if key in PATH_COLUMN_NAMES:
             path_cols.append( ( key, value ) )
-    found_data = False
     if path_cols:
         for col_name, value in path_cols:
-            #GALAXY_DATA_CANONICAL_PATH
             if value.startswith( GALAXY_DATA_CANONICAL_PATH ):
                 data_table_entry[col_name] = get_data_for_path( value, data_root_dir )
-                found_data = True
             else:
-                print 'unable to determine location of rsync data for', data_table_name, data_table_entry
+                print('unable to determine location of rsync data for', data_table_name, data_table_entry)
     return data_table_entry
 
+
 def fulfill_data_table_entries( data_table_entries, data_manager_dict, data_root_dir ):
-    for data_table_name, entries in data_table_entries.iteritems():
+    for data_table_name, entries in data_table_entries.items():
         for entry in entries:
             entry = get_data_and_munge_path( data_table_name, entry, data_root_dir )
             _add_data_table_entry( data_manager_dict, data_table_name, entry )
     return data_manager_dict
 
+
 def _add_data_table_entry( data_manager_dict, data_table_name, data_table_entry ):
     data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
     data_manager_dict['data_tables'][data_table_name] = data_manager_dict['data_tables'].get( data_table_name, [] )
     data_manager_dict['data_tables'][data_table_name].append( data_table_entry )
     return data_manager_dict
 
+
 def cleanup_before_exit( tmp_dir ):
     if tmp_dir and os.path.exists( tmp_dir ):
         shutil.rmtree( tmp_dir )
 
+
 def get_data_table_entries( params ):
     rval = {}
     data_table_entries = params.get( 'data_table_entries', None )
-    if data_table_entries :
+    if data_table_entries:
         for entry_text in data_table_entries.split( ',' ):
             entry_text = entry_text.strip().decode( 'base64' )
             entry_dict = loads( entry_text )
@@ -367,24 +386,26 @@
             rval[ data_table_name ].append( data_table_entry )
     return rval
 
+
 def main():
-    #Parse Command Line
     parser = optparse.OptionParser()
     (options, args) = parser.parse_args()
-    
+
     filename = args[0]
-    
+
     params = loads( open( filename ).read() )
     target_directory = params[ 'output_data' ][0]['extra_files_path']
     os.mkdir( target_directory )
     data_manager_dict = {}
-    
+
     data_table_entries = get_data_table_entries( params['param_dict'] )
-    
+
     # Populate the data Tables
     data_manager_dict = fulfill_data_table_entries( data_table_entries, data_manager_dict, target_directory )
-    
-    #save info to json file
+
+    # save info to json file
     open( filename, 'wb' ).write( dumps( data_manager_dict ) )
-        
-if __name__ == "__main__": main()
+
+
+if __name__ == "__main__":
+    main()
--- a/data_manager/data_manager_rsync.xml	Wed Oct 14 13:48:12 2015 -0400
+++ b/data_manager/data_manager_rsync.xml	Tue Apr 04 18:13:26 2017 -0400
@@ -1,23 +1,18 @@
 <tool id="data_manager_rsync_g2" name="Rsync with g2" version="0.0.1" tool_type="manage_data">
     <options sanitize="False" />
     <description>fetching</description>
-    <command interpreter="python">data_manager_rsync.py "${out_file}"</command>
-    <stdio>
-        <exit_code range="1:" err_level="fatal" />
-        <exit_code range=":-1" err_level="fatal" />
-    </stdio>
+    <command detect_errors="exit_code"><![CDATA[
+        python '$__tool_directory__/data_manager_rsync.py'
+        '${out_file}'
+    ]]></command>
     <inputs>
-        
         <param name="dbkey" type="genomebuild" label="dbkey to search for Reference Data" help="Specify ? to show all"/>
 
-        <param name="data_table_names" type="select" display="checkboxes" multiple="True" optional="True" 
-        label="Choose Desired Data Tables" dynamic_options="galaxy_code_get_available_data_tables( __trans__ )"
-        refresh_on_change="dbkey"/>
-        
-        
-        <param name="data_table_entries" type="select" display="checkboxes" multiple="True" optional="False" 
-        label="Choose Desired Data Tables Entries" dynamic_options="galaxy_code_get_available_data_tables_entries( __trans__, dbkey, data_table_names )"
-        refresh_on_change="dbkey"/>
+        <param name="data_table_names" type="select" display="checkboxes" multiple="True" optional="True" refresh_on_change="dbkey"
+            label="Choose Desired Data Tables" dynamic_options="galaxy_code_get_available_data_tables( __trans__ )" />
+
+        <param name="data_table_entries" type="select" display="checkboxes" multiple="true" optional="false" refresh_on_change="dbkey"
+            label="Choose Desired Data Tables Entries" dynamic_options="galaxy_code_get_available_data_tables_entries( __trans__, dbkey, data_table_names )" />
     </inputs>
     <outputs>
         <data name="out_file" format="data_manager_json" dbkey="dbkey"/>
@@ -37,12 +32,9 @@
 
 ------
 
-
-
 .. class:: infomark
 
-**Notice:** If you do not have a particular data table defined, then it will be created and persisted dynamically. 
-
+**Notice:** If you do not have a particular data table defined, then it will be created and persisted dynamically.
     </help>
     <code file="data_manager_rsync.py" />
 </tool>