Mercurial > repos > yating-l > gonramp_apollo_tools
changeset 0:ce4f91831680 draft default tip
planemo upload for repository https://github.com/Yating-L/suite_gonramp_apollo.git commit 5367a00befb467f162d1870edb91f9face72e894
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.rst Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,64 @@ +G-OnRamp Apollo Tools +===================== + +A suite of Galaxy tools for managing a local Apollo server. It includes following tools: + +JBrowseHub to Apollo +-------------------- + +This Galaxy tool is used to create or overwrite an organism on an Apollo server with a jbrowse hub created by JBrowse Archive Creator. + +Apollo User Manager +------------------- + +This Galaxy tool is used to manage Apollo users. The currently supported operation including: + + - Create a new user + - Delete a user + - Add a user to a group (If the group doesn't exist, create the group) + - Remove a user to a group + +The tool can do these operations on one student at a time. It can also do the operations on multiple students at a time by uploading a text file, which including students information. + +The text file can be either CSV (comma-delimited) or Tabular (tab-delimited). It should have a header line, including names for each column. Example text files: + +Text file for creating multiple users: + +.. csv-table:: + :header: "useremail", "firstname", "lastname", "password" + :widths: 20, 10, 10, 10 + + "test1@demo.com", "test1", "demo", "1234" + "test2@demo.com", "test2", "demo", "1234" + "test3@demo.com", "test3", "demo", "1234" + + +Text file for deleting multiple users: + +.. csv-table:: + :header: "useremail" + :widths: 20 + + "test1@demo.com" + "test2@demo.com" + "test3@demo.com" + +Text file for adding multiple users to a group: + +.. csv-table:: + :header: "useremail", "group" + :widths: 20, 20 + + "test1@demo.com", "Test group" + "test2@demo.com", "Test group" + "test3@demo.com", "Test group" + +Text file for removing multiple users to a group: + +.. csv-table:: + :header: "useremail", "group" + :widths: 20, 20 + + "test1@demo.com", "Test group" + "test2@demo.com", "Test group" + "test3@demo.com", "Test group"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/apollo/ApolloInstance.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,80 @@ +#!/usr/bin/env python +import os +import json +import shutil +import tempfile +import logging +import random +import string +from util import subtools +from mako.lookup import TemplateLookup + +from ApolloOrganism import ApolloOrganism +from ApolloUser import ApolloUser + +class ApolloInstance(object): + def __init__(self, apollo_host, apollo_admin, tool_directory): + self.apollo_host = apollo_host + self.tool_directory = tool_directory + self.logger = logging.getLogger(__name__) + self.apollo_admin = apollo_admin + self.apolloTemplate = self._getApolloTemplate() + self._arrow_init() + + + def _arrow_init(self): + subtools.verify_user_login(self.apollo_admin['user_email'], self.apollo_admin['password'], self.apollo_host) + arrow_config = tempfile.NamedTemporaryFile(bufsize=0) + with open(arrow_config.name, 'w') as conf: + htmlMakoRendered = self.apolloTemplate.render( + apollo_host = self.apollo_host, + admin_user = self.apollo_admin['user_email'], + admin_pw = self.apollo_admin['password'] + ) + conf.write(htmlMakoRendered) + + home_dir = os.path.expanduser('~') + arrow_config_dir = os.path.join(home_dir, '.apollo-arrow.yml') + shutil.copyfile(arrow_config.name, arrow_config_dir) + self.logger.debug("Initated arrow: apollo-arrow.yml= %s", arrow_config_dir) + + #TODO: Encode admin password + ''' + def _generatePassword(self, length=8): + chars = string.digits + string.letters + pw = ''.join([random.choice(chars) for _ in range(length)]) + return pw + ''' + + def _getApolloTemplate(self): + mylookup = TemplateLookup(directories=[os.path.join(self.tool_directory, 'templates')], + output_encoding='utf-8', encoding_errors='replace') + apolloTemplate = mylookup.get_template("apollo-arrow.yml") + return apolloTemplate + + + def manageApolloOrganism(self, organism_name, organism_dir, action): + organism = ApolloOrganism(organism_name, organism_dir) + if action == "add": + organism.addOrganism() + self.logger.info("Successfully add a new organism (%s) to Apollo", organism_name) + elif action == "overwrite": + organism.overwriteOrganism() + self.logger.info("Successfully overwrite the organism %s", organism_name) + else: + self.logger.error("Invalid operation %s", action) + exit(-1) + + def manageApolloUser(self, operations_dictionary = dict()): + for operation, users_list in operations_dictionary.items(): + apollo_user = ApolloUser(users_list) + if operation == "create": + apollo_user.createApolloUser() + elif operation == "delete": + apollo_user.deleteApolloUser() + elif operation == "add": + apollo_user.addApolloUserToGroup() + elif operation == "remove": + apollo_user.removeApolloUserFromeGroup() + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/apollo/ApolloOrganism.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import json +import logging +from util import subtools + +class ApolloOrganism(object): + + def __init__(self, organism_name, organism_dir): + self.organism_name = organism_name + self.organism_dir = organism_dir + self.logger = logging.getLogger(__name__) + + def addOrganism(self): + exist = subtools.arrow_get_organism(self.organism_name) + if not exist: + self.logger.debug("The organism does not exist.") + p = subtools.arrow_add_organism(self.organism_name, self.organism_dir) + if not p: + self.logger.error("The user is not authorized to add organism") + exit(-1) + organism = json.loads(p) + organism_id = organism['id'] + self.logger.debug("A new organism %s was added to Apollo instance", p) + return organism_id + else: + self.logger.error("The organism %s is already on Apollo instance! Rerun the tool to use a different species name or choose to overwrite the organism", self.organism_name) + exit(-1) + + #TODO: the JSON dictionary return by deleteOrganism still contains the deleted organism. Improve the API. + def deleteOrganism(self): + organism_id = subtools.arrow_get_organism(self.organism_name) + if organism_id: + self.logger.debug("Deleting the organism %s", self.organism_name) + subtools.arrow_delete_organism(organism_id) + if not subtools.arrow_get_organism(self.organism_name): + self.logger.debug("Organism %s has been deleted", self.organism_name) + else: + self.logger.error("Organism %s cannot be deleted", self.organism_name) + exit(-1) + else: + self.logger.error("Organism %s doesn't exist", self.organism_name) + exit(-1) + + #TODO: API update_organism not working. Improve the API to enable updating directory. + def overwriteOrganism(self): + self.deleteOrganism() + p = subtools.arrow_add_organism(self.organism_name, self.organism_dir) + if not p: + self.logger.error("The user is not authorized to add organism") + exit(-1) + organism = json.loads(p) + organism_id = organism['id'] + self.logger.debug("A new organism %s has been added to Apollo instance", p) + return organism_id + + + \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/apollo/ApolloUser.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,98 @@ +#!/usr/bin/python + + +import os +import sys +import json +import logging +from util import subtools + +class ApolloUser(object): + """ + This class is to manage Apollo users, such as create, delete, add users to a group or delete users from a group + + """ + + def __init__(self, users_list): + self.users_list = users_list + self.logger = logging.getLogger(__name__) + + + def createApolloUser(self): + for user in self.users_list: + if user['batch'] == "false": + subtools.arrow_create_user(user['useremail'], user['firstname'], user['lastname'], user['password']) + elif user['batch'] == "true": + users = self.parseUserInfoFile(user['format'], user['false_path']) + for u in users: + if not 'useremail' in u: + self.logger.error("Cannot find useremail in the text file, make sure you use the correct header, see README file for examples.") + if not 'firstname' in u: + self.logger.error("Cannot find firstname in the text file, make sure you use the correct header, see README file for examples.") + if not 'lastname' in u: + self.logger.error("Cannot find lastname in the text file, make sure you use the correct header, see README file for examples.") + if not 'password' in u: + self.logger.error("Cannot find password in the text file, make sure you use the correct header, see README file for examples.") + subtools.arrow_create_user(u['useremail'], u['firstname'], u['lastname'], u['password']) + + + def parseUserInfoFile(self, file_format, filename): + if file_format == "tab": + delimiter = '\t' + elif file_format == "csv": + delimiter = ',' + else: + self.logger.error("The %s format is not supported!", file_format) + with open(filename, 'r') as f: + lines = f.readlines() + headers = lines[0].split(delimiter) + users = [] + lines = lines[1:] + for l in lines: + l = l.split(delimiter) + info = dict() + fields = len(l) + for i in range(fields): + title = headers[i].rstrip() + info[title] = l[i].rstrip() + users.append(info) + return users + + def deleteApolloUser(self): + for user in self.users_list: + if user['batch'] == "false": + subtools.arrow_delete_user(user['useremail']) + elif user['batch'] == "true": + users = self.parseUserInfoFile(user['format'], user['false_path']) + for u in users: + if not 'useremail' in u: + self.logger.error("Cannot find useremail in the text file, make sure you use the correct header, see README file for examples.") + subtools.arrow_delete_user(u['useremail']) + + def addApolloUserToGroup(self): + for user in self.users_list: + if user['batch'] == "false": + subtools.arrow_add_to_group(user['group'], user['useremail']) + elif user['batch'] == "true": + users = self.parseUserInfoFile(user['format'], user['false_path']) + for u in users: + if not 'useremail' in u: + self.logger.error("Cannot find useremail in the text file, make sure you use the correct header, see README file for examples.") + if not 'group' in u: + self.logger.error("Cannot find group in the text file, make sure you use the correct header, see README file for examples.") + subtools.arrow_add_to_group(u['group'], u['useremail']) + + def removeApolloUserFromeGroup(self): + for user in self.users_list: + if user['batch'] == "false": + subtools.arrow_remove_from_group(user['group'], user['useremail']) + elif user['batch'] == "true": + users = self.parseUserInfoFile(user['format'], user['false_path']) + for u in users: + if not 'useremail' in u: + self.logger.error("Cannot find useremail in the text file, make sure you use the correct header, see README file for examples.") + if not 'group' in u: + self.logger.error("Cannot find group in the text file, make sure you use the correct header, see README file for examples.") + subtools.arrow_remove_from_group(u['group'], u['useremail']) + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/apolloUserManager.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,63 @@ +#!/usr/bin/env python +import os +import sys +import argparse +import json +import logging +import socket +from apollo.ApolloInstance import ApolloInstance +from apollo.ApolloUser import ApolloUser +from util.Reader import Reader +from util.Logger import Logger + + +def main(argv): + parser = argparse.ArgumentParser(description='Upload a hub to display on Apollo.') + parser.add_argument('-j', '--data_json', help='JSON file containing the metadata of the inputs') + parser.add_argument('-o', '--output', help='HTML output') + + # Get the args passed in parameter + args = parser.parse_args() + json_inputs_data = args.data_json + outputFile = args.output + + ##Parse JSON file with Reader + reader = Reader(json_inputs_data) + + # Begin init variables + + apollo_port = reader.getPortNum() + apollo_host = "http://localhost:"+ apollo_port + "/apollo" + apollo_admin_user = reader.getAdminUser() + toolDirectory = reader.getToolDir() + extra_files_path = reader.getExtFilesPath() + debug_mode = reader.getDebugMode() + operations_dictionary = reader.getOperationList() + + + + + #### Logging management #### + # If we are in Debug mode, also print in stdout the debug dump + log = Logger(tool_directory=toolDirectory, debug=debug_mode, extra_files_path=extra_files_path) + log.setup_logging() + + logging.info("#### Apollo User Manager: Start on Apollo instance: %s #### ", apollo_host) + logging.debug('JSON parameters: %s\n\n', json.dumps(reader.args)) + + # Set up apollo + apollo = ApolloInstance(apollo_host, apollo_admin_user, toolDirectory) + apollo.manageApolloUser(operations_dictionary) + outHtml(outputFile, apollo_host) + logging.info('#### Apollo User Manager: Congratulation! ####\n') + +def outHtml(outputFile, host_name): + with open(outputFile, 'w') as htmlfile: + htmlstr = 'The Apollo User Manager has done with operations on Apollo: <br>' + jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub on Apollo</a></li>' % host_name + htmlstr += jbrowse_hub + htmlfile.write(htmlstr) + + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/apolloUserManager.xml Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,290 @@ +<tool id="apollo_user_manager" name="Apollo User Manager" version="1.0.0"> + <description> + This Galaxy tool is used to manage Apollo users. + </description> + <macros> + <import>macros.xml</import> + </macros> + + <requirements> + <requirement type="package" version="3.0.3">apollo_api</requirement> + </requirements> + + <stdio> + </stdio> + + <command detect_errors="exit_code"><![CDATA[ + mkdir -p $output.extra_files_path; + ## Dump the tool parameters into a JSON file + python $json_file parameters.json; + python $__tool_directory__/apolloUserManager.py --data_json parameters.json -o $output + ]]></command> + <configfiles> + <configfile name="json_file"> +import json +import sys + +file_path = sys.argv[1] +#set global data_parameter_dict = {"operations": dict()} +## Function to retrieve the data of the inputs +#def prepare_json($operation_type, $data_dict, $batch) + #silent $data_dict.update({"batch": str($batch)}) + #if $operation_type in $data_parameter_dict["operations"] + #silent $data_parameter_dict["operations"][$operation_type].append($data_dict) + #else + #set array_inputs = [] + #silent $array_inputs.append($data_dict) + #silent $data_parameter_dict["operations"].update({$operation_type: $array_inputs}) + #end if +#end def + + + + +#for $i, $f in enumerate($operation) + #if $f.operation_type_selector.operation_type == "create" + #set batch = $f.operation_type_selector.batch_selector.batch + #if $batch == "false" + #set data_dict = {"useremail": str($f.operation_type_selector.batch_selector.user_email), + "firstname": str($f.operation_type_selector.batch_selector.firstname), + "lastname": str($f.operation_type_selector.batch_selector.lastname), + "password": str($f.operation_type_selector.batch_selector.password)} + + #else + #set data_dict = {"format": str($f.operation_type_selector.batch_selector.format), + "false_path": str($f.operation_type_selector.batch_selector.user_info_file)} + #end if + #silent $prepare_json("create", $data_dict, $batch) + + #elif $f.operation_type_selector.operation_type == "delete" + #set batch = $f.operation_type_selector.batch_selector.batch + #if $batch == "false" + #set data_dict = {"useremail": str($f.operation_type_selector.batch_selector.user_email)} + #else + #set data_dict = {"format": str($f.operation_type_selector.batch_selector.format), + "false_path": str($f.operation_type_selector.batch_selector.user_info_file)} + #end if + #silent $prepare_json("delete", $data_dict, $batch) + #elif $f.operation_type_selector.operation_type == "add" + #set batch = $f.operation_type_selector.batch_selector.batch + #if $batch == "false" + #set data_dict = {"useremail": str($f.operation_type_selector.batch_selector.user_email), + "group": str($f.operation_type_selector.batch_selector.group_name)} + #else + #set data_dict = {"format": str($f.operation_type_selector.batch_selector.format), + "false_path": str($f.operation_type_selector.batch_selector.user_info_file)} + #end if + #silent $prepare_json("add", $data_dict, $batch) + #elif $f.operation_type_selector.operation_type == "remove" + #set batch = $f.operation_type_selector.batch_selector.batch + #if $batch == "false" + #set data_dict = {"useremail": str($f.operation_type_selector.batch_selector.user_email), + "group": str($f.operation_type_selector.batch_selector.group_name)} + #else + #set data_dict = {"format": str($f.operation_type_selector.batch_selector.format), + "false_path": str($f.operation_type_selector.batch_selector.user_info_file)} + #end if + #silent $prepare_json("remove", $data_dict, $batch) + #end if +#end for + + +#set apollo_admin = {"user_email": str($admin_username), "password": str($admin_password)} +#silent $data_parameter_dict.update({"apollo_admin": $apollo_admin}) +#silent $data_parameter_dict.update({"tool_directory": str($__tool_directory__)}) +#silent $data_parameter_dict.update({"extra_files_path": str($output.extra_files_path)}) +#silent $data_parameter_dict.update({"port": str($advanced_options.port)}) +#silent $data_parameter_dict.update({"debug_mode": str($advanced_options.debug_mode)}) +with open(file_path, 'w') as f: + json.dump($data_parameter_dict, f) + </configfile> + </configfiles> + + + <inputs> + <param name="admin_username" type="text" label="Admin username" help="Login in with Apollo admin account"> + <sanitizer> + <valid initial="string.letters,string.digits"> + <add value="@-=_.()/+*^,:?!"/> + </valid> + </sanitizer> + </param> + + <param + name="admin_password" + type="text" + label="Admin password" + /> + + <repeat name="operation" title="New operation"> + <conditional name="operation_type_selector" > + <param name="operation_type" type="select" label="Create a user, delete a user, add a user to a group or remove a user from a group"> + <option value="create">Create a user</option> + <option value="delete">Delete a users</option> + <option value="add">Add a user to a group</option> + <option value="remove">Remove a user from a group</option> + </param> + <when value="create"> + <conditional name="batch_selector"> + <param name="batch" type="select" label="Manipulate a single user or multiple users"> + <option value="false" selected="true">Single user</option> + <option value="true">Multiple users</option> + </param> + <when value="false"> + <param name="user_email" type="text" label="User email" help="Specify the user email"> + <sanitizer> + <valid initial="string.letters,string.digits"> + <add value="@-=_.()/+*^,:?!"/> + </valid> + </sanitizer> + </param> + <param name="firstname" type="text" label="First name" /> + <param name="lastname" type="text" label="Last name" /> + <param name="password" type="text" label="Password" /> + </when> + <when value="true"> + <expand macro="upload_text_file" /> + </when> + </conditional> + </when> + <when value="delete"> + <conditional name="batch_selector"> + <param name="batch" type="select" label="Manipulate a single user or multiple users"> + <option value="false" selected="true">Single user</option> + <option value="true">Multiple users</option> + </param> + <when value="false"> + <param name="user_email" type="text" label="User email" help="Specify the user email"> + <sanitizer> + <valid initial="string.letters,string.digits"> + <add value="@-=_.()/+*^,:?!"/> + </valid> + </sanitizer> + </param> + </when> + <when value="true"> + <expand macro="upload_text_file" /> + </when> + </conditional> + </when> + <when value="add"> + <conditional name="batch_selector"> + <param name="batch" type="select" label="Manipulate a single user or multiple users"> + <option value="false" selected="true">Single user</option> + <option value="true">Multiple users</option> + </param> + <when value="false"> + <param name="user_email" type="text" label="User email" help="Specify the user email"> + <sanitizer> + <valid initial="string.letters,string.digits"> + <add value="@-=_.()/+*^,:?!"/> + </valid> + </sanitizer> + </param> + <param type="text" name="group_name" size="30" value="unknown" label="Group name" /> + </when> + <when value="true"> + <expand macro="upload_text_file" /> + </when> + </conditional> + </when> + <when value="remove"> + <conditional name="batch_selector"> + <param name="batch" type="select" label="Manipulate a single user or multiple users"> + <option value="false" selected="true">Single user</option> + <option value="true">Multiple users</option> + </param> + <when value="false"> + <param name="user_email" type="text" label="User email" help="Specify the user email"> + <sanitizer> + <valid initial="string.letters,string.digits"> + <add value="@-=_.()/+*^,:?!"/> + </valid> + </sanitizer> + </param> + <param type="text" name="group_name" size="30" value="unknown" label="Group name" /> + </when> + <when value="true"> + <expand macro="upload_text_file" /> + </when> + </conditional> + </when> + </conditional> + </repeat> + + <conditional name="advanced_options"> + <param name="advanced_options_selector" type="select" label="Advanced options"> + <option value="off" selected="true">Hide advanced options</option> + <option value="on">Display advanced options</option> + </param> + <!-- TODO: Avoid redundancy here --> + <when value="on"> + <param name="port" type="integer" min="8000" max="8888" value="8080" label="Port number of Apollo" /> + <param name="debug_mode" type="select" label="Activate debug mode"> + <option value="false" selected="true">No</option> + <option value="true">Yes</option> + <help> + Use this option if you are a G-OnRamp developer + </help> + </param> + </when> + <when value="off"> + <param name="port" type="hidden" value="8080" /> + <param name="debug_mode" type="hidden" + value="false"> + </param> + </when> + </conditional> + </inputs> + <outputs> + <data format="txt" name="output" label="${tool.name} on ${on_string}" /> + </outputs> + + <help> + This Galaxy tool is used to manage Apollo users.The currently supported operation including: + + - Create a new user + - Delete a user + - Add a user to a group (If the group doesn't exist, create the group) + - Remove a user to a group + +The tool can do these operations on one student at a time. It can also do the operations on multiple students at a time by uploading a text file, which including students information. + +The text file can be either CSV (comma-delimited) or Tabular (tab-delimited). It should have a header line, including names for each column. Example text files: + +Text file for creating multiple users: + +.. csv-table:: + :header: "useremail", "firstname", "lastname", "password" + :widths: 20, 10, 10, 10 + + "test1@demo.com", "test1", "demo", "1234" + "test2@demo.com", "test2", "demo", "1234" + "test3@demo.com", "test3", "demo", "1234" + + +Text file for deleting multiple users: + +.. csv-table:: + :header: "useremail" + :widths: 20 + + "test1@demo.com" + "test2@demo.com" + "test3@demo.com" + + +Text file for adding / removing multiple users from a group: + +.. csv-table:: + :header: "useremail", "group" + :widths: 20, 20 + + "test1@demo.com", "annotation_group1" + "test2@demo.com", "annotation_group1" + "test3@demo.com", "annotation_group1" + + </help> + <citations> + </citations> +</tool> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gonramp_apollo_tools.iml Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="PYTHON_MODULE" version="4"> + <component name="NewModuleRootManager" inherit-compiler-output="true"> + <exclude-output /> + <content url="file://$MODULE_DIR$" /> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> +</module> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/jbrowsehubToApollo.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,77 @@ +#!/usr/bin/env python +import os +import sys +import argparse +import json +import logging +import socket +from apollo.ApolloInstance import ApolloInstance +from util.Reader import Reader +from util.Logger import Logger + + +def main(argv): + parser = argparse.ArgumentParser(description='Upload a hub to display on Apollo.') + parser.add_argument('-j', '--data_json', help='JSON file containing the metadata of the inputs') + parser.add_argument('-o', '--output', help='HTML output') + + # Get the args passed in parameter + args = parser.parse_args() + json_inputs_data = args.data_json + outputFile = args.output + + ##Parse JSON file with Reader + reader = Reader(json_inputs_data) + + # Begin init variables + extra_files_path = reader.getExtFilesPath() + jbrowse_hub = reader.getJBrowseHubDir() + #user_email = reader.getUserEmail() + species_name = reader.getSpeciesName() + #apollo_host = reader.getApolloHost() + apollo_port = reader.getPortNum() + apollo_host = "http://localhost:"+ apollo_port + "/apollo" + #apollo_host = "http://localhost:8080/apollo" + #apollo_user = reader.getApolloUser() + apollo_admin_user = reader.getAdminUser() + toolDirectory = reader.getToolDir() + debug_mode = reader.getDebugMode() + action = reader.getAction() + + #### Logging management #### + # If we are in Debug mode, also print in stdout the debug dump + log = Logger(tool_directory=toolDirectory, debug=debug_mode, extra_files_path=extra_files_path) + log.setup_logging() + + logging.info("#### JBrowsehub To Apollo: Start to %s JBrowse Hub to Apollo instance: %s #### ", action, apollo_host) + logging.debug('JSON parameters: %s\n\n', json.dumps(reader.args)) + + # Set up apollo + apollo = ApolloInstance(apollo_host, apollo_admin_user, toolDirectory) + jbrowse_hub_dir = _getHubDir(jbrowse_hub) + apollo.manageApolloOrganism(species_name, jbrowse_hub_dir, action) + outHtml(outputFile, apollo_host, species_name, action) + + logging.info('#### JBrowsehub To Apollo: Congratulation! JBrowse Hub is uploaded! ####\n') + +def _getHubDir(extra_files_path): + for root, dirs, files in os.walk(extra_files_path): + for name in files: + if name == "trackList.json": + logging.debug("JBrowse hub directory: %s", root) + return root + logging.error("Cannot find jbrowsehub") + exit(-1) + +def outHtml(outputFile, host_name, species_name, action): + with open(outputFile, 'w') as htmlfile: + if action == "add": + htmlstr = 'The Organism "%s" is added on Apollo: <br>' % species_name + elif action == "overwrite": + htmlstr = 'The Organism "%s" is overwritten on Apollo: <br>' % species_name + jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub on Apollo</a></li>' % host_name + htmlstr += jbrowse_hub + htmlfile.write(htmlstr) + +if __name__ == "__main__": + main(sys.argv) \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/jbrowsehubToApollo.xml Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,110 @@ +<tool id="jbrowsehubtoapollo" name="JBrowseHub To Apollo" version="1.0.0"> + <description> + This Galaxy tool is used to prepare your files to be ready for displaying on Apollo + </description> + + <requirements> + <requirement type="package" version="3.0.3">apollo_api</requirement> + </requirements> + + <stdio> + </stdio> + + <command detect_errors="exit_code"><![CDATA[ + mkdir -p $output.extra_files_path; + ## Dump the tool parameters into a JSON file + python $json_file parameters.json; + python $__tool_directory__/jbrowsehubToApollo.py --data_json parameters.json -o $output + ]]></command> + <configfiles> + <configfile name="json_file"> +import json +import sys + +file_path = sys.argv[1] +#set global data_parameter_dict = {} +#silent $data_parameter_dict.update({"action": str($action)}) +#silent $data_parameter_dict.update({"action": str($action)}) +#silent $data_parameter_dict.update({"species_name": str($species_name)}) +#set apollo_admin = {"user_email": str($admin_username), "password": str($admin_password)} +#silent $data_parameter_dict.update({"apollo_admin": $apollo_admin}) +#silent $data_parameter_dict.update({"tool_directory": str($__tool_directory__)}) +#silent $data_parameter_dict.update({"extra_files_path": str($output.extra_files_path)}) +#slient $data_parameter_dict.update({"jbrowse_hub": str($jbrowse_hub.extra_files_path)}) +#silent $data_parameter_dict.update({"port": str($advanced_options.port)}) +#silent $data_parameter_dict.update({"debug_mode": str($advanced_options.debug_mode)}) +with open(file_path, 'w') as f: + json.dump($data_parameter_dict, f) + </configfile> + </configfiles> + + + <inputs> + <param + format="jbrowsehub" + type="data" + name="jbrowse_hub" + label="JBrowse Hub created by JBrowse Archive Creator" + /> + + <param name="action" type="select" label="Choose to create a new organism or overwrite an organism"> + <option value="add" selected="true">Add a new organism to Apollo</option> + <option value="overwrite">Overwrite an existing organism on Apollo</option> + </param> + + <param + name="species_name" + type="text" + size="30" + value="unknown" + label="Species name" + /> + + <param name="admin_username" type="text" label="Admin username" help="Login in with Apollo admin account"> + <sanitizer> + <valid initial="string.letters,string.digits"> + <add value="@-=_.()/+*^,:?!"/> + </valid> + </sanitizer> + </param> + + <param + name="admin_password" + type="text" + label="Admin password" + /> + + <conditional name="advanced_options"> + <param name="advanced_options_selector" type="select" label="Advanced options"> + <option value="off" selected="true">Hide advanced options</option> + <option value="on">Display advanced options</option> + </param> + <!-- TODO: Avoid redundancy here --> + <when value="on"> + <param name="port" type="integer" min="8000" max="8888" value="8080" label="Port number of Apollo" /> + <param name="debug_mode" type="select" label="Activate debug mode"> + <option value="false" selected="true">No</option> + <option value="true">Yes</option> + <help> + Use this option if you are a G-OnRamp developer + </help> + </param> + </when> + <when value="off"> + <param name="port" type="hidden" value="8080" /> + <param name="debug_mode" type="hidden" + value="false"> + </param> + </when> + </conditional> + </inputs> + <outputs> + <data format="html" name="output" label="${tool.name}" /> + </outputs> + + <help> + This Galaxy tool is used to create or overwrite an organism on an Apollo server with a jbrowse hub created by JBrowse Archive Creator. + </help> + <citations> + </citations> +</tool> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/logging.json Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,48 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + } + }, + + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + + "console_stderr": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "simple", + "stream": "ext://sys.stderr" + }, + + "debug_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "DEBUG", + "formatter": "simple", + "filename": "__main__.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + } + }, + + "loggers": { + "Reader": { + "level": "INFO", + "handlers": ["console"], + "propagate": "yes" + } + }, + + "root": { + "level": "DEBUG", + "handlers": ["console", "console_stderr", "debug_file_handler"] + } +} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros.xml Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,12 @@ +<macros> + + <xml name="upload_text_file"> + <param name="format" type="select" label="File format"> + <option value="csv" selected="true">CSV</option> + <option value="tab">Tabular</option> + </param> + <param name="user_info_file" type="data" format="tabular,csv" label="A text file stored a list of users' information" help="tab delimited file or csv file." /> + + </xml> + +</macros> \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/apollo-arrow.yml Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,10 @@ +## Apollo's Arrow: Global Configuration File. +# Each stanza should contian a single galaxy server to control. +# +# You can set the key __default to the name of a default instance +__default: local + +local: + url: ${apollo_host} + username: ${admin_user} + password: ${admin_pw}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/adduserstogroup.csv Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,4 @@ +useremail,group +test1@apollo.com,Test A +test2@apollo.com,Test A +test3@apollo.com,Test B \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/usersInfo.txt Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,4 @@ +useremail firstname lastname password +test1@apollo.com test1 apollo 1234 +test2@apollo.com test2 apollo 1234 +test3@apollo.com test3 apollo 4321 \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/usersInfo2.csv Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,4 @@ +useremail,firstname,lastname,password +test1@apollo.com,test1,apollo,1234 +test2@apollo.com,test2,apollo,1234 +test3@apollo.com,test3,apollo,4321 \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/todo.md Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,9 @@ +# JBrowseHubToApollo's TODO + +- [ ] Correct new Apollo user's email address. Galaxy santitizes '@' to '__at__' +- [ ] Check password for admin login + +### DONE + + +- [x] upload jbrowsehub to Apollo instance
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tool_dependencies.xml Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,23 @@ +<?xml version="1.0"?> +<tool_dependency> + + <package name="apollo_api" version="3.0.3"> + <install version="1.0"> + <actions> + <action type="download_by_url">https://github.com/galaxy-genome-annotation/python-apollo/archive/3.0.3.tar.gz</action> + <action type="make_directory">$INSTALL_DIR/apollo</action> + <action type="shell_command"> + export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR/apollo && + python setup.py install --install-lib $INSTALL_DIR/apollo + </action> + <action type="set_environment"> + <environment_variable action="append_to" name="PYTHONPATH">$INSTALL_DIR/apollo</environment_variable> + <environment_variable action="append_to" name="PATH">$INSTALL_DIR/apollo</environment_variable> + </action> + </actions> + </install> + </package> + + + +</tool_dependency>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/Logger.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,40 @@ +import os +import sys +import json +import logging +import logging.config + +#from util.Filters import TraceBackFormatter + +class Logger(object): + def __init__(self, tool_directory, debug="False", extra_files_path=None): + self.tool_directory = tool_directory + self.default_level = logging.INFO + self.debug = debug + self.extra_files_path = extra_files_path + + def setup_logging(self): + """ + Setup logging configuration + reference: https://fangpenlin.com/posts/2012/08/26/good-logging-practice-in-python/ + """ + config_path = os.path.join(self.tool_directory, 'logging.json') + default_level=logging.INFO + if self.debug.lower() == "true": + default_level=logging.DEBUG + if os.path.exists(config_path): + with open(config_path, 'rt') as f: + config = json.load(f) + config["handlers"]["console"]["level"] = default_level + if self.extra_files_path: + for i in config["handlers"]: + if "filename" in config["handlers"][i]: + config["handlers"][i]["filename"] = os.path.join(self.extra_files_path, config["handlers"][i]["filename"]) + logging.config.dictConfig(config) + else: + logging.config.dictConfig(config) + logging.warn("Extra files path is not set. The log files will exist at current working directory instead of final output folder") + else: + logging.basicConfig(level=default_level) + logging.warn("Cannot find logging configuration file!\n") +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/Reader.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,92 @@ +import json +import re +import logging +import codecs +import socket +from apollo.ApolloUser import ApolloUser +from util import santitizer + +class Reader(object): + + def __init__(self, input_json_file): + self.inputFile = input_json_file + self.args = self.loadJson() + + + def loadJson(self): + try: + data_file = codecs.open(self.inputFile, 'r', 'utf-8') + return json.load(data_file) + except IOError: + print "Cannot find JSON file\n" + exit(1) + + def getJBrowseHubDir(self): + try: + return self.args["jbrowse_hub"] + except KeyError: + print ("jbrowse_hub is not defined in the input file!") + exit(1) + + def getToolDir(self): + try: + return self.args["tool_directory"] + except KeyError: + print ("tool_directory is not defined in the input file!") + exit(1) + + def getExtFilesPath(self): + try: + return self.args["extra_files_path"] + except KeyError: + print ("extra_files_path is not defined in the input file!") + exit(1) + + def getUserEmail(self): + try: + return self.args["user_email"] + except KeyError: + print ("user_email is not defined in the input file!") + exit(1) + + def getDebugMode(self): + try: + return self.args["debug_mode"] + except KeyError: + print ("debug_mode is not defined in the input file!") + exit(1) + + def getPortNum(self): + try: + return self.args["port"] + except KeyError: + print ("port is not defined in the input file!") + exit(1) + + def getApolloHost(self): + #apollo_host = self.args.get("apollo_host") + hostname = socket.gethostname() + ip = socket.gethostbyname(hostname) + protocol = socket.getprotobyname(hostname) + apollo_host = str(protocol) + str(ip) + return apollo_host + + + def getSpeciesName(self): + species_name = santitizer.sanitize_name_input(self.args["species_name"]) + return species_name + + def getAction(self): + action = self.args.get("action") + return action + + def getAdminUser(self): + apollo_admin = self.args.get("apollo_admin") + return apollo_admin + + + def getOperationList(self): + l = self.args.get("operations") + return l + + \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/santitizer.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- + +""" +This class handles the subprocess calls of the different tools used +in HubArchiveCreator +""" + +import logging +import os +import subprocess +import sys +import string +import tempfile + + +def prefixTrackName(filename): + """ + santitize trackName. Because track name must begin with a letter and + contain only the following chars: [a-zA-Z0-9_]. + See the "track" Common settings at: + https://genome.ucsc.edu/goldenpath/help/trackDb/trackDbHub.html#bigPsl_-_Pairwise_Alignments + skip the santitization for cytoBandIdeo track + """ + if filename == 'cytoBandIdeo': + return filename + valid_chars = "_%s%s" % (string.ascii_letters, string.digits) + sanitize_name = ''.join([c if c in valid_chars else '_' for c in filename]) + sanitize_name = "gonramp_" + sanitize_name + return sanitize_name + +def sanitize_name_input(string_to_sanitize): + """ + Sanitize the string passed in parameter by replacing '/' and ' ' by '_' + + :param string_to_sanitize: + :return : + + :Example: + + >>> sanitize_name_input('this/is an//example') + this_is_an__example + """ + return string_to_sanitize \ + .replace("/", "_") \ + .replace(" ", "_") + +def sanitize_name_inputs(inputs_data): + """ + Sanitize value of the keys "name" of the dictionary passed in parameter. + + Because sometimes output from Galaxy, or even just file name, from user inputs, have spaces. + Also, it can contain '/' character and could break the use of os.path function. + + :param inputs_data: dict[string, dict[string, string]] + """ + for key in inputs_data: + inputs_data[key]["name"] = sanitize_name_input(inputs_data[key]["name"]) + +def sanitize_group_name(group_name): + return group_name.lower().replace(' ', '_') + +def sanitize_name(input_name): + """ + Galaxy will name all the files and dirs as *.dat, + the function can replace '.' to '_' for the dirs + """ + validChars = "_-%s%s" % (string.ascii_letters, string.digits) + sanitized_name = ''.join([c if c in validChars else '_' for c in input_name]) + return "gonramp_" + sanitized_name
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/subtools.py Fri Feb 16 10:57:13 2018 -0500 @@ -0,0 +1,220 @@ +#!/usr/bin/env python + +""" +This file include common used functions for converting file format to gff3 +""" +from collections import OrderedDict +import json +import subprocess +import os +import sys +import tempfile +import string +import logging + +class PopenError(Exception): + def __init__(self, cmd, error, return_code): + self.cmd = cmd + self.error = error + self.return_code = return_code + + def __str__(self): + message = "The subprocess {0} has returned the error: {1}.".format( + self.cmd, self.return_code) + message = ','.join( + (message, "Its error message is: {0}".format(self.error))) + return repr(message) + + +def _handleExceptionAndCheckCall(array_call, **kwargs): + """ + This class handle exceptions and call the tool. + It maps the signature of subprocess.check_call: + See https://docs.python.org/2/library/subprocess.html#subprocess.check_call + """ + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + shell = kwargs.get('shell', False) + stdin = kwargs.get('stdin', None) + + cmd = array_call[0] + + output = None + error = None + + # TODO: Check the value of array_call and <=[0] + logging.debug("Calling {0}:".format(cmd)) + logging.debug("%s", array_call) + logging.debug("---------") + + # TODO: Use universal_newlines option from Popen? + try: + p = subprocess.Popen(array_call, stdout=stdout, + stderr=stderr, shell=shell, stdin=stdin) + + # TODO: Change this because of possible memory issues => https://docs.python.org/2/library/subprocess.html#subprocess.Popen.communicate + + output, error = p.communicate() + + if stdout == subprocess.PIPE: + logging.debug("\t{0}".format(output)) + else: + logging.debug("\tOutput in file {0}".format(stdout.name)) + # If we detect an error from the subprocess, then we raise an exception + # TODO: Manage if we raise an exception for everything, or use CRITICAL etc... but not stop process + # TODO: The responsability of returning a sys.exit() should not be there, but up in the app. + if p.returncode: + if stderr == subprocess.PIPE: + raise PopenError(cmd, error, p.returncode) + else: + # TODO: To Handle properly with a design behind, if we received a option as a file for the error + raise Exception("Error when calling {0}. Error as been logged in your file {1}. Error code: {2}".format(cmd, stderr.name, p.returncode)) + + except OSError as e: + message = "The subprocess {0} has encountered an OSError: {1}".format( + cmd, e.strerror) + if e.filename: + message = '\n'.join( + (message, ", against this file: {0}".format(e.filename))) + logging.error(message) + sys.exit(-1) + except PopenError as p: + message = "The subprocess {0} has returned the error: {1}.".format( + p.cmd, p.return_code) + message = '\n'.join( + (message, "Its error message is: {0}".format(p.error))) + + logging.exception(message) + + sys.exit(p.return_code) + except Exception as e: + message = "The subprocess {0} has encountered an unknown error: {1}".format( + cmd, e) + logging.exception(message) + + sys.exit(-1) + return output + +def arrow_add_organism(organism_name, organism_dir, public=False): + array_call = ['arrow', 'organisms', 'add_organism', organism_name, organism_dir] + if public: + array_call.append('--public') + p = _handleExceptionAndCheckCall(array_call) + #p = subprocess.check_output(array_call) + return p + +def arrow_create_user(user_email, firstname, lastname, password, admin=False): + """ + Create a new user of Apollo, the default user_role is "user" + """ + array_call = ['arrow', 'users', 'create_user', user_email, firstname, lastname, password] + if admin: + array_call += ['--role', 'admin'] + p = _handleExceptionAndCheckCall(array_call) + j = json.loads(p) + if "userId" in j: + return j['userId'] + elif "error" in j: + logging.error("Got error message: %s", j['error']) + exit(-1) + + +def arrow_delete_user(user_email): + array_call = ['arrow', 'users', 'delete_user', user_email] + p = _handleExceptionAndCheckCall(array_call) + j = json.loads(p) + if "error" in j: + logging.error("Got error message: %s", j['error']) + exit(-1) + +def arrow_add_to_group(groupname, user_email): + if not arrow_get_groups(groupname): + arrow_create_group(groupname) + array_call = ['arrow', 'users', 'add_to_group', groupname, user_email] + p = _handleExceptionAndCheckCall(array_call) + j = json.loads(p) + if j != dict(): + logging.error("Error add user %s to group %s. The user doesn't exist", user_email, groupname) + + +def arrow_remove_from_group(groupname, user_email): + if arrow_get_groups(groupname): + array_call = ['arrow', 'users', 'remove_from_group', groupname, user_email] + p = _handleExceptionAndCheckCall(array_call) + j = json.loads(p) + if j != dict(): + logging.error("Error remove user %s to group %s. The user doesn't exist", user_email, groupname) + else: + logging.error("Group %s doesn't exist. Check if you spell the name correctly", groupname) + +def arrow_create_group(groupname): + if arrow_get_groups(groupname): + logging.error("Group %s already exist. Create a group with another name.", groupname) + array_call = ['arrow', 'groups', 'create_group', groupname] + p = _handleExceptionAndCheckCall(array_call) + +def arrow_get_groups(groupname): + array_call = ['arrow', 'groups', 'get_groups'] + p = _handleExceptionAndCheckCall(array_call) + all_groups = json.loads(p) + for g in all_groups: + if g['name'] == groupname: + return True + return False + +def arrow_update_organism_permissions(user_id, organism, **user_permissions): + array_call = ['arrow', 'users', 'update_organism_permissions', str(user_id), str(organism)] + admin = user_permissions.get("admin", False) + write = user_permissions.get("write", False) + read = user_permissions.get("read", False) + export = user_permissions.get("export", False) + if admin: + array_call.append('--administrate') + if write: + array_call.append('--write') + if read: + array_call.append('--read') + if export: + array_call.append('--export') + p = _handleExceptionAndCheckCall(array_call) + return p + +def arrow_get_users(user_email): + array_call = ['arrow', 'users', 'get_users'] + p = _handleExceptionAndCheckCall(array_call) + all_users = json.loads(p) + for d in all_users: + if d['username'] == user_email: + return d['userId'] + logging.error("Cannot find user %s", user_email) + +def arrow_get_organism(organism_name): + array_call= ['arrow', 'organisms', 'get_organisms', '--common_name', organism_name] + p = _handleExceptionAndCheckCall(array_call) + org = json.loads(p) + if 'error' not in org: + return org[0]['id'] + else: + logging.debug("Got error msg %s when look for organism %s.", org['error'], organism_name) + +def arrow_delete_organism(organism_id): + array_call = ['arrow', 'organisms', 'delete_organism', str(organism_id)] + p = _handleExceptionAndCheckCall(array_call) + return p + +def verify_user_login(username, password, apollo_host): + user_info = {'username': username, 'password': password} + array_call = ['curl', + '-b', 'cookies.txt', + '-c', 'cookies.txt', + '-H', 'Content-Type:application/json', + '-d', json.dumps(user_info), + apollo_host + '/Login?operation=login' + ] + p = _handleExceptionAndCheckCall(array_call) + msg = json.loads(p) + if 'error' in msg: + logging.error("The Authentication for user %s failed. Get error message %s", username, msg['error']) + exit(-1) + +