annotate env/lib/python3.9/site-packages/planemo/galaxy/config.py @ 0:4f3585e2f14b draft default tip

"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"

1 """Abstractions for setting up a Galaxy instance."""
2 from __future__ import absolute_import
3 from __future__ import print_function
4 
5 import abc
6 import contextlib
7 import os
8 import random
9 import shutil
10 import threading
11 from string import Template
12 from tempfile import mkdtemp
13 
14 from galaxy.containers.docker_model import DockerVolume
15 from galaxy.tool_util.deps import docker_util
16 from galaxy.util.commands import argv_to_str
17 from pkg_resources import parse_version
18 from six import (
19     add_metaclass,
20     iteritems
21 )
22 from six.moves import shlex_quote
23 
24 from planemo import git
25 from planemo.config import OptionSource
26 from planemo.deps import ensure_dependency_resolvers_conf_configured
27 from planemo.docker import docker_host_args
28 from planemo.galaxy.workflows import remote_runnable_to_workflow_id
29 from planemo.io import (
30     communicate,
31     kill_pid_file,
32     shell,
33     shell_join,
34     untar_to,
35     wait_on,
36     warn,
37     write_file,
38 )
39 from planemo.mulled import build_involucro_context
40 from planemo.shed import tool_shed_url
41 from planemo.virtualenv import DEFAULT_PYTHON_VERSION
42 from .api import (
43     DEFAULT_ADMIN_API_KEY,
44     gi,
45     user_api_key,
46 )
47 from .distro_tools import (
48     DISTRO_TOOLS_ID_TO_PATH
49 )
50 from .run import (
51     setup_common_startup_args,
52     setup_venv,
53 )
54 from .workflows import (
55     find_tool_ids,
56     import_workflow,
57     install_shed_repos,
58 )
59 
60 
61 NO_TEST_DATA_MESSAGE = (
62     "planemo couldn't find a target test-data directory, you should likely "
63     "create a test-data directory or pass an explicit path using --test_data."
64 )
65 
66 WEB_SERVER_CONFIG_TEMPLATE = """
67 [server:${server_name}]
68 use = egg:Paste#http
69 port = ${port}
70 host = ${host}
71 use_threadpool = True
72 threadpool_kill_thread_limit = 10800
73 [app:main]
74 paste.app_factory = galaxy.web.buildapp:app_factory
75 static_dir = static/
76 """
77 
78 TOOL_CONF_TEMPLATE = """<toolbox>
79   <tool file="data_source/upload.xml" />
80   ${tool_definition}
81 </toolbox>
82 """
83 
84 SHED_TOOL_CONF_TEMPLATE = """<?xml version="1.0"?>
85 <toolbox tool_path="${shed_tool_path}">
86 </toolbox>
87 """
88 
89 SHED_DATA_MANAGER_CONF_TEMPLATE = """<?xml version="1.0"?>
90 <data_managers>
91 </data_managers>
92 """
93 
94 EMPTY_JOB_METRICS_TEMPLATE = """<?xml version="1.0"?>
95 <job_metrics>
96 </job_metrics>
97 """
98 
99 TOOL_SHEDS_CONF = """<tool_sheds>
00   <tool_shed name="Target Shed" url="${shed_target_url}" />
01 </tool_sheds>
02 """
03 
04 JOB_CONFIG_LOCAL = """<job_conf>
05     <plugins>
06         <plugin id="planemo_runner" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
07     </plugins>
08     <handlers>
09         <handler id="main"/>
10     </handlers>
11     <destinations default="planemo_dest">
12         <destination id="planemo_dest" runner="planemo_runner">
13             <param id="require_container">${require_container}</param>
14             <param id="docker_enabled">${docker_enable}</param>
15             <param id="docker_sudo">${docker_sudo}</param>
16             <param id="docker_sudo_cmd">${docker_sudo_cmd}</param>
17             <param id="docker_cmd">${docker_cmd}</param>
18             ${docker_host_param}
19         </destination>
20         <destination id="upload_dest" runner="planemo_runner">
21             <param id="docker_enabled">false</param>
22         </destination>
23     </destinations>
24     <tools>
25         <tool id="upload1" destination="upload_dest" />
26     </tools>
27 </job_conf>
28 """
29 
30 LOGGING_TEMPLATE = """
31 ## Configure Python loggers.
32 [loggers]
33 keys = root,paste,displayapperrors,galaxydeps,galaxymasterapikey,galaxy
34 
35 [handlers]
36 keys = console
37 
38 [formatters]
39 keys = generic
40 
41 [logger_root]
42 level = WARN
43 handlers = console
44 
45 [logger_paste]
46 level = WARN
47 handlers = console
48 qualname = paste
49 propagate = 0
50 
51 [logger_galaxydeps]
52 level = DEBUG
53 handlers = console
54 qualname = galaxy.tools.deps
55 propagate = 0
56 
57 [logger_galaxymasterapikey]
58 level = WARN
59 handlers = console
60 qualname = galaxy.web.framework.webapp
61 propagate = 0
62 
63 [logger_displayapperrors]
64 level = ERROR
65 handlers =
66 qualname = galaxy.datatypes.display_applications.application
67 propagate = 0
68 
69 [logger_galaxy]
70 level = ${log_level}
71 handlers = console
72 qualname = galaxy
73 propagate = 0
74 
75 [handler_console]
76 class = StreamHandler
77 args = (sys.stderr,)
78 level = DEBUG
79 formatter = generic
80 
81 [formatter_generic]
82 format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
83 """
84 
85 REFGENIE_CONFIG_TEMPLATE = """
86 config_version: 0.3
87 genome_folder: '%s'
88 genome_servers: ['http://refgenomes.databio.org']
89 genomes: null
90 """
91 
92 EMPTY_TOOL_CONF_TEMPLATE = """<toolbox></toolbox>"""
93 
94 DEFAULT_GALAXY_BRANCH = "master"
95 DEFAULT_GALAXY_SOURCE = "https://github.com/galaxyproject/galaxy"
96 CWL_GALAXY_SOURCE = "https://github.com/common-workflow-language/galaxy"
97 
98 DATABASE_LOCATION_TEMPLATE = "sqlite:///%s?isolation_level=IMMEDIATE"
99 
00 COMMAND_STARTUP_COMMAND = './scripts/common_startup.sh ${COMMON_STARTUP_ARGS}'
01 
02 CLEANUP_IGNORE_ERRORS = True
03 DEFAULT_GALAXY_BRAND = 'Configured by Planemo'
04 DEFAULT_TOOL_INSTALL_TIMEOUT = 60 * 60 * 1
05 UNINITIALIZED = object()
06 
07 
08 @contextlib.contextmanager
09 def galaxy_config(ctx, runnables, **kwds):
10     """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
11     c = local_galaxy_config
12     if kwds.get("dockerize", False):
13         c = docker_galaxy_config
14     elif kwds.get("external", False):
15         c = external_galaxy_config
16     log_thread = None
17     try:
18         with c(ctx, runnables, **kwds) as config:
19             if kwds.get('daemon'):
20                 log_thread = threading.Thread(target=read_log, args=(ctx, config.log_file))
21                 log_thread.daemon = True
22                 log_thread.start()
23             yield config
24     finally:
25         if log_thread:
26             log_thread.join(1)
27 
28 
29 def read_log(ctx, log_path):
30     log_fh = None
31     e = threading.Event()
32     try:
33         while e:
34             if os.path.exists(log_path):
35                 if not log_fh:
36                     # Open in append so we start at the end of the log file
37                     log_fh = open(log_path, 'a+')
38                 log_lines = log_fh.read()
39                 if log_lines:
40                     ctx.log(log_lines)
41             e.wait(1)
42     finally:
43         if log_fh:
44             log_fh.close()
45 
46 
47 def simple_docker_volume(path):
48     path = os.path.abspath(path)
49     return DockerVolume("%s:%s:rw" % (path, path))
50 
51 
52 @contextlib.contextmanager
53 def docker_galaxy_config(ctx, runnables, for_tests=False, **kwds):
54     """Set up a ``GalaxyConfig`` for Docker container."""
55     test_data_dir = _find_test_data(runnables, **kwds)
56 
57     with _config_directory(ctx, **kwds) as config_directory:
58         def config_join(*args):
59             return os.path.join(config_directory, *args)
60 
61         ensure_dependency_resolvers_conf_configured(ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml"))
62         _handle_job_metrics(config_directory, kwds)
63         _handle_refgenie_config(config_directory, kwds)
64 
65         shed_tool_conf = "config/shed_tool_conf.xml"
66         all_tool_paths = _all_tool_paths(runnables, **kwds)
67 
68         tool_directories = set([])  # Things to mount...
69         for tool_path in all_tool_paths:
70             directory = os.path.dirname(os.path.normpath(tool_path))
71             if os.path.exists(directory):
72                 tool_directories.add(directory)
73 
74         # TODO: remap these.
75         tool_volumes = []
76         for tool_directory in tool_directories:
77             volume = simple_docker_volume(tool_directory)
78             tool_volumes.append(volume)
79 
80         empty_tool_conf = config_join("empty_tool_conf.xml")
81 
82         tool_conf = config_join("tool_conf.xml")
83 
84         shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
85         _ensure_directory(shed_tool_path)
86 
87         sheds_config_path = _configure_sheds_config_file(
88             ctx, config_directory, **kwds
89         )
90         port = _get_port(kwds)
91         properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests)
92         _handle_container_resolution(ctx, kwds, properties)
93         master_api_key = _get_master_api_key(kwds)
94 
95         template_args = dict(
96             shed_tool_path=shed_tool_path,
97             tool_conf=tool_conf,
98         )
99         tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
00 
01         _write_tool_conf(ctx, all_tool_paths, tool_conf)
02         write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)
03 
04         properties.update(dict(
05             tool_config_file=tool_config_file,
06             tool_sheds_config_file=sheds_config_path,
07             migrated_tools_config=empty_tool_conf,
08         ))
09 
10         server_name = "planemo%d" % random.randint(0, 100000)
11 
12         # Value substitutions in Galaxy properties - for consistency with
13         # non-Dockerized version.
14         template_args = dict(
15         )
16         env = _build_env_for_galaxy(properties, template_args)
17         env["NONUSE"] = "nodejs,proftp,reports"
18         if ctx.verbose:
19             env["GALAXY_LOGGING"] = "full"
20 
21         # TODO: setup FTP upload dir and disable FTP server in container.
22         _build_test_env(properties, env)
23 
24         docker_target_kwds = docker_host_args(**kwds)
25         volumes = tool_volumes + [simple_docker_volume(config_directory)]
26         export_directory = kwds.get("export_directory", None)
27         if export_directory is not None:
28             volumes.append(DockerVolume("%s:/export:rw" % export_directory))
29 
30         # TODO: Allow this to real Docker volumes and allow multiple.
31         extra_volume = kwds.get("docker_extra_volume")
32         if extra_volume:
33             volumes.append(simple_docker_volume(extra_volume))
34         yield DockerGalaxyConfig(
35             ctx,
36             config_directory,
37             env,
38             test_data_dir,
39             port,
40             server_name,
41             master_api_key,
42             runnables,
43             docker_target_kwds=docker_target_kwds,
44             volumes=volumes,
45             export_directory=export_directory,
46             kwds=kwds,
47         )
48 
49 
50 @contextlib.contextmanager
51 def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
52     """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
53     test_data_dir = _find_test_data(runnables, **kwds)
54     tool_data_table = _find_tool_data_table(
55         runnables,
56         test_data_dir=test_data_dir,
57         **kwds
58     )
59     data_manager_config_paths = [r.data_manager_conf_path for r in runnables if r.data_manager_conf_path]
60     galaxy_root = _find_galaxy_root(ctx, **kwds)
61     install_galaxy = kwds.get("install_galaxy", False)
62     if galaxy_root is not None:
63         if os.path.isdir(galaxy_root) and not os.listdir(galaxy_root):
64             os.rmdir(galaxy_root)
65         if os.path.isdir(galaxy_root) and install_galaxy:
66             raise Exception("%s is an existing non-empty directory, cannot install Galaxy again" % galaxy_root)
67 
68     # Duplicate block in docker variant above.
69     if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
70         if ctx.get_option_source("docker") != OptionSource.cli:
71             kwds["docker"] = True
72         else:
73             raise Exception("Specified no docker and mulled containers together.")
74 
75     with _config_directory(ctx, **kwds) as config_directory:
76         def config_join(*args):
77             return os.path.join(config_directory, *args)
78 
79         install_env = {}
80         if kwds.get('galaxy_skip_client_build', True):
81             install_env['GALAXY_SKIP_CLIENT_BUILD'] = '1'
82         if galaxy_root is None:
83             galaxy_root = config_join("galaxy-dev")
84         if not os.path.isdir(galaxy_root):
85             _build_eggs_cache(ctx, install_env, kwds)
86             _install_galaxy(ctx, galaxy_root, install_env, kwds)
87 
88         if parse_version(kwds.get('galaxy_python_version') or DEFAULT_PYTHON_VERSION) >= parse_version('3'):
89             # on python 3 we use gunicorn,
90             # which requires 'main' as server name
91             server_name = 'main'
92         else:
93             server_name = "planemo%d" % random.randint(0, 100000)
94         # Once we don't have to support earlier than 18.01 - try putting these files
95         # somewhere better than with Galaxy.
96         log_file = "%s.log" % server_name
97         pid_file = "%s.pid" % server_name
98         ensure_dependency_resolvers_conf_configured(ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml"))
99         _handle_job_config_file(config_directory, server_name, kwds)
00         _handle_job_metrics(config_directory, kwds)
01         _handle_refgenie_config(config_directory, kwds)
02         file_path = kwds.get("file_path") or config_join("files")
03         _ensure_directory(file_path)
04 
05         tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
06         _ensure_directory(tool_dependency_dir)
07 
08         shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
09         all_tool_paths = _all_tool_paths(runnables, **kwds)
10         empty_tool_conf = config_join("empty_tool_conf.xml")
11 
12         tool_conf = config_join("tool_conf.xml")
13 
14         shed_data_manager_config_file = config_join("shed_data_manager_conf.xml")
15 
16         shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
17         _ensure_directory(shed_tool_path)
18 
19         sheds_config_path = _configure_sheds_config_file(
20             ctx, config_directory, **kwds
21         )
22 
23         database_location = config_join("galaxy.sqlite")
24         master_api_key = _get_master_api_key(kwds)
25         dependency_dir = os.path.join(config_directory, "deps")
26         _ensure_directory(shed_tool_path)
27         port = _get_port(kwds)
28         template_args = dict(
29             port=port,
30             host=kwds.get("host", "127.0.0.1"),
31             server_name=server_name,
32             temp_directory=config_directory,
33             shed_tool_path=shed_tool_path,
34             database_location=database_location,
35             tool_conf=tool_conf,
36             debug=kwds.get("debug", "true"),
37             id_secret=kwds.get("id_secret", "test_secret"),
38             log_level="DEBUG" if ctx.verbose else "INFO",
39         )
40         tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
41         # Setup both galaxy_email and older test user [e-mail]
42         # as admins for command_line, etc...
43         properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests)
44         properties.update(dict(
45             server_name="main",
46             ftp_upload_dir_template="${ftp_upload_dir}",
47             ftp_upload_purge="False",
48             ftp_upload_dir=test_data_dir or os.path.abspath('.'),
49             ftp_upload_site="Test Data",
50             check_upload_content="False",
51             tool_dependency_dir=dependency_dir,
52             file_path=file_path,
53             new_file_path="${temp_directory}/tmp",
54             tool_config_file=tool_config_file,
55             tool_sheds_config_file=sheds_config_path,
56             manage_dependency_relationships="False",
57             job_working_directory="${temp_directory}/job_working_directory",
58             template_cache_path="${temp_directory}/compiled_templates",
59             citation_cache_type="file",
60             citation_cache_data_dir="${temp_directory}/citations/data",
61             citation_cache_lock_dir="${temp_directory}/citations/lock",
62             database_auto_migrate="True",
63             enable_beta_tool_formats="True",
64             id_secret="${id_secret}",
65             log_level="${log_level}",
66             debug="${debug}",
67             watch_tools="auto",
68             default_job_shell="/bin/bash",  # For conda dependency resolution
69             tool_data_table_config_path=tool_data_table,
70             data_manager_config_file=",".join(data_manager_config_paths) or None,  # without 'or None' may raise IOError in galaxy (see #946)
71             integrated_tool_panel_config=("${temp_directory}/"
72                                           "integrated_tool_panel_conf.xml"),
73             migrated_tools_config=empty_tool_conf,
74             test_data_dir=test_data_dir,  # TODO: make gx respect this
75             shed_data_manager_config_file=shed_data_manager_config_file,
76         ))
77         _handle_container_resolution(ctx, kwds, properties)
78         write_file(config_join("logging.ini"), _sub(LOGGING_TEMPLATE, template_args))
79         properties["database_connection"] = _database_connection(database_location, **kwds)
80 
81         _handle_kwd_overrides(properties, kwds)
82 
83         # TODO: consider following property
84         # watch_tool = False
85         # datatypes_config_file = config/datatypes_conf.xml
86         # welcome_url = /static/welcome.html
87         # logo_url = /
88         # sanitize_all_html = True
89         # serve_xss_vulnerable_mimetypes = False
90         # track_jobs_in_database = None
91         # outputs_to_working_directory = False
92         # retry_job_output_collection = 0
93 
94         env = _build_env_for_galaxy(properties, template_args)
95         env.update(install_env)
96         _build_test_env(properties, env)
97         env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf
98         env['GALAXY_TEST_DBURI'] = properties["database_connection"]
99 
00         env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
01         env["GALAXY_TEST_LOGGING_CONFIG"] = config_join("logging.ini")
02         env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
03         # Following are needed in 18.01 to prevent Galaxy from changing log and pid.
04         # https://github.com/galaxyproject/planemo/issues/788
05         env["GALAXY_LOG"] = log_file
06         env["GALAXY_PID"] = pid_file
07         web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
08         write_file(config_join("galaxy.ini"), web_config)
09         _write_tool_conf(ctx, all_tool_paths, tool_conf)
10         write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)
11 
12         shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
13         # Write a new shed_tool_conf.xml if needed.
14         write_file(shed_tool_conf, shed_tool_conf_contents, force=False)
15 
16         write_file(shed_data_manager_config_file, SHED_DATA_MANAGER_CONF_TEMPLATE)
17 
18         yield LocalGalaxyConfig(
19             ctx,
20             config_directory,
21             env,
22             test_data_dir,
23             port,
24             server_name,
25             master_api_key,
26             runnables,
27             galaxy_root,
28             kwds,
29         )
30 
31 
32 def _all_tool_paths(runnables, **kwds):
33     tool_paths = [r.path for r in runnables if r.has_tools and not r.data_manager_conf_path]
34     all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
35     for runnable in runnables:
36         if runnable.type.name == "galaxy_workflow":
37             tool_ids = find_tool_ids(runnable.path)
38             for tool_id in tool_ids:
39                 tool_paths = DISTRO_TOOLS_ID_TO_PATH.get(tool_id)
40                 if tool_paths:
41                     if isinstance(tool_paths, str):
42                         tool_paths = [tool_paths]
43                     all_tool_paths.extend(tool_paths)
44 
45     return all_tool_paths
46 
47 
48 def _shared_galaxy_properties(config_directory, kwds, for_tests):
49     """Setup properties useful for local and Docker Galaxy instances.
50 
51     Most things related to paths, etc... are very different between Galaxy
52     modalities and many taken care of internally to the container in that mode.
53     But this method sets up API stuff, tool, and job stuff that can be shared.
54     """
55     master_api_key = _get_master_api_key(kwds)
56     user_email = _user_email(kwds)
57     properties = {
58         'master_api_key': master_api_key,
59         'admin_users': "%s,[e-mail]" % user_email,
60         'expose_dataset_path': "True",
61         'cleanup_job': 'never',
62         'collect_outputs_from': "job_working_directory",
63         'allow_path_paste': "True",
64         'check_migrate_tools': "False",
65         'use_cached_dependency_manager': str(kwds.get("conda_auto_install", False)),
66         'brand': kwds.get("galaxy_brand", DEFAULT_GALAXY_BRAND),
67         'strict_cwl_validation': str(not kwds.get("non_strict_cwl", False)),
68     }
69     if kwds.get("galaxy_single_user", True):
70         properties['single_user'] = user_email
71 
72     if for_tests:
73         empty_dir = os.path.join(config_directory, "empty")
74         _ensure_directory(empty_dir)
75         properties["tour_config_dir"] = empty_dir
76         properties["interactive_environment_plugins_directory"] = empty_dir
77         properties["visualization_plugins_directory"] = empty_dir
78         properties["refgenie_config_file"] = kwds.get('refgenie_config_file', '')
79     return properties
80 
81 
82 @contextlib.contextmanager
83 def external_galaxy_config(ctx, runnables, for_tests=False, **kwds):
84     yield BaseGalaxyConfig(
85         ctx=ctx,
86         galaxy_url=kwds.get("galaxy_url", None),
87         master_api_key=_get_master_api_key(kwds),
88         user_api_key=kwds.get("galaxy_user_key", None),
89         runnables=runnables,
90         kwds=kwds
91     )
92 
93 
94 def _get_master_api_key(kwds):
95     master_api_key = kwds.get("galaxy_admin_key") or DEFAULT_ADMIN_API_KEY
96     return master_api_key
97 
98 
99 def _get_port(kwds):
00     port = int(kwds.get("port", 9090))
01     return port
02 
03 
04 def _user_email(kwds):
05     user_email = kwds.get("galaxy_email")
06     return user_email
07 
08 
09 @contextlib.contextmanager
10 def _config_directory(ctx, **kwds):
11     config_directory = kwds.get("config_directory", None)
12     created_config_directory = False
13     if not config_directory:
14         created_config_directory = True
15         config_directory = os.path.realpath(mkdtemp())
16         ctx.vlog("Created directory for Galaxy configuration [%s]" % config_directory)
17     try:
18         yield config_directory
19     finally:
20         cleanup = not kwds.get("no_cleanup", False)
21         if created_config_directory and cleanup:
22             shutil.rmtree(config_directory)
23 
24 
25 @add_metaclass(abc.ABCMeta)
26 class GalaxyInterface(object):
27     """Abstraction around a Galaxy instance.
28 
29     Description of a Galaxy instance and how to interact with it - this could
30     potentially be a remote, already running instance or an instance Planemo manages
31     to execute some task(s).
32     """
33 
34     @abc.abstractproperty
35     def gi(self):
36         """Return an admin bioblend Galaxy instance for API interactions."""
37 
38     @abc.abstractproperty
39     def user_gi(self):
40         """Return a user-backed bioblend Galaxy instance for API interactions."""
41 
42     @abc.abstractmethod
43     def install_repo(self, *args, **kwds):
44         """Install specified tool shed repository."""
45 
46     @abc.abstractproperty
47     def tool_shed_client(self):
48         """Return a admin bioblend tool shed client."""
49 
50     @abc.abstractmethod
51     def wait_for_all_installed(self):
52         """Wait for all queued up repositories installs to complete."""
53 
54     @abc.abstractmethod
55     def install_workflows(self):
56         """Install all workflows configured with these planemo arguments."""
57 
58     @abc.abstractmethod
59     def workflow_id(self, path):
60         """Get installed workflow API ID for input path."""
61 
62     @abc.abstractproperty
63     def version_major(self):
64         """Return target Galaxy version."""
65 
66     @abc.abstractproperty
67     def user_api_config(self):
68         """Return the API indicated configuration for user session.
69 
70         Calling .config.get_config() with admin GI session would yield
71         a different object (admins have different view of Galaxy's
72         configuration).
73         """
74 
75     @property
76     def user_is_admin(self):
77         return self.user_api_config["is_admin_user"]
78 
79 
80 @add_metaclass(abc.ABCMeta)
81 class GalaxyConfig(GalaxyInterface):
82     """Specialization of GalaxyInterface for Galaxy instances Planemo manages itself.
83 
84     This assumes more than an API connection is available - Planemo needs to be able to
85     start and stop the Galaxy instance, recover logs, etc... There are currently two
86     implementations - a locally executed Galaxy and one running inside a Docker containe
87     """
88 
89     @abc.abstractproperty
90     def kill(self):
91         """Stop the running instance."""
92 
93     @abc.abstractmethod
94     def startup_command(self, ctx, **kwds):
95         """Return a shell command used to startup this instance.
96 
97         Among other common planmo kwds, this should respect the
98         ``daemon`` keyword.
99         """
00 
01     @abc.abstractproperty
02     def log_contents(self):
03         """Retrieve text of log for running Galaxy instance."""
04 
05     @abc.abstractmethod
06     def cleanup(self):
07         """Cleanup allocated resources to run this instance."""
08 
09     @abc.abstractproperty
10     def use_path_paste(self):
11         """Use path paste to upload data.
12 
13         This will only be an option if the target user key is an
14         admin user key.
15         """
16 
17 
18 class BaseGalaxyConfig(GalaxyInterface):
19 
20     def __init__(
21         self,
22         ctx,
23         galaxy_url,
24         master_api_key,
25         user_api_key,
26         runnables,
27         kwds,
28     ):
29         self._ctx = ctx
30         self.galaxy_url = galaxy_url
31         self.master_api_key = master_api_key
32         self._user_api_key = user_api_key
33         self.runnables = runnables
34         self._kwds = kwds
35         self._workflow_ids = {}
36 
37         self._target_version = UNINITIALIZED
38         self._target_user_config = UNINITIALIZED
39 
40     @property
41     def gi(self):
42         assert self.galaxy_url
43         return gi(url=self.galaxy_url, key=self.master_api_key)
44 
45     @property
46     def user_gi(self):
47         user_api_key = self.user_api_key
48         assert user_api_key
49         return self._gi_for_key(user_api_key)
50 
51     @property
52     def user_api_key(self):
53         # TODO: thread-safe
54         if self._user_api_key is None:
55             # TODO: respect --galaxy_email - seems like a real bug
56             self._user_api_key = user_api_key(self.gi)
57 
58         return self._user_api_key
59 
60     def _gi_for_key(self, key):
61         assert self.galaxy_url
62         return gi(url=self.galaxy_url, key=key)
63 
64     def install_repo(self, *args, **kwds):
65         self.tool_shed_client.install_repository_revision(
66             *args, **kwds
67         )
68 
69     @property
70     def tool_shed_client(self):
71         return self.gi.toolShed
72 
73     def wait_for_all_installed(self):
74         def status_ready(repo):
75             status = repo["status"]
76             if status in ["Installing", "New"]:
77                 return None
78             if status == "Installed":
79                 return True
80             raise Exception("Error installing repo status is %s" % status)
81 
82         def ready():
83             repos = self.tool_shed_client.get_repositories()
84             ready = all(map(status_ready, repos))
85             return ready or None
86 
87         wait_on(ready, "galaxy tool installation", timeout=DEFAULT_TOOL_INSTALL_TIMEOUT)
88 
89     def install_workflows(self):
90         for runnable in self.runnables:
91             if runnable.type.name in ["galaxy_workflow", "cwl_workflow"] and not runnable.is_remote_workflow_uri:
92                 self._install_workflow(runnable)
93 
94     def _install_workflow(self, runnable):
95         if self._kwds.get("shed_install") and (self._kwds.get("engine") != "external_galaxy" or self._kwds.get("galaxy_admin_key")):
96             install_shed_repos(runnable,
97                                self.gi,
98                                self._kwds.get("ignore_dependency_problems", False),
99                                self._kwds.get("install_tool_dependencies", False),
00                                self._kwds.get("install_resolver_dependencies", True),
01                                self._kwds.get("install_repository_dependencies", True))
02 
03         default_from_path = self._kwds.get("workflows_from_path", False)
04         # TODO: Allow serialization so this doesn't need to assume a
05         # shared filesystem with Galaxy server.
06         from_path = default_from_path or (runnable.type.name == "cwl_workflow")
07         workflow = import_workflow(
08             runnable.path, admin_gi=self.gi, user_gi=self.user_gi, from_path=from_path
09         )
10         self._workflow_ids[runnable.path] = workflow["id"]
11 
12     def workflow_id_for_runnable(self, runnable):
13         if runnable.is_remote_workflow_uri:
14             workflow_id = remote_runnable_to_workflow_id(runnable)
15         else:
16             workflow_id = self.workflow_id(runnable.path)
17         return workflow_id
18 
19     def workflow_id(self, path):
20         return self._workflow_ids[path]
21 
22     @property
23     def use_path_paste(self):
24         option = self._kwds.get("paste_test_data_paths")
25         if option is None:
26             return self.default_use_path_paste
27         else:
28             return option
29 
30     @property
31     def default_use_path_paste(self):
32         return False
33 
34     @property
35     def version_major(self):
36         """Return target Galaxy version."""
37         if self._target_version is UNINITIALIZED:
38             self._target_version = self.user_gi.config.get_version()["version_major"]
39         return self._target_version
40 
41     @property
42     def user_api_config(self):
43         """Return the API indicated configuration for user session."""
44         if self._target_user_config is UNINITIALIZED:
45             self._target_user_config = self.user_gi.config.get_config()
46         return self._target_user_config
47 
48 
49 class BaseManagedGalaxyConfig(BaseGalaxyConfig):
50 
51     def __init__(
52         self,
53         ctx,
54         config_directory,
55         env,
56         test_data_dir,
57         port,
58         server_name,
59         master_api_key,
60         runnables,
61         kwds,
62     ):
63         galaxy_url = "http://localhost:%d" % port
64         super(BaseManagedGalaxyConfig, self).__init__(
65             ctx=ctx,
66             galaxy_url=galaxy_url,
67             master_api_key=master_api_key,
68             user_api_key=None,
69             runnables=runnables,
70             kwds=kwds
71         )
72         self.config_directory = config_directory
73         self.env = env
74         self.test_data_dir = test_data_dir
75         self.port = port
76         self.server_name = server_name
77 
78 
79 class DockerGalaxyConfig(BaseManagedGalaxyConfig):
80     """A :class:`GalaxyConfig` description of a Dockerized Galaxy instance."""
81 
82     def __init__(
83         self,
84         ctx,
85         config_directory,
86         env,
87         test_data_dir,
88         port,
89         server_name,
90         master_api_key,
91         runnables,
92         docker_target_kwds,
93         volumes,
94         export_directory,
95         kwds,
96     ):
97         super(DockerGalaxyConfig, self).__init__(
98             ctx,
99             config_directory,
00             env,
01             test_data_dir,
02             port,
03             server_name,
04             master_api_key,
05             runnables,
06             kwds,
07         )
08         self.docker_target_kwds = docker_target_kwds
09         self.volumes = volumes
10         self.export_directory = export_directory
11 
12     def kill(self):
13         """Kill planemo container..."""
14         kill_command = docker_util.kill_command(
15             self.server_name,
16             **self.docker_target_kwds
17         )
18         return shell(kill_command)
19 
20     def startup_command(self, ctx, **kwds):
21         """Return a shell command used to startup this instance.
22 
23         Among other common planmo kwds, this should respect the
24         ``daemon`` keyword.
25         """
26         daemon = kwds.get("daemon", False)
27         daemon_str = "" if not daemon else " -d"
28         docker_run_extras = "-p %s:80%s" % (self.port, daemon_str)
29         env_directives = ["%s='%s'" % item for item in self.env.items()]
30         image = kwds.get("docker_galaxy_image", "bgruening/galaxy-stable")
31         run_command = docker_util.build_docker_run_command(
32             "", image,
33             interactive=False,
34             env_directives=env_directives,
35             working_directory=None,
36             name=self.server_name,
37             run_extra_arguments=docker_run_extras,
38             set_user=False,
39             volumes=self.volumes,
40             **self.docker_target_kwds
41         )
42         chmod_command = [
43             "chmod",
44             "-R",
45             "o+rwx",
46             self.config_directory,
47         ]
48         if self.export_directory:
49             chmod_command.append(self.export_directory)
50 
51         return shell_join(
52             argv_to_str(chmod_command),
53             run_command,
54         )
55 
56     @property
57     def log_contents(self):
58         logs_command = docker_util.logs_command(
59             self.server_name,
60             **self.docker_target_kwds
61         )
62         output, _ = communicate(
63             logs_command
64         )
65         return output
66 
67     def cleanup(self):
68         shutil.rmtree(self.config_directory, CLEANUP_IGNORE_ERRORS)
69 
70 
71 class LocalGalaxyConfig(BaseManagedGalaxyConfig):
72     """A local, non-containerized implementation of :class:`GalaxyConfig`."""
73 
74     def __init__(
75         self,
76         ctx,
77         config_directory,
78         env,
79         test_data_dir,
80         port,
81         server_name,
82         master_api_key,
83         runnables,
84         galaxy_root,
85         kwds,
86     ):
87         super(LocalGalaxyConfig, self).__init__(
88             ctx,
89             config_directory,
90             env,
91             test_data_dir,
92             port,
93             server_name,
94             master_api_key,
95             runnables,
96             kwds,
97         )
98         self.galaxy_root = galaxy_root
99 
00     def kill(self):
01         if self._ctx.verbose:
02             shell(["ps", "ax"])
03             exists = os.path.exists(self.pid_file)
04             print("Killing pid file [%s]" % self.pid_file)
05             print("pid_file exists? [%s]" % exists)
06             if exists:
07                 print("pid_file contents are [%s]" % open(self.pid_file, "r").read())
08         kill_pid_file(self.pid_file)
09 
10     def startup_command(self, ctx, **kwds):
11         """Return a shell command used to startup this instance.
12 
13         Among other common planemo kwds, this should respect the
14         ``daemon`` keyword.
15         """
16         daemon = kwds.get("daemon", False)
17         # TODO: Allow running dockerized Galaxy here instead.
18         setup_venv_command = setup_venv(ctx, kwds)
19         run_script = "%s $COMMON_STARTUP_ARGS" % shlex_quote(os.path.join(self.galaxy_root, "run.sh"))
20         if daemon:
21             run_script += " --daemon"
22             self.env["GALAXY_RUN_ALL"] = "1"
23         else:
24             run_script += " --server-name %s" % shlex_quote(self.server_name)
25         server_ini = os.path.join(self.config_directory, "galaxy.ini")
26         self.env["GALAXY_CONFIG_FILE"] = server_ini
27         if parse_version(kwds.get('galaxy_python_version') or DEFAULT_PYTHON_VERSION) >= parse_version('3'):
28             # We need to start under gunicorn
29             self.env['APP_WEBSERVER'] = 'gunicorn'
30             self.env['GUNICORN_CMD_ARGS'] = "--timeout={timeout} --capture-output --bind={host}:{port} --name={server_name}".format(
31                 timeout=DEFAULT_TOOL_INSTALL_TIMEOUT,
32                 host=kwds.get('host', '127.0.0.1'),
33                 port=kwds['port'],
34                 server_name=self.server_name,
35             )
36         cd_to_galaxy_command = ['cd', self.galaxy_root]
37         return shell_join(
38             cd_to_galaxy_command,
39             setup_venv_command,
40             setup_common_startup_args(),
41             run_script,
42         )
43 
44     @property
45     def log_file(self):
46         """Log file used when planemo serves this Galaxy instance."""
47         file_name = "%s.log" % self.server_name
48         return os.path.join(self.galaxy_root, file_name)
49 
50     @property
51     def pid_file(self):
52         pid_file_name = "%s.pid" % self.server_name
53         return os.path.join(self.galaxy_root, pid_file_name)
54 
55     @property
56     def log_contents(self):
57         if not os.path.exists(self.log_file):
58             return ""
59         with open(self.log_file, "r") as f:
60             return f.read()
61 
62     def cleanup(self):
63         shutil.rmtree(self.config_directory, CLEANUP_IGNORE_ERRORS)
64 
65     @property
66     def default_use_path_paste(self):
67         # If Planemo started a local, native Galaxy instance assume files URLs can be
68         # pasted.
69         return self.user_is_admin
70 
71 
72 def _database_connection(database_location, **kwds):
73     default_connection = DATABASE_LOCATION_TEMPLATE % database_location
74     database_connection = kwds.get("database_connection") or default_connection
75     return database_connection
76 
77 
78 def _find_galaxy_root(ctx, **kwds):
79     root_prop = "galaxy_root"
80     cwl = kwds.get("cwl", False)
81     if cwl:
82         root_prop = "cwl_galaxy_root"
83     galaxy_root = kwds.get(root_prop, None)
84     if galaxy_root:
85         return galaxy_root
86     else:
87         par_dir = os.getcwd()
88         while True:
89             run = os.path.join(par_dir, "run.sh")
90             config = os.path.join(par_dir, "config")
91             if os.path.isfile(run) and os.path.isdir(config):
92                 return par_dir
93             new_par_dir = os.path.dirname(par_dir)
94             if new_par_dir == par_dir:
95                 break
96             par_dir = new_par_dir
97     return None
98 
99 
00 def _find_test_data(runnables, **kwds):
01     test_data_search_path = "."
02     runnables = [r for r in runnables if r.has_tools]
03     if len(runnables) > 0:
04         test_data_search_path = runnables[0].test_data_search_path
05 
06     # Find test data directory associated with path.
07     test_data = kwds.get("test_data", None)
08     if test_data:
09         return os.path.abspath(test_data)
10     else:
11         test_data = _search_tool_path_for(test_data_search_path, "test-data")
12         if test_data:
13             return test_data
14     warn(NO_TEST_DATA_MESSAGE)
15     return None
16 
17 
18 def _find_tool_data_table(runnables, test_data_dir, **kwds):
19     tool_data_search_path = "."
20     runnables = [r for r in runnables if r.has_tools]
21     if len(runnables) > 0:
22         tool_data_search_path = runnables[0].tool_data_search_path
23 
24     tool_data_table = kwds.get("tool_data_table", None)
25     if tool_data_table:
26         return os.path.abspath(tool_data_table)
27     else:
28         extra_paths = [test_data_dir] if test_data_dir else []
29         return _search_tool_path_for(
30             tool_data_search_path,
31             "tool_data_table_conf.xml.test",
32             extra_paths,
33         ) or _search_tool_path_for(  # if all else fails just use sample
34             tool_data_search_path,
35             "tool_data_table_conf.xml.sample"
36         )
37 
38 
39 def _search_tool_path_for(path, target, extra_paths=None):
40     """Check for presence of a target in different artifact directories."""
41     if extra_paths is None:
42         extra_paths = []
43     if not os.path.isdir(path):
44         tool_dir = os.path.dirname(path)
45     else:
46         tool_dir = path
47     possible_dirs = [tool_dir, "."] + extra_paths
48     for possible_dir in possible_dirs:
49         possible_path = os.path.join(possible_dir, target)
50         if os.path.exists(possible_path):
51             return os.path.abspath(possible_path)
52     return None
53 
54 
55 def _configure_sheds_config_file(ctx, config_directory, **kwds):
56     if "shed_target" not in kwds:
57         kwds = kwds.copy()
58         kwds["shed_target"] = "toolshed"
59     shed_target_url = tool_shed_url(ctx, **kwds)
60     contents = _sub(TOOL_SHEDS_CONF, {"shed_target_url": shed_target_url})
61     tool_sheds_conf = os.path.join(config_directory, "tool_sheds_conf.xml")
62     write_file(tool_sheds_conf, contents)
63     return tool_sheds_conf
64 
65 
66 def _tool_conf_entry_for(tool_paths):
67     tool_definitions = ""
68     for tool_path in tool_paths:
69         if os.path.isdir(tool_path):
70             tool_definitions += '''<tool_dir dir="%s" />''' % tool_path
71         else:
72             tool_definitions += '''<tool file="%s" />''' % tool_path
73     return tool_definitions
74 
75 
76 def _install_galaxy(ctx, galaxy_root, env, kwds):
77     if not kwds.get("no_cache_galaxy", False):
78         _install_galaxy_via_git(ctx, galaxy_root, env, kwds)
79     else:
80         _install_galaxy_via_download(ctx, galaxy_root, env, kwds)
81 
82 
83 def _install_galaxy_via_download(ctx, galaxy_root, env, kwds):
84     branch = _galaxy_branch(kwds)
85     untar_to("https://codeload.github.com/galaxyproject/galaxy/tar.gz/" + branch, tar_args=['-xvzf', '-', 'galaxy-' + branch], dest_dir=galaxy_root)
86     _install_with_command(ctx, galaxy_root, env, kwds)
87 
88 
89 def _install_galaxy_via_git(ctx, galaxy_root, env, kwds):
90     gx_repo = _ensure_galaxy_repository_available(ctx, kwds)
91     branch = _galaxy_branch(kwds)
92     command = git.command_clone(ctx, gx_repo, galaxy_root, branch=branch)
93     exit_code = shell(command, env=env)
94     if exit_code != 0:
95         raise Exception("Failed to glone Galaxy via git")
96     _install_with_command(ctx, galaxy_root, env, kwds)
97 
98 
99 def _build_eggs_cache(ctx, env, kwds):
00     if kwds.get("no_cache_galaxy", False):
01         return None
02     workspace = ctx.workspace
03     eggs_path = os.path.join(workspace, "gx_eggs")
04     if not os.path.exists(eggs_path):
05         os.makedirs(eggs_path)
06     env["GALAXY_EGGS_PATH"] = eggs_path
07 
08 
09 def _galaxy_branch(kwds):
10     branch = kwds.get("galaxy_branch", None)
11     if branch is None:
12         cwl = kwds.get("cwl", False)
13         branch = "cwl-1.0" if cwl else None
14     if branch is None:
15         branch = DEFAULT_GALAXY_BRANCH
16 
17     return branch
18 
19 
20 def _galaxy_source(kwds):
21     source = kwds.get("galaxy_source", None)
22     if source is None:
23         cwl = kwds.get("cwl", False)
24         source = CWL_GALAXY_SOURCE if cwl else None
25     if source is None:
26         source = DEFAULT_GALAXY_SOURCE
27 
28     return source
29 
30 
31 def _install_with_command(ctx, galaxy_root, env, kwds):
32     setup_venv_command = setup_venv(ctx, kwds)
33     install_cmd = shell_join(
34         setup_venv_command,
35         setup_common_startup_args(),
36         COMMAND_STARTUP_COMMAND,
37     )
38     exit_code = shell(install_cmd, cwd=galaxy_root, env=env)
39     if exit_code != 0:
40         raise Exception("Failed to install Galaxy via command [%s]" % install_cmd)
41     if not os.path.exists(galaxy_root):
42         raise Exception("Failed to create Galaxy directory [%s]" % galaxy_root)
43     if not os.path.exists(os.path.join(galaxy_root, "lib")):
44         raise Exception("Failed to create Galaxy directory [%s], lib missing" % galaxy_root)
45 
46 
47 def _ensure_galaxy_repository_available(ctx, kwds):
48     workspace = ctx.workspace
49     cwl = kwds.get("cwl", False)
50     galaxy_source = kwds.get('galaxy_source')
51     if galaxy_source and galaxy_source != DEFAULT_GALAXY_SOURCE:
52         sanitized_repo_name = "".join(c if c.isalnum() else '_' for c in kwds['galaxy_source']).rstrip()[:255]
53         gx_repo = os.path.join(workspace, "gx_repo_%s" % sanitized_repo_name)
54     else:
55         gx_repo = os.path.join(workspace, "gx_repo")
56     if cwl:
57         gx_repo += "_cwl"
58     if os.path.exists(gx_repo):
59         # Convert the git repository from bare to mirror, if needed
60         shell(['git', '--git-dir', gx_repo, 'config', 'remote.origin.fetch', '+refs/*:refs/*'])
61         shell(['git', '--git-dir', gx_repo, 'config', 'remote.origin.mirror', 'true'])
62         # Attempt remote update - but don't fail if not interweb, etc...
63         shell("git --git-dir %s remote update >/dev/null 2>&1" % gx_repo)
64     else:
65         remote_repo = _galaxy_source(kwds)
66         command = git.command_clone(ctx, remote_repo, gx_repo, mirror=True)
67         shell(command)
68     return gx_repo
69 
70 
71 def _build_env_for_galaxy(properties, template_args):
72     env = {}
73     for key, value in iteritems(properties):
74         if value is not None:  # Do not override None with empty string
75             var = "GALAXY_CONFIG_OVERRIDE_%s" % key.upper()
76             value = _sub(value, template_args)
77             env[var] = value
78     return env
79 
80 
81 def _build_test_env(properties, env):
82     # Keeping these environment variables around for a little while but
83     # many are probably not needed as of the following commit.
84     # https://bitbucket.org/galaxy/galaxy-central/commits/d7dd1f9
85     test_property_variants = {
86         'GALAXY_TEST_JOB_CONFIG_FILE': 'job_config_file',
87         'GALAXY_TEST_MIGRATED_TOOL_CONF': 'migrated_tools_config',
88         'GALAXY_TEST_TOOL_CONF': 'tool_config_file',
89         'GALAXY_TEST_FILE_DIR': 'test_data_dir',
90         'GALAXY_TOOL_DEPENDENCY_DIR': 'tool_dependency_dir',
91         # Next line would be required for tool shed tests.
92         # 'GALAXY_TEST_TOOL_DEPENDENCY_DIR': 'tool_dependency_dir',
93     }
94     for test_key, gx_key in test_property_variants.items():
95         value = properties.get(gx_key, None)
96         if value is not None:
97             env[test_key] = value
98 
99 
00 def _handle_job_config_file(config_directory, server_name, kwds):
01     job_config_file = kwds.get("job_config_file", None)
02     if not job_config_file:
03         template_str = JOB_CONFIG_LOCAL
04         job_config_file = os.path.join(
05             config_directory,
06             "job_conf.xml",
07         )
08         docker_enable = str(kwds.get("docker", False))
09         docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST)
10         docker_host_param = ""
11         if docker_host:
12             docker_host_param = """<param id="docker_host">%s</param>""" % docker_host
13 
14         conf_contents = Template(template_str).safe_substitute({
15             "server_name": server_name,
16             "docker_enable": docker_enable,
17             "require_container": "false",
18             "docker_sudo": str(kwds.get("docker_sudo", False)),
19             "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
20             "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
21             "docker_host_param": docker_host_param,
22         })
23         write_file(job_config_file, conf_contents)
24     kwds["job_config_file"] = job_config_file
25 
26 
27 def _write_tool_conf(ctx, tool_paths, tool_conf_path):
28     tool_definition = _tool_conf_entry_for(tool_paths)
29     tool_conf_template_kwds = dict(tool_definition=tool_definition)
30     tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, tool_conf_template_kwds)
31     write_file(tool_conf_path, tool_conf_contents)
32     ctx.vlog(
33         "Writing tool_conf to path %s with contents [%s]",
34         tool_conf_path,
35         tool_conf_contents,
36     )
37 
38 
39 def _handle_container_resolution(ctx, kwds, galaxy_properties):
40     if kwds.get("mulled_containers", False):
41         galaxy_properties["enable_beta_mulled_containers"] = "True"
42         involucro_context = build_involucro_context(ctx, **kwds)
43         galaxy_properties["involucro_auto_init"] = "False"  # Use planemo's
44         galaxy_properties["involucro_path"] = involucro_context.involucro_bin
45 
46 
47 def _handle_job_metrics(config_directory, kwds):
48     metrics_conf = os.path.join(config_directory, "job_metrics_conf.xml")
49     with open(metrics_conf, "w") as fh:
50         fh.write(EMPTY_JOB_METRICS_TEMPLATE)
51     kwds["job_metrics_config_file"] = metrics_conf
52 
53 
54 def _handle_refgenie_config(config_directory, kwds):
55     refgenie_dir = os.path.join(config_directory, 'refgenie')
56     _ensure_directory(refgenie_dir)
57     refgenie_config = os.path.join(refgenie_dir, "genome_config.yaml")
58     with open(refgenie_config, "w") as fh:
59         fh.write(REFGENIE_CONFIG_TEMPLATE % (refgenie_dir))
60     kwds["refgenie_config_file"] = refgenie_config
61 
62 
63 def _handle_kwd_overrides(properties, kwds):
64     kwds_gx_properties = [
65         'job_config_file',
66         'job_metrics_config_file',
67         'dependency_resolvers_config_file',
68     ]
69     for prop in kwds_gx_properties:
70         val = kwds.get(prop, None)
71         if val:
72             properties[prop] = val
73 
74 
75 def _sub(template, args):
76     if template is None:
77         return ''
78     return Template(template).safe_substitute(args)
79 
80 
81 def _ensure_directory(path):
82     if path is not None and not os.path.exists(path):
83         os.makedirs(path)
84 
85 
86 __all__ = (
87     "DATABASE_LOCATION_TEMPLATE",
88     "galaxy_config",
89 )