Mercurial > repos > shellac > sam_consensus_v3
diff env/lib/python3.9/site-packages/galaxy/tool_util/deps/container_resolvers/mulled.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/env/lib/python3.9/site-packages/galaxy/tool_util/deps/container_resolvers/mulled.py Mon Mar 22 18:12:50 2021 +0000 @@ -0,0 +1,629 @@ +"""This module describes the :class:`MulledContainerResolver` ContainerResolver plugin.""" + +import logging +import os +import subprocess +from typing import NamedTuple, Optional + +from galaxy.util import ( + safe_makedirs, + string_as_bool, + unicodify, + which, +) +from galaxy.util.commands import shell +from ..container_classes import CONTAINER_CLASSES +from ..container_resolvers import ( + ContainerResolver, +) +from ..docker_util import build_docker_images_command +from ..mulled.mulled_build import ( + DEFAULT_CHANNELS, + ensure_installed, + InvolucroContext, + mull_targets, +) +from ..mulled.mulled_build_tool import requirements_to_mulled_targets +from ..mulled.util import ( + mulled_tags_for, + split_tag, + v1_image_name, + v2_image_name, + version_sorted, +) +from ..requirements import ( + ContainerDescription, + DEFAULT_CONTAINER_SHELL, +) + +log = logging.getLogger(__name__) + + +class CachedMulledImageSingleTarget(NamedTuple): + package_name: str + version: str + build: str + image_identifier: str + + multi_target: bool = False + + +class CachedV1MulledImageMultiTarget(NamedTuple): + hash: str + build: str + image_identifier: str + + multi_target: str = "v1" + + +class CachedV2MulledImageMultiTarget(NamedTuple): + image_name: str + version_hash: str + build: str + image_identifier: str + + multi_target: str = "v2" + + @property + def package_hash(target): + # Make this work for Singularity file name or fully qualified Docker repository + # image names. + image_name = target.image_name + if "/" not in image_name: + return image_name + else: + return image_name.rsplit("/")[-1] + + +def list_docker_cached_mulled_images(namespace=None, hash_func="v2", resolution_cache=None): + cache_key = "galaxy.tool_util.deps.container_resolvers.mulled:cached_images" + if resolution_cache is not None and cache_key in resolution_cache: + images_and_versions = resolution_cache.get(cache_key) + else: + command = build_docker_images_command(truncate=True, sudo=False, to_str=False) + try: + images_and_versions = unicodify(subprocess.check_output(command)).strip().splitlines() + except subprocess.CalledProcessError: + log.info("Call to `docker images` failed, configured container resolution may be broken") + return [] + images_and_versions = [":".join(l.split()[0:2]) for l in images_and_versions[1:]] + if resolution_cache is not None: + resolution_cache[cache_key] = images_and_versions + + def output_line_to_image(line): + image = identifier_to_cached_target(line, hash_func, namespace=namespace) + return image + + name_filter = get_filter(namespace) + sorted_images = version_sorted([_ for _ in filter(name_filter, images_and_versions)]) + raw_images = (output_line_to_image(_) for _ in sorted_images) + return [i for i in raw_images if i is not None] + + +def identifier_to_cached_target(identifier, hash_func, namespace=None): + if ":" in identifier: + image_name, version = identifier.rsplit(":", 1) + else: + image_name = identifier + version = None + + if not version or version == "latest": + version = None + + image = None + prefix = "" + if namespace is not None: + prefix = "quay.io/%s/" % namespace + if image_name.startswith(prefix + "mulled-v1-"): + if hash_func == "v2": + return None + + hash = image_name + build = None + if version and version.isdigit(): + build = version + image = CachedV1MulledImageMultiTarget(hash, build, identifier) + elif image_name.startswith(prefix + "mulled-v2-"): + if hash_func == "v1": + return None + + version_hash = None + build = None + + if version and "-" in version: + version_hash, build = version.rsplit("-", 1) + elif version.isdigit(): + version_hash, build = None, version + elif version: + log.debug("Unparsable mulled image tag encountered [%s]" % version) + + image = CachedV2MulledImageMultiTarget(image_name, version_hash, build, identifier) + else: + build = None + if version and "--" in version: + version, build = split_tag(version) + if prefix and image_name.startswith(prefix): + image_name = image_name[len(prefix):] + image = CachedMulledImageSingleTarget(image_name, version, build, identifier) + return image + + +def list_cached_mulled_images_from_path(directory, hash_func="v2"): + contents = os.listdir(directory) + sorted_images = version_sorted(contents) + raw_images = map(lambda name: identifier_to_cached_target(name, hash_func), sorted_images) + return [i for i in raw_images if i is not None] + + +def get_filter(namespace): + prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace + return lambda name: name.startswith(prefix) and name.count("/") == 2 + + +def find_best_matching_cached_image(targets, cached_images, hash_func): + if len(targets) == 0: + return None + + image = None + if len(targets) == 1: + target = targets[0] + for cached_image in cached_images: + if cached_image.multi_target: + continue + if not cached_image.package_name == target.package_name: + continue + if not target.version or target.version == cached_image.version: + image = cached_image + break + elif hash_func == "v2": + name = v2_image_name(targets) + if ":" in name: + package_hash, version_hash = name.split(":", 2) + else: + package_hash, version_hash = name, None + + for cached_image in cached_images: + if cached_image.multi_target != "v2": + continue + + if version_hash is None: + # Just match on package hash... + if package_hash == cached_image.package_hash: + image = cached_image + break + else: + # Match on package and version hash... + if package_hash == cached_image.package_hash and version_hash == cached_image.version_hash: + image = cached_image + break + + elif hash_func == "v1": + name = v1_image_name(targets) + for cached_image in cached_images: + if cached_image.multi_target != "v1": + continue + + if name == cached_image.hash: + image = cached_image + break + return image + + +def docker_cached_container_description(targets, namespace, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL, resolution_cache=None): + if len(targets) == 0: + return None + + cached_images = list_docker_cached_mulled_images(namespace, hash_func=hash_func, resolution_cache=resolution_cache) + image = find_best_matching_cached_image(targets, cached_images, hash_func) + + container = None + if image: + container = ContainerDescription( + image.image_identifier, + type="docker", + shell=shell, + ) + + return container + + +def singularity_cached_container_description(targets, cache_directory, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL): + if len(targets) == 0: + return None + + if not os.path.exists(cache_directory): + return None + + cached_images = list_cached_mulled_images_from_path(cache_directory, hash_func=hash_func) + image = find_best_matching_cached_image(targets, cached_images, hash_func) + + container = None + if image: + container = ContainerDescription( + os.path.abspath(os.path.join(cache_directory, image.image_identifier)), + type="singularity", + shell=shell, + ) + + return container + + +def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None, session=None): + unresolved_cache_key = "galaxy.tool_util.deps.container_resolvers.mulled:unresolved" + if resolution_cache is not None: + if unresolved_cache_key not in resolution_cache: + resolution_cache[unresolved_cache_key] = set() + unresolved_cache = resolution_cache.get(unresolved_cache_key) + else: + unresolved_cache = set() + + mulled_resolution_cache = None + if resolution_cache and hasattr(resolution_cache, 'mulled_resolution_cache'): + mulled_resolution_cache = resolution_cache.mulled_resolution_cache + + name = None + + def cached_name(cache_key): + if mulled_resolution_cache: + if cache_key in mulled_resolution_cache: + return resolution_cache.get(cache_key) + return None + + if len(targets) == 1: + target = targets[0] + target_version = target.version + cache_key = f"ns[{namespace}]__single__{target.package_name}__@__{target_version}" + if cache_key in unresolved_cache: + return None + name = cached_name(cache_key) + if name: + return name + + tags = mulled_tags_for(namespace, target.package_name, resolution_cache=resolution_cache, session=session) + + if tags: + for tag in tags: + if '--' in tag: + version, _ = split_tag(tag) + else: + version = tag + if target_version and version == target_version: + name = f"{target.package_name}:{tag}" + break + + else: + def first_tag_if_available(image_name): + if ":" in image_name: + repo_name, tag_prefix = image_name.split(":", 2) + else: + repo_name = image_name + tag_prefix = None + tags = mulled_tags_for(namespace, repo_name, tag_prefix=tag_prefix, resolution_cache=resolution_cache, session=session) + return tags[0] if tags else None + + if hash_func == "v2": + base_image_name = v2_image_name(targets) + elif hash_func == "v1": + base_image_name = v1_image_name(targets) + else: + raise Exception("Unimplemented mulled hash_func [%s]" % hash_func) + + cache_key = f"ns[{namespace}]__{hash_func}__{base_image_name}" + if cache_key in unresolved_cache: + return None + name = cached_name(cache_key) + if name: + return name + + tag = first_tag_if_available(base_image_name) + if tag: + if ":" in base_image_name: + assert hash_func != "v1" + # base_image_name of form <package_hash>:<version_hash>, expand tag + # to include build number in tag. + name = "{}:{}".format(base_image_name.split(":")[0], tag) + else: + # base_image_name of form <package_hash>, simply add build number + # as tag to fully qualify image. + name = f"{base_image_name}:{tag}" + + if name and mulled_resolution_cache: + mulled_resolution_cache.put(cache_key, name) + + if name is None: + unresolved_cache.add(name) + + return name + + +class CliContainerResolver(ContainerResolver): + + container_type = 'docker' + cli = 'docker' + + def __init__(self, *args, **kwargs): + self._cli_available = bool(which(self.cli)) + super().__init__(*args, **kwargs) + + @property + def cli_available(self): + return self._cli_available + + @cli_available.setter + def cli_available(self, value): + if not value: + log.info('{} CLI not available, cannot list or pull images in Galaxy process. Does not impact kubernetes.'.format(self.cli)) + self._cli_available = value + + +class SingularityCliContainerResolver(CliContainerResolver): + + container_type = 'singularity' + cli = 'singularity' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cache_directory = kwargs.get("cache_directory", os.path.join(kwargs['app_info'].container_image_cache_path, "singularity", "mulled")) + safe_makedirs(self.cache_directory) + + +class CachedMulledDockerContainerResolver(CliContainerResolver): + + resolver_type = "cached_mulled" + shell = '/bin/bash' + + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds): + super().__init__(app_info=app_info, **kwds) + self.namespace = namespace + self.hash_func = hash_func + + def resolve(self, enabled_container_types, tool_info, **kwds): + if not self.cli_available or tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + resolution_cache = kwds.get("resolution_cache") + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell, resolution_cache=resolution_cache) + + def __str__(self): + return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace + + +class CachedMulledSingularityContainerResolver(SingularityCliContainerResolver): + + resolver_type = "cached_mulled_singularity" + shell = '/bin/bash' + + def __init__(self, app_info=None, hash_func="v2", **kwds): + super().__init__(app_info=app_info, **kwds) + self.hash_func = hash_func + + def resolve(self, enabled_container_types, tool_info, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) + + def __str__(self): + return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory + + +class MulledDockerContainerResolver(CliContainerResolver): + """Look for mulled images matching tool dependencies.""" + + resolver_type = "mulled" + shell = '/bin/bash' + protocol: Optional[str] = None + + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): + super().__init__(app_info=app_info, **kwds) + self.namespace = namespace + self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) + + def cached_container_description(self, targets, namespace, hash_func, resolution_cache): + try: + return docker_cached_container_description(targets, namespace, hash_func, resolution_cache) + except subprocess.CalledProcessError: + # We should only get here if a docker binary is available, but command quits with a non-zero exit code, + # e.g if the docker daemon is not available + log.exception('An error occured while listing cached docker image. Docker daemon may need to be restarted.') + return None + + def pull(self, container): + if self.cli_available: + command = container.build_pull_command() + shell(command) + + @property + def can_list_containers(self): + return self.cli_available + + def resolve(self, enabled_container_types, tool_info, install=False, session=None, **kwds): + resolution_cache = kwds.get("resolution_cache") + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + if len(targets) == 0: + return None + + name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace, resolution_cache=resolution_cache, session=session) + if name: + container_id = f"quay.io/{self.namespace}/{name}" + if self.protocol: + container_id = f"{self.protocol}{container_id}" + container_description = ContainerDescription( + container_id, + type=self.container_type, + shell=self.shell, + ) + if self.can_list_containers: + if install and not self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func, + resolution_cache=resolution_cache, + ): + destination_info = {} + destination_for_container_type = kwds.get('destination_for_container_type') + if destination_for_container_type: + destination_info = destination_for_container_type(self.container_type) + container = CONTAINER_CLASSES[self.container_type](container_description.identifier, + self.app_info, + tool_info, + destination_info, + {}, + container_description) + self.pull(container) + if not self.auto_install: + container_description = self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func, + resolution_cache=resolution_cache, + ) or container_description + return container_description + + def __str__(self): + return "MulledDockerContainerResolver[namespace=%s]" % self.namespace + + +class MulledSingularityContainerResolver(SingularityCliContainerResolver, MulledDockerContainerResolver): + + resolver_type = "mulled_singularity" + protocol = 'docker://' + + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): + super().__init__(app_info=app_info, **kwds) + self.namespace = namespace + self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) + + def cached_container_description(self, targets, namespace, hash_func, resolution_cache): + return singularity_cached_container_description(targets, + cache_directory=self.cache_directory, + hash_func=hash_func) + + @property + def can_list_containers(self): + # Only needs access to path, doesn't require CLI + return True + + def pull(self, container): + if self.cli_available: + cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace) + shell(cmds=cmds) + + def __str__(self): + return "MulledSingularityContainerResolver[namespace=%s]" % self.namespace + + +class BuildMulledDockerContainerResolver(CliContainerResolver): + """Build for Docker mulled images matching tool dependencies.""" + + resolver_type = "build_mulled" + shell = '/bin/bash' + builds_on_resolution = True + + def __init__(self, app_info=None, namespace="local", hash_func="v2", auto_install=True, **kwds): + super().__init__(app_info=app_info, **kwds) + self._involucro_context_kwds = { + 'involucro_bin': self._get_config_option("involucro_path", None) + } + self.namespace = namespace + self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) + self._mulled_kwds = { + 'namespace': namespace, + 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), + 'hash_func': self.hash_func, + 'command': 'build-and-test', + } + self.auto_init = self._get_config_option("involucro_auto_init", True) + + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + if len(targets) == 0: + return None + if self.auto_install or install: + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) + + def _get_involucro_context(self): + involucro_context = InvolucroContext(**self._involucro_context_kwds) + self.enabled = ensure_installed(involucro_context, self.auto_init) + return involucro_context + + def __str__(self): + return "BuildDockerContainerResolver[namespace=%s]" % self.namespace + + +class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver): + """Build for Singularity mulled images matching tool dependencies.""" + + resolver_type = "build_mulled_singularity" + shell = '/bin/bash' + builds_on_resolution = True + + def __init__(self, app_info=None, hash_func="v2", auto_install=True, **kwds): + super().__init__(app_info=app_info, **kwds) + self._involucro_context_kwds = { + 'involucro_bin': self._get_config_option("involucro_path", None) + } + self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) + self._mulled_kwds = { + 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), + 'hash_func': self.hash_func, + 'command': 'build-and-test', + 'singularity': True, + 'singularity_image_dir': self.cache_directory, + } + self.auto_init = self._get_config_option("involucro_auto_init", True) + + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + if len(targets) == 0: + return None + + if self.auto_install or install: + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) + + def _get_involucro_context(self): + involucro_context = InvolucroContext(**self._involucro_context_kwds) + self.enabled = ensure_installed(involucro_context, self.auto_init) + return involucro_context + + def __str__(self): + return "BuildSingularityContainerResolver[cache_directory=%s]" % self.cache_directory + + +def mulled_targets(tool_info): + return requirements_to_mulled_targets(tool_info.requirements) + + +__all__ = ( + "CachedMulledDockerContainerResolver", + "CachedMulledSingularityContainerResolver", + "MulledDockerContainerResolver", + "MulledSingularityContainerResolver", + "BuildMulledDockerContainerResolver", + "BuildMulledSingularityContainerResolver", +)