Mercurial > repos > shellac > sam_consensus_v3
view env/lib/python3.9/site-packages/galaxy/tool_util/deps/mulled/mulled_build.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
line wrap: on
line source
#!/usr/bin/env python """Build a mulled image for specified conda targets. Examples Build a mulled image with: mulled-build build 'samtools=1.3.1--4,bedtools=2.22' """ import json import logging import os import shlex import shutil import stat import string import subprocess import sys from sys import platform as _platform import yaml from galaxy.tool_util.deps import installable from galaxy.tool_util.deps.conda_util import best_search_result from galaxy.tool_util.deps.docker_util import command_list as docker_command_list from galaxy.util import ( commands, safe_makedirs, unicodify, ) from ._cli import arg_parser from .util import ( build_target, conda_build_target_str, create_repository, get_file_from_recipe_url, PrintProgress, quay_repository, v1_image_name, v2_image_name, ) from ..conda_compat import MetaData log = logging.getLogger(__name__) DIRNAME = os.path.dirname(__file__) DEFAULT_BASE_IMAGE = "bgruening/busybox-bash:0.1" DEFAULT_EXTENDED_BASE_IMAGE = "bioconda/extended-base-image:latest" DEFAULT_CHANNELS = ["conda-forge", "bioconda"] DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}" DEFAULT_BINDS = ["build/dist:/usr/local/"] DEFAULT_WORKING_DIR = '/source/' IS_OS_X = _platform == "darwin" INVOLUCRO_VERSION = "1.1.2" DEST_BASE_IMAGE = os.environ.get('DEST_BASE_IMAGE', None) CONDA_IMAGE = os.environ.get('CONDA_IMAGE', None) SINGULARITY_TEMPLATE = """Bootstrap: docker From: %(base_image)s %%setup echo "Copying conda environment" mkdir -p /tmp/conda cp -r /data/dist/* /tmp/conda/ %%post rm -R /usr/local || true mkdir -p /usr/local cp -R /tmp/conda/* /usr/local/ %%test %(container_test)s """ def involucro_link(): if IS_OS_X: url = "https://github.com/mvdbeek/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION else: url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION return url def get_tests(args, pkg_path): """Extract test cases given a recipe's meta.yaml file.""" recipes_dir = args.recipes_dir tests = [] input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path)) recipe_meta = MetaData(input_dir) tests_commands = recipe_meta.get_value('test/commands') tests_imports = recipe_meta.get_value('test/imports') requirements = recipe_meta.get_value('requirements/run') if tests_imports or tests_commands: if tests_commands: tests.append(' && '.join(tests_commands)) if tests_imports and 'python' in requirements: tests.append(' && '.join('python -c "import %s"' % imp for imp in tests_imports)) elif tests_imports and ('perl' in requirements or 'perl-threaded' in requirements): tests.append(' && '.join('''perl -e "use %s;"''' % imp for imp in tests_imports)) tests = ' && '.join(tests) tests = tests.replace('$R ', 'Rscript ') return tests def get_pkg_name(args, pkg_path): """Extract the package name from a given meta.yaml file.""" recipes_dir = args.recipes_dir input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path)) recipe_meta = MetaData(input_dir) return recipe_meta.get_value('package/name') def get_affected_packages(args): """Return a list of all meta.yaml file that where modified/created recently. Length of time to check for indicated by the ``hours`` parameter. """ recipes_dir = args.recipes_dir hours = args.diff_hours cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours] changed_files = unicodify(subprocess.check_output(cmd, cwd=recipes_dir)).splitlines() pkg_list = {x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')} for pkg in pkg_list: if pkg and os.path.exists(os.path.join(recipes_dir, pkg)): yield (get_pkg_name(args, pkg), get_tests(args, pkg)) def conda_versions(pkg_name, file_name): """Return all conda version strings for a specified package name.""" j = json.load(open(file_name)) ret = list() for pkg in j['packages'].values(): if pkg['name'] == pkg_name: ret.append('{}--{}'.format(pkg['version'], pkg['build'])) return ret def get_conda_hits_for_targets(targets, conda_context): search_results = (best_search_result(t, conda_context, platform='linux-64')[0] for t in targets) return [r for r in search_results if r] def base_image_for_targets(targets, conda_context=None): hits = get_conda_hits_for_targets(targets, conda_context or CondaInDockerContext()) for hit in hits: try: tarball = get_file_from_recipe_url(hit['url']) meta_content = unicodify(tarball.extractfile('info/about.json').read()) if json.loads(meta_content).get('extra', {}).get('container', {}).get('extended-base', False): return DEFAULT_EXTENDED_BASE_IMAGE elif yaml.safe_load(unicodify(tarball.extractfile('info/recipe/meta.yaml').read())).get('extra', {}).get('container', {}).get('extended-base', False): return DEFAULT_EXTENDED_BASE_IMAGE except Exception: log.warning("Could not load metadata.yaml for '%s', version '%s'", hit['name'], hit['version'], exc_info=True) return DEFAULT_BASE_IMAGE class BuildExistsException(Exception): """Exception indicating mull_targets is skipping an existing build. If mull_targets is called with rebuild=False and the target built is already published an instance of this exception is thrown. """ def mull_targets( targets, involucro_context=None, command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers", test='true', test_files=None, image_build=None, name_override=None, repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True, oauth_token=None, hash_func="v2", singularity=False, singularity_image_dir="singularity_import", base_image=None, determine_base_image=True, ): targets = list(targets) if involucro_context is None: involucro_context = InvolucroContext() image_function = v1_image_name if hash_func == "v1" else v2_image_name if len(targets) > 1 and image_build is None: # Force an image build in this case - this seems hacky probably # shouldn't work this way but single case broken else wise. image_build = "0" repo_template_kwds = { "namespace": namespace, "image": image_function(targets, image_build=image_build, name_override=name_override) } repo = string.Template(repository_template).safe_substitute(repo_template_kwds) if not rebuild or "push" in command: repo_name = repo_template_kwds["image"].split(":", 1)[0] repo_data = quay_repository(repo_template_kwds["namespace"], repo_name) if not rebuild: tags = repo_data.get("tags", []) target_tag = None if ":" in repo_template_kwds["image"]: image_name_parts = repo_template_kwds["image"].split(":") assert len(image_name_parts) == 2, ": not allowed in image name [%s]" % repo_template_kwds["image"] target_tag = image_name_parts[1] if tags and (target_tag is None or target_tag in tags): raise BuildExistsException() if "push" in command and "error_type" in repo_data and oauth_token: # Explicitly create the repository so it can be built as public. create_repository(repo_template_kwds["namespace"], repo_name, oauth_token) for channel in channels: if channel.startswith('file://'): bind_path = channel[7:] binds.append(f'/{bind_path}:/{bind_path}') channels = ",".join(channels) target_str = ",".join(map(conda_build_target_str, targets)) bind_str = ",".join(binds) involucro_args = [ '-f', '%s/invfile.lua' % DIRNAME, '-set', "CHANNELS=%s" % channels, '-set', "TARGETS=%s" % target_str, '-set', "REPO=%s" % repo, '-set', "BINDS=%s" % bind_str, ] dest_base_image = None if base_image: dest_base_image = base_image elif DEST_BASE_IMAGE: dest_base_image = DEST_BASE_IMAGE elif determine_base_image: dest_base_image = base_image_for_targets(targets) if dest_base_image: involucro_args.extend(["-set", "DEST_BASE_IMAGE=%s" % dest_base_image]) if CONDA_IMAGE: involucro_args.extend(["-set", "CONDA_IMAGE=%s" % CONDA_IMAGE]) if verbose: involucro_args.extend(["-set", "VERBOSE=1"]) if singularity: singularity_image_name = repo_template_kwds['image'] involucro_args.extend(["-set", "SINGULARITY=1"]) involucro_args.extend(["-set", "SINGULARITY_IMAGE_NAME=%s" % singularity_image_name]) involucro_args.extend(["-set", "SINGULARITY_IMAGE_DIR=%s" % singularity_image_dir]) involucro_args.extend(["-set", f"USER_ID={os.getuid()}:{os.getgid()}"]) if test: involucro_args.extend(["-set", "TEST=%s" % test]) if conda_version is not None: verbose = "--verbose" if verbose else "--quiet" involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes conda={conda_version}"]) involucro_args.append(command) if test_files: test_bind = [] for test_file in test_files: if ':' not in test_file: if os.path.exists(test_file): test_bind.append(f"{test_file}:{DEFAULT_WORKING_DIR}/{test_file}") else: if os.path.exists(test_file.split(':')[0]): test_bind.append(test_file) if test_bind: involucro_args.insert(6, '-set') involucro_args.insert(7, "TEST_BINDS=%s" % ",".join(test_bind)) cmd = involucro_context.build_command(involucro_args) print('Executing: ' + ' '.join(shlex.quote(_) for _ in cmd)) if dry_run: return 0 ensure_installed(involucro_context, True) if singularity: if not os.path.exists(singularity_image_dir): safe_makedirs(singularity_image_dir) with open(os.path.join(singularity_image_dir, 'Singularity.def'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % {'container_test': test, 'base_image': dest_base_image or DEFAULT_BASE_IMAGE} sin_def.write(fill_template) with PrintProgress(): ret = involucro_context.exec_command(involucro_args) if singularity: # we can not remove this folder as it contains the image wich is owned by root pass # shutil.rmtree('./singularity_import') return ret def context_from_args(args): verbose = "2" if not args.verbose else "3" return InvolucroContext(involucro_bin=args.involucro_path, verbose=verbose) class CondaInDockerContext: @property def conda_exec(self): conda_image = CONDA_IMAGE or 'continuumio/miniconda3:latest' return docker_command_list('run', [conda_image, 'conda']) @property def _override_channels_args(self): override_channels_args = ['--override-channels'] for channel in DEFAULT_CHANNELS: override_channels_args.extend(["--channel", channel]) return override_channels_args class InvolucroContext(installable.InstallableContext): installable_description = "Involucro" def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"): if involucro_bin is None: if os.path.exists("./involucro"): self.involucro_bin = "./involucro" else: self.involucro_bin = "involucro" else: self.involucro_bin = involucro_bin self.shell_exec = shell_exec or commands.shell self.verbose = verbose def build_command(self, involucro_args): return [self.involucro_bin, "-v=%s" % self.verbose] + involucro_args def exec_command(self, involucro_args): cmd = self.build_command(involucro_args) # Create ./build dir manually, otherwise Docker will do it as root created_build_dir = False if not os.path.exists('build'): created_build_dir = True os.mkdir('./build') try: res = self.shell_exec(cmd) finally: # delete build directory in any case if created_build_dir: shutil.rmtree('./build') return res def is_installed(self): return os.path.exists(self.involucro_bin) def can_install(self): return True @property def parent_path(self): return os.path.dirname(os.path.abspath(self.involucro_bin)) def ensure_installed(involucro_context, auto_init): return installable.ensure_installed(involucro_context, install_involucro, auto_init) def install_involucro(involucro_context): install_path = os.path.abspath(involucro_context.involucro_bin) involucro_context.involucro_bin = install_path download_cmd = commands.download_command(involucro_link(), to=install_path) exit_code = involucro_context.shell_exec(download_cmd) if exit_code: return exit_code try: os.chmod(install_path, os.stat(install_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) return 0 except Exception: log.exception("Failed to make file '%s' executable", install_path) return 1 def add_build_arguments(parser): """Base arguments describing how to 'mull'.""" parser.add_argument('--involucro-path', dest="involucro_path", default=None, help="Path to involucro (if not set will look in working directory and on PATH).") parser.add_argument('--dry-run', dest='dry_run', action="store_true", help='Just print commands instead of executing them.') parser.add_argument('--verbose', dest='verbose', action="store_true", help='Cause process to be verbose.') parser.add_argument('--singularity', action="store_true", help='Additionally build a singularity image.') parser.add_argument('--singularity-image-dir', dest="singularity_image_dir", help="Directory to write singularity images too.") parser.add_argument('-n', '--namespace', dest='namespace', default="biocontainers", help='quay.io namespace.') parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE, help='Docker repository target for publication (only quay.io or compat. API is currently supported).') parser.add_argument('-c', '--channels', dest='channels', default=",".join(DEFAULT_CHANNELS), help='Comma separated list of target conda channels.') parser.add_argument('--conda-version', dest="conda_version", default=None, help="Change to specified version of Conda before installing packages.") parser.add_argument('--oauth-token', dest="oauth_token", default=None, help="If set, use this token when communicating with quay.io API.") parser.add_argument('--check-published', dest="rebuild", action='store_false') parser.add_argument('--hash', dest="hash", choices=["v1", "v2"], default="v2") def add_single_image_arguments(parser): parser.add_argument("--name-override", dest="name_override", default=None, help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images") parser.add_argument("--image-build", dest="image_build", default=None, help="Build a versioned variant of this image.") def target_str_to_targets(targets_raw): def parse_target(target_str): if "=" in target_str: package_name, version = target_str.split("=", 1) build = None if "=" in version: version, build = version.split('=') elif "--" in version: version, build = version.split('--') target = build_target(package_name, version, build) else: target = build_target(target_str) return target targets = [parse_target(_) for _ in targets_raw.split(",")] return targets def args_to_mull_targets_kwds(args): kwds = {} if hasattr(args, "image_build"): kwds["image_build"] = args.image_build if hasattr(args, "name_override"): kwds["name_override"] = args.name_override if hasattr(args, "namespace"): kwds["namespace"] = args.namespace if hasattr(args, "dry_run"): kwds["dry_run"] = args.dry_run if hasattr(args, "singularity"): kwds["singularity"] = args.singularity if hasattr(args, "test"): kwds["test"] = args.test if hasattr(args, "test_files"): if args.test_files: kwds["test_files"] = args.test_files.split(",") if hasattr(args, "channels"): kwds["channels"] = args.channels.split(',') if hasattr(args, "command"): kwds["command"] = args.command if hasattr(args, "repository_template"): kwds["repository_template"] = args.repository_template if hasattr(args, "conda_version"): kwds["conda_version"] = args.conda_version if hasattr(args, "oauth_token"): kwds["oauth_token"] = args.oauth_token if hasattr(args, "rebuild"): kwds["rebuild"] = args.rebuild if hasattr(args, "hash"): kwds["hash_func"] = args.hash if hasattr(args, "singularity_image_dir") and args.singularity_image_dir: kwds["singularity_image_dir"] = args.singularity_image_dir kwds["involucro_context"] = context_from_args(args) return kwds def main(argv=None): """Main entry-point for the CLI tool.""" parser = arg_parser(argv, globals()) add_build_arguments(parser) add_single_image_arguments(parser) parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)') parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).") parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).") parser.add_argument('--test', help='Provide a test command for the container.') parser.add_argument('--test-files', help='Provide test-files that may be required to run the test command. Individual mounts are separated by comma.' 'The source:dest docker syntax is respected. If relative file paths are given, files will be mounted in /source/<relative_file_path>') args = parser.parse_args() targets = target_str_to_targets(args.targets) sys.exit(mull_targets(targets, **args_to_mull_targets_kwds(args))) __all__ = ("main", ) if __name__ == '__main__': main()