Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/deps/resolvers/conda.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 """ | |
2 This is still an experimental module and there will almost certainly be backward | |
3 incompatible changes coming. | |
4 """ | |
5 | |
6 import logging | |
7 import os | |
8 import re | |
9 | |
10 import galaxy.tool_util.deps.installable | |
11 import galaxy.tool_util.deps.requirements | |
12 from . import ( | |
13 Dependency, | |
14 DependencyException, | |
15 DependencyResolver, | |
16 ListableDependencyResolver, | |
17 MappableDependencyResolver, | |
18 MultipleDependencyResolver, | |
19 NullDependency, | |
20 SpecificationPatternDependencyResolver, | |
21 ) | |
22 from ..conda_util import ( | |
23 build_isolated_environment, | |
24 cleanup_failed_install, | |
25 cleanup_failed_install_of_environment, | |
26 CondaContext, | |
27 CondaTarget, | |
28 hash_conda_packages, | |
29 install_conda, | |
30 install_conda_target, | |
31 install_conda_targets, | |
32 installed_conda_targets, | |
33 is_conda_target_installed, | |
34 USE_PATH_EXEC_DEFAULT, | |
35 ) | |
36 | |
37 | |
38 DEFAULT_BASE_PATH_DIRECTORY = "_conda" | |
39 DEFAULT_CONDARC_OVERRIDE = "_condarc" | |
40 # Conda channel order from highest to lowest, following the one used in | |
41 # https://github.com/bioconda/bioconda-recipes/blob/master/config.yml | |
42 DEFAULT_ENSURE_CHANNELS = "conda-forge,bioconda,defaults" | |
43 CONDA_SOURCE_CMD = """[ "$(basename "$CONDA_DEFAULT_ENV")" = "$(basename '{environment_path}')" ] || {{ | |
44 MAX_TRIES=3 | |
45 COUNT=0 | |
46 while [ $COUNT -lt $MAX_TRIES ]; do | |
47 . '{activate_path}' '{environment_path}' > conda_activate.log 2>&1 | |
48 if [ $? -eq 0 ];then | |
49 break | |
50 else | |
51 let COUNT=COUNT+1 | |
52 if [ $COUNT -eq $MAX_TRIES ];then | |
53 echo "Failed to activate conda environment! Error was:" | |
54 cat conda_activate.log | |
55 exit 1 | |
56 fi | |
57 sleep 10s | |
58 fi | |
59 done | |
60 }} """ | |
61 | |
62 | |
63 log = logging.getLogger(__name__) | |
64 | |
65 | |
66 class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): | |
67 dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init', 'use_local'] | |
68 resolver_type = "conda" | |
69 config_options = { | |
70 'prefix': None, | |
71 'exec': None, | |
72 'debug': None, | |
73 'ensure_channels': DEFAULT_ENSURE_CHANNELS, | |
74 'auto_install': False, | |
75 'auto_init': True, | |
76 'copy_dependencies': False, | |
77 'use_local': False, | |
78 } | |
79 _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") | |
80 | |
81 def __init__(self, dependency_manager, **kwds): | |
82 read_only = _string_as_bool(kwds.get('read_only', 'false')) | |
83 self.read_only = read_only | |
84 self._setup_mapping(dependency_manager, **kwds) | |
85 self.versionless = _string_as_bool(kwds.get('versionless', 'false')) | |
86 self.dependency_manager = dependency_manager | |
87 | |
88 def get_option(name): | |
89 return dependency_manager.get_resolver_option(self, name, explicit_resolver_options=kwds) | |
90 | |
91 # Conda context options (these define the environment) | |
92 conda_prefix = get_option("prefix") | |
93 if conda_prefix is None: | |
94 conda_prefix = os.path.join( | |
95 dependency_manager.default_base_path, DEFAULT_BASE_PATH_DIRECTORY | |
96 ) | |
97 conda_prefix = os.path.abspath(conda_prefix) | |
98 | |
99 self.conda_prefix_parent = os.path.dirname(conda_prefix) | |
100 | |
101 condarc_override = get_option("condarc_override") | |
102 if condarc_override is None: | |
103 condarc_override = os.path.join( | |
104 dependency_manager.default_base_path, DEFAULT_CONDARC_OVERRIDE | |
105 ) | |
106 | |
107 copy_dependencies = _string_as_bool(get_option("copy_dependencies")) | |
108 use_local = _string_as_bool(get_option("use_local")) | |
109 conda_exec = get_option("exec") | |
110 debug = _string_as_bool(get_option("debug")) | |
111 ensure_channels = get_option("ensure_channels") | |
112 use_path_exec = get_option("use_path_exec") | |
113 if use_path_exec is None: | |
114 use_path_exec = USE_PATH_EXEC_DEFAULT | |
115 else: | |
116 use_path_exec = _string_as_bool(use_path_exec) | |
117 if ensure_channels is None: | |
118 ensure_channels = DEFAULT_ENSURE_CHANNELS | |
119 | |
120 conda_context = CondaContext( | |
121 conda_prefix=conda_prefix, | |
122 conda_exec=conda_exec, | |
123 debug=debug, | |
124 ensure_channels=ensure_channels, | |
125 condarc_override=condarc_override, | |
126 use_path_exec=use_path_exec, | |
127 copy_dependencies=copy_dependencies, | |
128 use_local=use_local, | |
129 ) | |
130 self.use_local = use_local | |
131 self.ensure_channels = ensure_channels | |
132 | |
133 # Conda operations options (these define how resolution will occur) | |
134 auto_install = _string_as_bool(get_option("auto_install")) | |
135 self.auto_init = _string_as_bool(get_option("auto_init")) | |
136 self.conda_context = conda_context | |
137 self.disabled = not galaxy.tool_util.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init) | |
138 if self.auto_init and not self.disabled: | |
139 self.conda_context.ensure_conda_build_installed_if_needed() | |
140 self.auto_install = auto_install | |
141 self.copy_dependencies = copy_dependencies | |
142 | |
143 def clean(self, **kwds): | |
144 return self.conda_context.exec_clean() | |
145 | |
146 def uninstall(self, requirements): | |
147 """Uninstall requirements installed by install_all or multiple install statements.""" | |
148 all_resolved = [r for r in self.resolve_all(requirements) if r.dependency_type] | |
149 if not all_resolved: | |
150 all_resolved = [self.resolve(requirement) for requirement in requirements] | |
151 all_resolved = [r for r in all_resolved if r.dependency_type] | |
152 if not all_resolved: | |
153 return None | |
154 environments = {os.path.basename(dependency.environment_path) for dependency in all_resolved} | |
155 return self.uninstall_environments(environments) | |
156 | |
157 def uninstall_environments(self, environments): | |
158 environments = [env if not env.startswith(self.conda_context.envs_path) else os.path.basename(env) for env in environments] | |
159 return_codes = [self.conda_context.exec_remove([env]) for env in environments] | |
160 final_return_code = 0 | |
161 for env, return_code in zip(environments, return_codes): | |
162 if return_code == 0: | |
163 log.debug("Conda environment '%s' successfully removed." % env) | |
164 else: | |
165 log.debug("Conda environment '%s' could not be removed." % env) | |
166 final_return_code = return_code | |
167 return final_return_code | |
168 | |
169 def install_all(self, conda_targets): | |
170 if self.read_only: | |
171 return False | |
172 | |
173 env = self.merged_environment_name(conda_targets) | |
174 return_code = install_conda_targets(conda_targets, conda_context=self.conda_context, env_name=env) | |
175 if return_code != 0: | |
176 is_installed = False | |
177 else: | |
178 # Recheck if installed | |
179 is_installed = self.conda_context.has_env(env) | |
180 | |
181 if not is_installed: | |
182 log.debug("Removing failed conda install of {}".format(str(conda_targets))) | |
183 cleanup_failed_install_of_environment(env, conda_context=self.conda_context) | |
184 | |
185 return is_installed | |
186 | |
187 def resolve_all(self, requirements, **kwds): | |
188 """ | |
189 Some combinations of tool requirements need to be resolved all at once, so that Conda can select a compatible | |
190 combination of dependencies. This method returns a list of MergedCondaDependency instances (one for each requirement) | |
191 if all requirements have been successfully resolved, or an empty list if any of the requirements could not be resolved. | |
192 | |
193 Parameters specific to this resolver are: | |
194 | |
195 preserve_python_environment: Boolean, controls whether the python environment should be maintained during job creation for tools | |
196 that rely on galaxy being importable. | |
197 | |
198 install: Controls if `requirements` should be installed. If `install` is True and the requirements are not installed | |
199 an attempt is made to install the requirements. If `install` is None requirements will only be installed if | |
200 `conda_auto_install` has been activated and the requirements are not yet installed. If `install` is | |
201 False will not install requirements. | |
202 """ | |
203 if len(requirements) == 0: | |
204 return [] | |
205 | |
206 if not os.path.isdir(self.conda_context.conda_prefix): | |
207 return [] | |
208 | |
209 for requirement in requirements: | |
210 if requirement.type != "package": | |
211 return [] | |
212 | |
213 ToolRequirements = galaxy.tool_util.deps.requirements.ToolRequirements | |
214 expanded_requirements = ToolRequirements([self._expand_requirement(r) for r in requirements]) | |
215 if self.versionless: | |
216 conda_targets = [CondaTarget(r.name, version=None) for r in expanded_requirements] | |
217 else: | |
218 conda_targets = [CondaTarget(r.name, version=r.version) for r in expanded_requirements] | |
219 | |
220 preserve_python_environment = kwds.get("preserve_python_environment", False) | |
221 | |
222 env = self.merged_environment_name(conda_targets) | |
223 dependencies = [] | |
224 | |
225 is_installed = self.conda_context.has_env(env) | |
226 install = kwds.get('install', None) | |
227 if install is None: | |
228 # Default behavior, install dependencies if conda_auto_install is active. | |
229 install = not is_installed and self.auto_install | |
230 elif install: | |
231 # Install has been set to True, install if not yet installed. | |
232 install = not is_installed | |
233 if install: | |
234 is_installed = self.install_all(conda_targets) | |
235 | |
236 if is_installed: | |
237 for requirement in requirements: | |
238 dependency = MergedCondaDependency( | |
239 self.conda_context, | |
240 self.conda_context.env_path(env), | |
241 exact=not self.versionless or requirement.version is None, | |
242 name=requirement.name, | |
243 version=requirement.version, | |
244 preserve_python_environment=preserve_python_environment, | |
245 dependency_resolver=self, | |
246 ) | |
247 dependencies.append(dependency) | |
248 | |
249 return dependencies | |
250 | |
251 def merged_environment_name(self, conda_targets): | |
252 if len(conda_targets) > 1: | |
253 # For continuity with mulled containers this is kind of nice. | |
254 return "mulled-v1-%s" % hash_conda_packages(conda_targets) | |
255 else: | |
256 assert len(conda_targets) == 1 | |
257 return conda_targets[0].install_environment | |
258 | |
259 def resolve(self, requirement, **kwds): | |
260 requirement = self._expand_requirement(requirement) | |
261 name, version, type = requirement.name, requirement.version, requirement.type | |
262 | |
263 # Check for conda just not being there, this way we can enable | |
264 # conda by default and just do nothing in not configured. | |
265 if not os.path.isdir(self.conda_context.conda_prefix): | |
266 return NullDependency(version=version, name=name) | |
267 | |
268 if type != "package": | |
269 return NullDependency(version=version, name=name) | |
270 | |
271 exact = not self.versionless or version is None | |
272 if self.versionless: | |
273 version = None | |
274 | |
275 conda_target = CondaTarget(name, version=version) | |
276 is_installed = is_conda_target_installed( | |
277 conda_target, conda_context=self.conda_context | |
278 ) | |
279 | |
280 preserve_python_environment = kwds.get("preserve_python_environment", False) | |
281 | |
282 job_directory = kwds.get("job_directory", None) | |
283 install = kwds.get('install', None) | |
284 if install is None: | |
285 install = not is_installed and self.auto_install | |
286 elif install: | |
287 install = not is_installed | |
288 if install: | |
289 is_installed = self.install_dependency(name=name, version=version, type=type) | |
290 | |
291 if not is_installed: | |
292 return NullDependency(version=version, name=name) | |
293 | |
294 # Have installed conda_target and job_directory to send it to. | |
295 # If dependency is for metadata generation, store environment in conda-metadata-env | |
296 if kwds.get("metadata", False): | |
297 conda_env = "conda-metadata-env" | |
298 else: | |
299 conda_env = "conda-env" | |
300 | |
301 if job_directory: | |
302 conda_environment = os.path.join(job_directory, conda_env) | |
303 else: | |
304 conda_environment = self.conda_context.env_path(conda_target.install_environment) | |
305 | |
306 return CondaDependency( | |
307 self.conda_context, | |
308 conda_environment, | |
309 exact, | |
310 name, | |
311 version, | |
312 preserve_python_environment=preserve_python_environment, | |
313 ) | |
314 | |
315 def _expand_requirement(self, requirement): | |
316 return self._expand_specs(self._expand_mappings(requirement)) | |
317 | |
318 def unused_dependency_paths(self, toolbox_requirements_status): | |
319 """ | |
320 Identify all local environments that are not needed to build requirements_status. | |
321 | |
322 We try to resolve the requirements, and we note every environment_path that has been taken. | |
323 """ | |
324 used_paths = set() | |
325 for dependencies in toolbox_requirements_status.values(): | |
326 for dependency in dependencies: | |
327 if dependency.get('dependency_type') == 'conda': | |
328 path = os.path.basename(dependency['environment_path']) | |
329 used_paths.add(path) | |
330 dir_contents = set(os.listdir(self.conda_context.envs_path) if os.path.exists(self.conda_context.envs_path) else []) | |
331 unused_paths = dir_contents.difference(used_paths) # New set with paths in dir_contents but not in used_paths | |
332 unused_paths = [os.path.join(self.conda_context.envs_path, p) for p in unused_paths] | |
333 return unused_paths | |
334 | |
335 def list_dependencies(self): | |
336 for install_target in installed_conda_targets(self.conda_context): | |
337 name = install_target.package | |
338 version = install_target.version | |
339 yield self._to_requirement(name, version) | |
340 | |
341 def _install_dependency(self, name, version, type, **kwds): | |
342 "Returns True on (seemingly) successfull installation" | |
343 # should be checked before called | |
344 assert not self.read_only | |
345 | |
346 if type != "package": | |
347 log.warning("Cannot install dependencies of type '%s'" % type) | |
348 return False | |
349 | |
350 if self.versionless: | |
351 version = None | |
352 | |
353 conda_target = CondaTarget(name, version=version) | |
354 | |
355 is_installed = is_conda_target_installed( | |
356 conda_target, conda_context=self.conda_context | |
357 ) | |
358 | |
359 if is_installed: | |
360 return is_installed | |
361 | |
362 return_code = install_conda_target(conda_target, conda_context=self.conda_context) | |
363 if return_code != 0: | |
364 is_installed = False | |
365 else: | |
366 # Recheck if installed | |
367 is_installed = is_conda_target_installed( | |
368 conda_target, conda_context=self.conda_context | |
369 ) | |
370 if not is_installed: | |
371 log.debug(f"Removing failed conda install of {name}, version '{version}'") | |
372 cleanup_failed_install(conda_target, conda_context=self.conda_context) | |
373 | |
374 return is_installed | |
375 | |
376 @property | |
377 def prefix(self): | |
378 return self.conda_context.conda_prefix | |
379 | |
380 | |
381 class MergedCondaDependency(Dependency): | |
382 dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version', 'dependency_resolver'] | |
383 dependency_type = 'conda' | |
384 | |
385 def __init__(self, conda_context, environment_path, exact, name=None, version=None, preserve_python_environment=False, dependency_resolver=None): | |
386 self.activate = conda_context.activate | |
387 self.conda_context = conda_context | |
388 self.environment_path = environment_path | |
389 self._exact = exact | |
390 self._name = name | |
391 self._version = version | |
392 self.cache_path = None | |
393 self._preserve_python_environment = preserve_python_environment | |
394 self.dependency_resolver = dependency_resolver | |
395 | |
396 @property | |
397 def exact(self): | |
398 return self._exact | |
399 | |
400 @property | |
401 def name(self): | |
402 return self._name | |
403 | |
404 @property | |
405 def version(self): | |
406 return self._version | |
407 | |
408 def shell_commands(self): | |
409 if self._preserve_python_environment: | |
410 # On explicit testing the only such requirement I am aware of is samtools - and it seems to work | |
411 # fine with just appending the PATH as done below. Other tools may require additional | |
412 # variables in the future. | |
413 return """export PATH=$PATH:'{}/bin' """.format( | |
414 self.environment_path, | |
415 ) | |
416 else: | |
417 return CONDA_SOURCE_CMD.format( | |
418 activate_path=self.activate, | |
419 environment_path=self.environment_path | |
420 ) | |
421 | |
422 | |
423 class CondaDependency(Dependency): | |
424 dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version', 'dependency_resolver'] | |
425 dependency_type = 'conda' | |
426 cacheable = True | |
427 | |
428 def __init__(self, conda_context, environment_path, exact, name=None, version=None, preserve_python_environment=False, dependency_resolver=None): | |
429 self.activate = conda_context.activate | |
430 self.conda_context = conda_context | |
431 self.environment_path = environment_path | |
432 self._exact = exact | |
433 self._name = name | |
434 self._version = version | |
435 self.cache_path = None | |
436 self._preserve_python_environment = preserve_python_environment | |
437 self.dependency_resolver = dependency_resolver | |
438 | |
439 @property | |
440 def exact(self): | |
441 return self._exact | |
442 | |
443 @property | |
444 def name(self): | |
445 return self._name | |
446 | |
447 @property | |
448 def version(self): | |
449 return self._version | |
450 | |
451 def build_cache(self, cache_path): | |
452 self.set_cache_path(cache_path) | |
453 self.build_environment() | |
454 | |
455 def set_cache_path(self, cache_path): | |
456 self.cache_path = cache_path | |
457 self.environment_path = cache_path | |
458 | |
459 def build_environment(self): | |
460 env_path, exit_code = build_isolated_environment( | |
461 CondaTarget(self.name, self.version), | |
462 conda_context=self.conda_context, | |
463 path=self.environment_path, | |
464 copy=self.conda_context.copy_dependencies, | |
465 ) | |
466 if exit_code: | |
467 if len(os.path.abspath(self.environment_path)) > 79: | |
468 # TODO: remove this once conda_build version 2 is released and packages have been rebuilt. | |
469 raise DependencyException("Conda dependency failed to build job environment. " | |
470 "This is most likely a limitation in conda. " | |
471 "You can try to shorten the path to the job_working_directory.") | |
472 raise DependencyException("Conda dependency seemingly installed but failed to build job environment.") | |
473 | |
474 def shell_commands(self): | |
475 if not self.cache_path: | |
476 # Build an isolated environment if not using a cached dependency manager | |
477 self.build_environment() | |
478 if self._preserve_python_environment: | |
479 # On explicit testing the only such requirement I am aware of is samtools - and it seems to work | |
480 # fine with just appending the PATH as done below. Other tools may require additional | |
481 # variables in the future. | |
482 return """export PATH=$PATH:'{}/bin' """.format( | |
483 self.environment_path, | |
484 ) | |
485 else: | |
486 return CONDA_SOURCE_CMD.format( | |
487 activate_path=self.activate, | |
488 environment_path=self.environment_path | |
489 ) | |
490 | |
491 | |
492 def _string_as_bool(value): | |
493 return str(value).lower() == "true" | |
494 | |
495 | |
496 __all__ = ('CondaDependencyResolver', 'DEFAULT_ENSURE_CHANNELS') |