Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/deps/__init__.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 """ | |
2 Dependency management for tools. | |
3 """ | |
4 | |
5 import json | |
6 import logging | |
7 import os.path | |
8 import shutil | |
9 | |
10 from galaxy.util import ( | |
11 hash_util, | |
12 plugin_config, | |
13 string_as_bool, | |
14 ) | |
15 from galaxy.util.oset import OrderedSet | |
16 from .container_resolvers import ContainerResolver | |
17 from .dependencies import ToolInfo | |
18 from .requirements import ( | |
19 ContainerDescription, | |
20 ToolRequirement, | |
21 ToolRequirements | |
22 ) | |
23 from .resolvers import ( | |
24 ContainerDependency, | |
25 NullDependency, | |
26 ) | |
27 from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver | |
28 | |
29 log = logging.getLogger(__name__) | |
30 | |
31 CONFIG_VAL_NOT_FOUND = object() | |
32 | |
33 | |
34 def build_dependency_manager(app_config_dict=None, resolution_config_dict=None, conf_file=None, default_tool_dependency_dir=None): | |
35 """Build a DependencyManager object from app and/or resolution config. | |
36 | |
37 If app_config_dict is specified, it should be application configuration information | |
38 and configuration options are generally named to identify the context of dependency | |
39 management (e.g. conda_prefix not prefix or use_cached_dependency_manager not cache). | |
40 resolution_config_dict if specified is assumed to be the to_dict() version of a | |
41 DependencyManager and should only contain dependency configuration options. | |
42 """ | |
43 | |
44 if app_config_dict is None: | |
45 app_config_dict = {} | |
46 else: | |
47 app_config_dict = app_config_dict.copy() | |
48 | |
49 tool_dependency_dir = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) | |
50 if tool_dependency_dir and tool_dependency_dir.lower() == "none": | |
51 app_config_dict["tool_dependency_dir"] = None | |
52 | |
53 if resolution_config_dict is None and "dependency_resolution" in app_config_dict: | |
54 resolution_config_dict = app_config_dict["dependency_resolution"] | |
55 | |
56 if resolution_config_dict: | |
57 # Convert local to_dict options into global ones. | |
58 | |
59 # to_dict() has "cache", "cache_dir", "use", "default_base_path", "resolvers", "precache" | |
60 app_config_props_from_resolution_config = { | |
61 "use_tool_dependencies": resolution_config_dict.get("use", None), | |
62 "tool_dependency_dir": resolution_config_dict.get("default_base_path", None), | |
63 "dependency_resolvers": resolution_config_dict.get("resolvers", None), | |
64 "tool_dependency_cache_dir": resolution_config_dict.get("cache_dir", None), | |
65 "precache_dependencies": resolution_config_dict.get("precache", None), | |
66 "use_cached_dependency_manager": resolution_config_dict.get("cache", None), | |
67 } | |
68 | |
69 for key, value in app_config_props_from_resolution_config.items(): | |
70 if value is not None: | |
71 app_config_dict[key] = value | |
72 | |
73 use_tool_dependencies = app_config_dict.get("use_tool_dependencies", None) | |
74 # if we haven't set an explicit True or False, try to infer from config... | |
75 if use_tool_dependencies is None: | |
76 use_tool_dependencies = app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir) is not None or \ | |
77 app_config_dict.get("dependency_resolvers") or \ | |
78 (conf_file and os.path.exists(conf_file)) | |
79 | |
80 if use_tool_dependencies: | |
81 dependency_manager_kwds = { | |
82 "default_base_path": app_config_dict.get("tool_dependency_dir", default_tool_dependency_dir), | |
83 "conf_file": conf_file, | |
84 "app_config": app_config_dict, | |
85 } | |
86 if string_as_bool(app_config_dict.get("use_cached_dependency_manager")): | |
87 dependency_manager = CachedDependencyManager(**dependency_manager_kwds) | |
88 else: | |
89 dependency_manager = DependencyManager(**dependency_manager_kwds) | |
90 else: | |
91 dependency_manager = NullDependencyManager() | |
92 | |
93 return dependency_manager | |
94 | |
95 | |
96 class DependencyManager: | |
97 """ | |
98 A DependencyManager attempts to resolve named and versioned dependencies by | |
99 searching for them under a list of directories. Directories should be | |
100 of the form: | |
101 | |
102 $BASE/name/version/... | |
103 | |
104 and should each contain a file 'env.sh' which can be sourced to make the | |
105 dependency available in the current shell environment. | |
106 """ | |
107 cached = False | |
108 | |
109 def __init__(self, default_base_path, conf_file=None, app_config=None): | |
110 """ | |
111 Create a new dependency manager looking for packages under the paths listed | |
112 in `base_paths`. The default base path is app.config.tool_dependency_dir. | |
113 """ | |
114 if app_config is None: | |
115 app_config = {} | |
116 if not os.path.exists(default_base_path): | |
117 log.warning("Path '%s' does not exist, ignoring", default_base_path) | |
118 if not os.path.isdir(default_base_path): | |
119 log.warning("Path '%s' is not directory, ignoring", default_base_path) | |
120 self.__app_config = app_config | |
121 self.default_base_path = os.path.abspath(default_base_path) | |
122 self.resolver_classes = self.__resolvers_dict() | |
123 | |
124 plugin_source = None | |
125 dependency_resolver_dicts = app_config.get("dependency_resolvers") | |
126 if dependency_resolver_dicts is not None: | |
127 plugin_source = plugin_config.PluginConfigSource('dict', dependency_resolver_dicts) | |
128 else: | |
129 plugin_source = self.__build_dependency_resolvers_plugin_source(conf_file) | |
130 self.dependency_resolvers = self.__parse_resolver_conf_plugins(plugin_source) | |
131 self._enabled_container_types = [] | |
132 self._destination_for_container_type = {} | |
133 | |
134 def set_enabled_container_types(self, container_types_to_destinations): | |
135 """Set the union of all enabled container types.""" | |
136 self._enabled_container_types = [container_type for container_type in container_types_to_destinations.keys()] | |
137 # Just pick first enabled destination for a container type, probably covers the most common deployment scenarios | |
138 self._destination_for_container_type = container_types_to_destinations | |
139 | |
140 def get_destination_info_for_container_type(self, container_type, destination_id=None): | |
141 if destination_id is None: | |
142 return next(iter(self._destination_for_container_type[container_type])).params | |
143 else: | |
144 for destination in self._destination_for_container_type[container_type]: | |
145 if destination.id == destination_id: | |
146 return destination.params | |
147 | |
148 @property | |
149 def enabled_container_types(self): | |
150 """Returns the union of enabled container types.""" | |
151 return self._enabled_container_types | |
152 | |
153 def get_resolver_option(self, resolver, key, explicit_resolver_options=None): | |
154 """Look in resolver-specific settings for option and then fallback to global settings. | |
155 """ | |
156 if explicit_resolver_options is None: | |
157 explicit_resolver_options = {} | |
158 default = resolver.config_options.get(key) | |
159 config_prefix = resolver.resolver_type | |
160 global_key = f"{config_prefix}_{key}" | |
161 value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND) | |
162 if value is CONFIG_VAL_NOT_FOUND: | |
163 value = self.get_app_option(global_key, default) | |
164 | |
165 return value | |
166 | |
167 def get_app_option(self, key, default=None): | |
168 value = CONFIG_VAL_NOT_FOUND | |
169 if isinstance(self.__app_config, dict): | |
170 value = self.__app_config.get(key, CONFIG_VAL_NOT_FOUND) | |
171 else: | |
172 value = getattr(self.__app_config, key, CONFIG_VAL_NOT_FOUND) | |
173 if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"): | |
174 value = self.__app_config.config_dict.get(key, CONFIG_VAL_NOT_FOUND) | |
175 if value is CONFIG_VAL_NOT_FOUND: | |
176 value = default | |
177 return value | |
178 | |
179 @property | |
180 def precache(self): | |
181 return string_as_bool(self.get_app_option("precache_dependencies", True)) | |
182 | |
183 def dependency_shell_commands(self, requirements, **kwds): | |
184 requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds) | |
185 ordered_dependencies = OrderedSet(requirements_to_dependencies.values()) | |
186 return [dependency.shell_commands() for dependency in ordered_dependencies if not isinstance(dependency, ContainerDependency)] | |
187 | |
188 def requirements_to_dependencies(self, requirements, **kwds): | |
189 """ | |
190 Takes a list of requirements and returns a dictionary | |
191 with requirements as key and dependencies as value caching | |
192 these on the tool instance if supplied. | |
193 """ | |
194 requirement_to_dependency = self._requirements_to_dependencies_dict(requirements, **kwds) | |
195 | |
196 if 'tool_instance' in kwds: | |
197 kwds['tool_instance'].dependencies = [dep.to_dict() for dep in requirement_to_dependency.values()] | |
198 | |
199 return requirement_to_dependency | |
200 | |
201 def _requirements_to_dependencies_dict(self, requirements, search=False, **kwds): | |
202 """Build simple requirements to dependencies dict for resolution.""" | |
203 requirement_to_dependency = {} | |
204 index = kwds.get('index') | |
205 install = kwds.get('install', False) | |
206 resolver_type = kwds.get('resolver_type') | |
207 include_containers = kwds.get('include_containers', False) | |
208 container_type = kwds.get('container_type') | |
209 require_exact = kwds.get('exact', False) | |
210 return_null_dependencies = kwds.get('return_null', False) | |
211 | |
212 resolvable_requirements = requirements.resolvable | |
213 | |
214 tool_info_kwds = dict(requirements=resolvable_requirements) | |
215 if 'tool_instance' in kwds: | |
216 tool = kwds['tool_instance'] | |
217 tool_info_kwds['tool_id'] = tool.id | |
218 tool_info_kwds['tool_version'] = tool.version | |
219 tool_info_kwds['container_descriptions'] = tool.containers | |
220 tool_info_kwds['requires_galaxy_python_environment'] = tool.requires_galaxy_python_environment | |
221 | |
222 tool_info = ToolInfo(**tool_info_kwds) | |
223 | |
224 for i, resolver in enumerate(self.dependency_resolvers): | |
225 | |
226 if index is not None and i != index: | |
227 continue | |
228 | |
229 if resolver_type is not None and resolver.resolver_type != resolver_type: | |
230 continue | |
231 | |
232 if container_type is not None and getattr(resolver, "container_type", None) != container_type: | |
233 continue | |
234 | |
235 _requirement_to_dependency = {k: v for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)} | |
236 | |
237 if len(_requirement_to_dependency) == len(resolvable_requirements): | |
238 # Shortcut - resolution complete. | |
239 break | |
240 | |
241 # Check requirements all at once | |
242 all_unmet = len(_requirement_to_dependency) == 0 | |
243 if hasattr(resolver, "resolve_all"): | |
244 resolve = resolver.resolve_all | |
245 elif isinstance(resolver, ContainerResolver): | |
246 if not include_containers: | |
247 continue | |
248 if not install and resolver.builds_on_resolution: | |
249 # don't want to build images here | |
250 continue | |
251 if not resolver.resolver_type.startswith(('cached', 'explicit', 'fallback')) and not (search or install): | |
252 # These would look up available containers using the quay API, | |
253 # we only want to do this if we search for containers | |
254 continue | |
255 resolve = resolver.resolve | |
256 else: | |
257 resolve = None | |
258 if all_unmet and resolve is not None: | |
259 # TODO: Handle specs. | |
260 dependencies = resolve(requirements=resolvable_requirements, | |
261 enabled_container_types=self.enabled_container_types, | |
262 destination_for_container_type=self.get_destination_info_for_container_type, | |
263 tool_info=tool_info, | |
264 **kwds) | |
265 if dependencies: | |
266 if isinstance(dependencies, ContainerDescription): | |
267 dependencies = [ContainerDependency(dependencies, name=r.name, version=r.version, container_resolver=resolver) for r in resolvable_requirements] | |
268 assert len(dependencies) == len(resolvable_requirements) | |
269 for requirement, dependency in zip(resolvable_requirements, dependencies): | |
270 log.debug(dependency.resolver_msg) | |
271 requirement_to_dependency[requirement] = dependency | |
272 | |
273 # Shortcut - resolution complete. | |
274 break | |
275 | |
276 if not isinstance(resolver, ContainerResolver): | |
277 | |
278 # Check individual requirements | |
279 for requirement in resolvable_requirements: | |
280 if requirement in _requirement_to_dependency: | |
281 continue | |
282 | |
283 dependency = resolver.resolve(requirement, **kwds) | |
284 if require_exact and not dependency.exact: | |
285 continue | |
286 | |
287 if not isinstance(dependency, NullDependency): | |
288 log.debug(dependency.resolver_msg) | |
289 requirement_to_dependency[requirement] = dependency | |
290 elif return_null_dependencies: | |
291 log.debug(dependency.resolver_msg) | |
292 dependency.version = requirement.version | |
293 requirement_to_dependency[requirement] = dependency | |
294 | |
295 return requirement_to_dependency | |
296 | |
297 def uses_tool_shed_dependencies(self): | |
298 return any(map(lambda r: isinstance(r, ToolShedPackageDependencyResolver), self.dependency_resolvers)) | |
299 | |
300 def find_dep(self, name, version=None, type='package', **kwds): | |
301 log.debug(f'Find dependency {name} version {version}') | |
302 requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)]) | |
303 dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds) | |
304 if len(dep_dict) > 0: | |
305 return next(iter(dep_dict.values())) # get first dep | |
306 else: | |
307 return NullDependency(name=name, version=version) | |
308 | |
309 def __build_dependency_resolvers_plugin_source(self, conf_file): | |
310 if not conf_file: | |
311 return self.__default_dependency_resolvers_source() | |
312 if not os.path.exists(conf_file): | |
313 log.debug("Unable to find config file '%s'", conf_file) | |
314 return self.__default_dependency_resolvers_source() | |
315 plugin_source = plugin_config.plugin_source_from_path(conf_file) | |
316 return plugin_source | |
317 | |
318 def __default_dependency_resolvers_source(self): | |
319 return plugin_config.PluginConfigSource('dict', [ | |
320 {"type": "tool_shed_packages"}, | |
321 {"type": "galaxy_packages"}, | |
322 {"type": "conda"}, | |
323 {"type": "galaxy_packages", "versionless": True}, | |
324 {"type": "conda", "versionless": True}, | |
325 ]) | |
326 | |
327 def __parse_resolver_conf_plugins(self, plugin_source): | |
328 """ | |
329 """ | |
330 extra_kwds = dict(dependency_manager=self) | |
331 # Use either 'type' from YAML definition or 'resolver_type' from to_dict definition. | |
332 return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds, plugin_type_keys=['type', 'resolver_type']) | |
333 | |
334 def __resolvers_dict(self): | |
335 import galaxy.tool_util.deps.resolvers | |
336 return plugin_config.plugins_dict(galaxy.tool_util.deps.resolvers, 'resolver_type') | |
337 | |
338 def to_dict(self): | |
339 return { | |
340 "use": True, | |
341 "cache": self.cached, | |
342 "precache": self.precache, | |
343 "cache_dir": getattr(self, "tool_dependency_cache_dir", None), | |
344 "default_base_path": self.default_base_path, | |
345 "resolvers": [m.to_dict() for m in self.dependency_resolvers], | |
346 } | |
347 | |
348 | |
349 class CachedDependencyManager(DependencyManager): | |
350 cached = True | |
351 | |
352 def __init__(self, default_base_path, **kwd): | |
353 super().__init__(default_base_path=default_base_path, **kwd) | |
354 self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") or os.path.join(default_base_path, "_cache") | |
355 | |
356 def build_cache(self, requirements, **kwds): | |
357 resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) | |
358 cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable] | |
359 hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies) | |
360 if os.path.exists(hashed_dependencies_dir): | |
361 if kwds.get('force_rebuild', False): | |
362 try: | |
363 shutil.rmtree(hashed_dependencies_dir) | |
364 except Exception: | |
365 log.warning("Could not delete cached dependencies directory '%s'" % hashed_dependencies_dir) | |
366 raise | |
367 else: | |
368 log.debug("Cached dependencies directory '%s' already exists, skipping build", hashed_dependencies_dir) | |
369 return | |
370 [dep.build_cache(hashed_dependencies_dir) for dep in cacheable_dependencies] | |
371 | |
372 def dependency_shell_commands(self, requirements, **kwds): | |
373 """ | |
374 Runs a set of requirements through the dependency resolvers and returns | |
375 a list of commands required to activate the dependencies. If dependencies | |
376 are cacheable and the cache does not exist, will try to create it. | |
377 If cached environment exists or is successfully created, will generate | |
378 commands to activate it. | |
379 """ | |
380 resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) | |
381 cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable] | |
382 hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies) | |
383 if not os.path.exists(hashed_dependencies_dir) and self.precache: | |
384 # Cache not present, try to create it | |
385 self.build_cache(requirements, **kwds) | |
386 if os.path.exists(hashed_dependencies_dir): | |
387 [dep.set_cache_path(hashed_dependencies_dir) for dep in cacheable_dependencies] | |
388 commands = [dep.shell_commands() for dep in resolved_dependencies.values()] | |
389 return commands | |
390 | |
391 def hash_dependencies(self, resolved_dependencies): | |
392 """Return hash for dependencies""" | |
393 resolved_dependencies = [(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies] | |
394 hash_str = json.dumps(sorted(resolved_dependencies)) | |
395 return hash_util.new_secure_hash(hash_str)[:8] # short hash | |
396 | |
397 def get_hashed_dependencies_path(self, resolved_dependencies): | |
398 """ | |
399 Returns the path to the hashed dependencies directory (but does not evaluate whether the path exists). | |
400 | |
401 :param resolved_dependencies: list of resolved dependencies | |
402 :type resolved_dependencies: list | |
403 | |
404 :return: path | |
405 :rtype: str | |
406 """ | |
407 req_hashes = self.hash_dependencies(resolved_dependencies) | |
408 return os.path.abspath(os.path.join(self.tool_dependency_cache_dir, req_hashes)) | |
409 | |
410 | |
411 class NullDependencyManager(DependencyManager): | |
412 cached = False | |
413 | |
414 def __init__(self, default_base_path=None, conf_file=None, app_config=None): | |
415 if app_config is None: | |
416 app_config = {} | |
417 self.__app_config = app_config | |
418 self.resolver_classes = set() | |
419 self.dependency_resolvers = [] | |
420 self._enabled_container_types = [] | |
421 self._destination_for_container_type = {} | |
422 self.default_base_path = None | |
423 | |
424 def uses_tool_shed_dependencies(self): | |
425 return False | |
426 | |
427 def dependency_shell_commands(self, requirements, **kwds): | |
428 return [] | |
429 | |
430 def find_dep(self, name, version=None, type='package', **kwds): | |
431 return NullDependency(version=version, name=name) | |
432 | |
433 def to_dict(self): | |
434 return {"use": False} |