Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/deps/views.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 from galaxy import exceptions | |
2 from galaxy.util import asbool, listify | |
3 from .dependencies import ToolInfo | |
4 from .resolvers import ( | |
5 ContainerDependency, | |
6 NullDependency, | |
7 ) | |
8 | |
9 | |
10 class DependencyResolversView: | |
11 """ Provide a RESTfulish/JSONy interface to a galaxy.tool_util.deps.DependencyResolver | |
12 object. This can be adapted by the Galaxy web framework or other web apps. | |
13 """ | |
14 | |
15 def __init__(self, app): | |
16 self._app = app | |
17 | |
18 def index(self): | |
19 return [r.to_dict() for r in self._dependency_resolvers] | |
20 | |
21 def show(self, index): | |
22 return self._dependency_resolver(index).to_dict() | |
23 | |
24 def reload(self): | |
25 self.toolbox.reload_dependency_manager() | |
26 | |
27 def manager_requirements(self): | |
28 requirements = [] | |
29 for index, resolver in enumerate(self._dependency_resolvers): | |
30 if not hasattr(resolver, "list_dependencies"): | |
31 continue | |
32 for requirement in resolver.list_dependencies(): | |
33 requirements.append({"index": index, "requirement": requirement.to_dict()}) | |
34 return requirements | |
35 | |
36 def resolver_requirements(self, index): | |
37 requirements = [] | |
38 resolver = self._dependency_resolver(index) | |
39 if not hasattr(resolver, "list_dependencies"): | |
40 raise exceptions.NotImplemented() | |
41 for requirement in resolver.list_dependencies(): | |
42 requirements.append(requirement.to_dict()) | |
43 return requirements | |
44 | |
45 def manager_dependency(self, **kwds): | |
46 return self._dependency(**kwds) | |
47 | |
48 def resolver_dependency(self, index, **kwds): | |
49 return self._dependency(**kwds) | |
50 | |
51 def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None, **kwd): | |
52 """ | |
53 Resolves dependencies to build a requirements status in the admin panel/API | |
54 """ | |
55 to_deps_kwds = { | |
56 'install': False, | |
57 'return_null': True, | |
58 'installed_tool_dependencies': installed_tool_dependencies | |
59 } | |
60 to_deps_kwds.update(kwd) | |
61 dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, | |
62 **to_deps_kwds) | |
63 for tool, requirements in tool_requirements_d.items()} | |
64 return dependencies_per_tool | |
65 | |
66 def uninstall_dependencies(self, index=None, resolver_type=None, container_type=None, **payload): | |
67 """Attempt to uninstall requirements. Returns 0 if successfull, else None.""" | |
68 requirements = payload.get('requirements') | |
69 if not requirements: | |
70 return None | |
71 if index: | |
72 resolver = self._dependency_resolvers[index] | |
73 if resolver.can_uninstall_dependencies: | |
74 return resolver.uninstall(requirements) | |
75 elif resolver_type: | |
76 for resolver in self._dependency_resolvers: | |
77 if resolver.resolver_type == resolver_type and resolver.can_uninstall_dependencies: | |
78 return resolver.uninstall(requirements) | |
79 elif container_type: | |
80 for resolver in self._dependency_resolvers: | |
81 if getattr(resolver, "container_type", None) == container_type and resolver.can_uninstall_dependencies: | |
82 return resolver.uninstall(requirements) | |
83 else: | |
84 for index in self.uninstallable_resolvers: | |
85 return_code = self._dependency_resolvers[index].uninstall(requirements) | |
86 if return_code == 0: | |
87 return return_code | |
88 return None | |
89 | |
90 @property | |
91 def unused_dependency_paths(self): | |
92 """List dependencies that are not currently installed.""" | |
93 unused_dependencies = [] | |
94 toolbox_requirements_status = self.toolbox_requirements_status | |
95 for resolver in self._dependency_resolvers: | |
96 if hasattr(resolver, 'unused_dependency_paths'): | |
97 unused_dependencies.extend(resolver.unused_dependency_paths(toolbox_requirements_status)) | |
98 return set(unused_dependencies) | |
99 | |
100 def remove_unused_dependency_paths(self, envs): | |
101 """ | |
102 Remove dependencies that are not currently used. | |
103 | |
104 Returns a list of all environments that have been successfully removed. | |
105 """ | |
106 envs_to_remove = set(envs) | |
107 toolbox_requirements_status = self.toolbox_requirements_status | |
108 removed_environments = set() | |
109 for resolver in self._dependency_resolvers: | |
110 if hasattr(resolver, 'unused_dependency_paths') and hasattr(resolver, 'uninstall_environments'): | |
111 unused_dependencies = resolver.unused_dependency_paths(toolbox_requirements_status) | |
112 can_remove = envs_to_remove & set(unused_dependencies) | |
113 exit_code = resolver.uninstall_environments(can_remove) | |
114 if exit_code == 0: | |
115 removed_environments = removed_environments.union(can_remove) | |
116 envs_to_remove = envs_to_remove.difference(can_remove) | |
117 return list(removed_environments) | |
118 | |
119 def install_dependencies(self, requirements, **kwds): | |
120 kwds['install'] = True | |
121 return self._dependency_manager._requirements_to_dependencies_dict(requirements, **kwds) | |
122 | |
123 def install_dependency(self, index=None, **payload): | |
124 """ | |
125 Installs dependency using highest priority resolver that supports dependency installation | |
126 (Currently only the conda resolver supports this). If index is given, attempt | |
127 installation directly using the corresponding resolver. | |
128 Returns True on success, False on failure. | |
129 payload is dictionary that must container name, version and type, | |
130 e.g. {'name': 'numpy', version='1.9.1', type='package'} | |
131 """ | |
132 if index: | |
133 return self._install_dependency(index, **payload) | |
134 else: | |
135 for index in self.installable_resolvers: | |
136 success = self._install_dependency(index, **payload) | |
137 if success: | |
138 return success | |
139 return False | |
140 | |
141 def _install_dependency(self, index, **payload): | |
142 """ | |
143 Resolver install dependency should return True when installation succeeds, | |
144 False if not successful | |
145 """ | |
146 resolver = self._dependency_resolver(index) | |
147 if resolver.read_only: | |
148 raise exceptions.RequestParameterInvalidException("Attempted to install on a read_only dependency resolver.") | |
149 if resolver.disabled: | |
150 raise exceptions.RequestParameterInvalidException("Attempted to install on a disabled dependency resolver.") | |
151 | |
152 name, version, type, extra_kwds = self._parse_dependency_info(payload) | |
153 return resolver.install_dependency( | |
154 name=name, | |
155 version=version, | |
156 type=type, | |
157 **extra_kwds | |
158 ) | |
159 | |
160 def _dependency(self, index=None, **kwds): | |
161 if index is not None: | |
162 index = int(index) | |
163 | |
164 name, version, type, extra_kwds = self._parse_dependency_info(kwds) | |
165 resolve_kwds = dict( | |
166 job_directory=None, | |
167 index=index, | |
168 **extra_kwds | |
169 ) | |
170 dependency = self._dependency_manager.find_dep( | |
171 name, version=version, type=type, **resolve_kwds | |
172 ) | |
173 return dependency.to_dict() | |
174 | |
175 def _parse_dependency_info(self, kwds): | |
176 extra_kwds = kwds.copy() | |
177 name = extra_kwds.pop("name", None) | |
178 if name is None: | |
179 raise exceptions.RequestParameterMissingException("Missing 'name' parameter required for resolution.") | |
180 version = extra_kwds.pop("version", None) | |
181 type = extra_kwds.pop("type", "package") | |
182 return name, version, type, extra_kwds | |
183 | |
184 def _dependency_resolver(self, index): | |
185 index = int(index) | |
186 return self._dependency_resolvers[index] | |
187 | |
188 @property | |
189 def _dependency_manager(self): | |
190 return self._app.toolbox.dependency_manager | |
191 | |
192 @property | |
193 def _dependency_resolvers(self): | |
194 dependency_manager = self._dependency_manager | |
195 dependency_resolvers = dependency_manager.dependency_resolvers | |
196 return dependency_resolvers | |
197 | |
198 @property | |
199 def installable_resolvers(self): | |
200 """ | |
201 List index for all active resolvers that have the 'install_dependency' attribute. | |
202 """ | |
203 return [index for index, resolver in enumerate(self._dependency_resolvers) if not resolver.read_only and not resolver.disabled] | |
204 | |
205 @property | |
206 def uninstallable_resolvers(self): | |
207 """ | |
208 List index for all active resolvers that can uninstall dependencies that have been installed through this resolver. | |
209 """ | |
210 return [index for index, resolver in enumerate(self._dependency_resolvers) if resolver.can_uninstall_dependencies and not resolver.disabled] | |
211 | |
212 @property | |
213 def tool_ids_by_requirements(self): | |
214 """Dictionary with requirements as keys, and tool_ids as values.""" | |
215 tool_ids_by_requirements = {} | |
216 if not self._app.toolbox.tools_by_id: | |
217 return {} | |
218 for tid, tool in self._app.toolbox.tools_by_id.items(): | |
219 if tool.tool_requirements not in tool_ids_by_requirements: | |
220 tool_ids_by_requirements[tool.tool_requirements] = [tid] | |
221 else: | |
222 tool_ids_by_requirements[tool.tool_requirements].append(tid) | |
223 return tool_ids_by_requirements | |
224 | |
225 @property | |
226 def toolbox_requirements_status(self): | |
227 return self.summarize_requirements() | |
228 | |
229 def summarize_requirements(self, **kwds): | |
230 summary_kwds = {} | |
231 if 'index' in kwds: | |
232 summary_kwds['index'] = int(kwds['index']) | |
233 if 'include_containers' in kwds: | |
234 summary_kwds['include_containers'] = asbool(kwds['include_containers']) | |
235 if 'container_type' in kwds: | |
236 summary_kwds['container_type'] = kwds['container_type'] | |
237 if 'resolver_type' in kwds: | |
238 summary_kwds['resolver_type'] = kwds['resolver_type'] | |
239 if 'search' in kwds: | |
240 summary_kwds['search'] = asbool(kwds['search']) | |
241 if 'install' in kwds: | |
242 summary_kwds['install'] = asbool(kwds['install']) | |
243 | |
244 tool_ids_by_requirements = self.tool_ids_by_requirements | |
245 statuses = {r: self.get_requirements_status(tool_requirements_d={tids[0]: r}, | |
246 installed_tool_dependencies=self._app.toolbox.tools_by_id[tids[0]].installed_tool_dependencies, **summary_kwds) | |
247 for r, tids in tool_ids_by_requirements.items()} | |
248 if kwds.get("for_json", False): | |
249 # All public attributes of this class should be returning JSON - this is meant to mimic a restful API. | |
250 rval = [] | |
251 for requirements, status in statuses.items(): | |
252 item = {} | |
253 item["requirements"] = requirements.to_dict() | |
254 item["status"] = status | |
255 item["tool_ids"] = tool_ids_by_requirements[requirements] | |
256 rval.append(item) | |
257 statuses = rval | |
258 return statuses | |
259 | |
260 def summarize_tools(self, **kwds): | |
261 summary_kwds = {} | |
262 if 'index' in kwds: | |
263 summary_kwds['index'] = int(kwds['index']) | |
264 if 'include_containers' in kwds: | |
265 summary_kwds['include_containers'] = asbool(kwds['include_containers']) | |
266 if 'container_type' in kwds: | |
267 summary_kwds['container_type'] = kwds['container_type'] | |
268 if 'resolver_type' in kwds: | |
269 summary_kwds['resolver_type'] = kwds['resolver_type'] | |
270 if 'search' in kwds: | |
271 summary_kwds['search'] = asbool(kwds['search']) | |
272 if 'install' in kwds: | |
273 summary_kwds['install'] = asbool(kwds['install']) | |
274 | |
275 tool_ids = pop_tool_ids(kwds) | |
276 | |
277 rval = [] | |
278 for tid, tool in self._app.toolbox.tools_by_id.items(): | |
279 if tool_ids and tid not in tool_ids: | |
280 continue | |
281 | |
282 requirements = tool.tool_requirements | |
283 status = self.get_requirements_status(tool_requirements_d={tid: requirements}, | |
284 installed_tool_dependencies=tool.installed_tool_dependencies, | |
285 tool_instance=tool, **summary_kwds) | |
286 item = {} | |
287 item["requirements"] = requirements.to_dict() | |
288 item["status"] = status | |
289 item["tool_ids"] = [tid] | |
290 rval.append(item) | |
291 return rval | |
292 | |
293 def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None, **kwd): | |
294 dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies, **kwd) | |
295 # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool. | |
296 # We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates. | |
297 requirements = [r.resolvable for r in tool_requirements_d.values()] | |
298 flat_tool_requirements = set().union(*requirements) | |
299 flat_dependencies = [] | |
300 for requirements_odict in dependencies.values(): | |
301 for requirement in requirements_odict: | |
302 if requirement in flat_tool_requirements: | |
303 flat_dependencies.append(requirements_odict[requirement]) | |
304 flat_tool_requirements.remove(requirement) | |
305 return [d.to_dict() for d in flat_dependencies] | |
306 | |
307 def clean(self, index=None, **kwds): | |
308 if index: | |
309 resolver = self._dependency_resolver(index) | |
310 if not hasattr(resolver, "clean"): | |
311 raise exceptions.NotImplemented() | |
312 else: | |
313 resolver.clean() | |
314 return "OK" | |
315 else: | |
316 [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')] | |
317 return "OK" | |
318 | |
319 | |
320 class ContainerResolutionView: | |
321 """ | |
322 """ | |
323 | |
324 def __init__(self, app): | |
325 self._app = app | |
326 | |
327 def index(self): | |
328 return [r.to_dict() for r in self._container_resolvers] | |
329 | |
330 def show(self, index): | |
331 return self._container_resolver(index).to_dict() | |
332 | |
333 def resolve(self, **kwds): | |
334 find_best_kwds = { | |
335 'install': False, | |
336 'enabled_container_types': ['docker', 'singularity'], | |
337 'resolution_cache': kwds.get("resolution_cache"), | |
338 'session': kwds.get('session'), | |
339 } | |
340 | |
341 if 'index' in kwds: | |
342 find_best_kwds['index'] = int(kwds['index']) | |
343 if 'container_type' in kwds: | |
344 find_best_kwds['enabled_container_types'] = [kwds['container_type']] | |
345 if 'resolver_type' in kwds: | |
346 find_best_kwds['resolver_type'] = kwds['resolver_type'] | |
347 if 'install' in kwds: | |
348 find_best_kwds['install'] = asbool(kwds['install']) | |
349 | |
350 tool_info_kwds = {} | |
351 tool_id = kwds["tool_id"] | |
352 tool = self._app.toolbox.tools_by_id[tool_id] | |
353 | |
354 requirements = tool.tool_requirements | |
355 tool_info_kwds = dict(requirements=requirements) | |
356 requirements_only = asbool(kwds.get("requirements_only", False)) | |
357 # If requirements_only, simply use the tool to load a requirement set from, | |
358 # mimics the default behavior of searching for containers through the dependency resolution | |
359 # component. Not useful for tool execution but perhaps when summarizing mulled containers for requirements. | |
360 if not requirements_only: | |
361 tool_info_kwds['container_descriptions'] = tool.containers | |
362 tool_info_kwds['requires_galaxy_python_environment'] = tool.requires_galaxy_python_environment | |
363 tool_info_kwds['tool_id'] = tool.id | |
364 tool_info_kwds['tool_version'] = tool.version | |
365 | |
366 find_best_kwds["tool_info"] = ToolInfo(**tool_info_kwds) | |
367 | |
368 # Consider implementing 'search' to match dependency resolution API. | |
369 resolved_container_description = self._app.container_finder.resolve( | |
370 **find_best_kwds | |
371 ) | |
372 if resolved_container_description: | |
373 status = ContainerDependency(resolved_container_description.container_description, container_resolver=resolved_container_description.container_resolver).to_dict() | |
374 else: | |
375 status = NullDependency().to_dict() | |
376 return { | |
377 "tool_id": kwds["tool_id"], | |
378 "status": status, | |
379 "requirements": requirements.to_dict() | |
380 } | |
381 | |
382 def resolve_toolbox(self, **kwds): | |
383 rval = [] | |
384 resolve_kwds = kwds.copy() | |
385 tool_ids = pop_tool_ids(resolve_kwds) | |
386 resolve_kwds["resolution_cache"] = self._app.container_finder.resolution_cache() | |
387 if tool_ids is not None: | |
388 tool_ids = listify(tool_ids) | |
389 for tool_id, tool in self._app.toolbox.tools_by_id.items(): | |
390 if tool_ids is not None and tool_id not in tool_ids: | |
391 continue | |
392 | |
393 if tool.tool_action.produces_real_jobs: | |
394 rval.append(self.resolve(tool_id=tool_id, **resolve_kwds)) | |
395 return rval | |
396 | |
397 @property | |
398 def _container_resolvers(self): | |
399 return self._app.container_finder.container_resolvers | |
400 | |
401 def _container_resolver(self, index): | |
402 index = int(index) | |
403 return self._container_resolvers[index] | |
404 | |
405 | |
406 def pop_tool_ids(kwds): | |
407 tool_ids = None | |
408 if "tool_ids" in kwds: | |
409 tool_ids = listify(kwds.pop("tool_ids")) | |
410 if "tool_ids[]" in kwds: | |
411 tool_ids = listify(kwds.pop("tool_ids[]")) | |
412 if "tool_id" in kwds: | |
413 tool_ids = [kwds.pop("tool_id")] | |
414 return tool_ids |