Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/deps/resolvers/lmod.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 """ | |
2 This is a prototype dependency resolver to be able to use the "LMOD environment modules system" from TACC to solve package requirements | |
3 | |
4 LMOD official website: https://www.tacc.utexas.edu/research-development/tacc-projects/lmod | |
5 | |
6 LMOD @ Github: https://github.com/TACC/Lmod | |
7 | |
8 """ | |
9 import logging | |
10 from io import StringIO | |
11 from os import getenv | |
12 from os.path import exists | |
13 from subprocess import ( | |
14 PIPE, | |
15 Popen | |
16 ) | |
17 | |
18 from . import ( | |
19 Dependency, | |
20 DependencyResolver, | |
21 MappableDependencyResolver, | |
22 NullDependency, | |
23 ) | |
24 | |
25 log = logging.getLogger(__name__) | |
26 | |
27 DEFAULT_LMOD_PATH = getenv('LMOD_CMD') | |
28 DEFAULT_SETTARG_PATH = getenv('LMOD_SETTARG_CMD') | |
29 DEFAULT_MODULEPATH = getenv('MODULEPATH') | |
30 DEFAULT_MAPPING_FILE = 'config/lmod_modules_mapping.yml' | |
31 INVALID_LMOD_PATH_MSG = "The following LMOD executable could not be found: %s. Either your LMOD Dependency Resolver is misconfigured or LMOD is improperly installed on your system !" | |
32 EMPTY_MODULEPATH_MSG = "No valid LMOD MODULEPATH defined ! Either your LMOD Dependency Resolver is misconfigured or LMOD is improperly installed on your system !" | |
33 | |
34 | |
35 class LmodDependencyResolver(DependencyResolver, MappableDependencyResolver): | |
36 """Dependency resolver based on the LMOD environment modules system""" | |
37 | |
38 dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath'] | |
39 resolver_type = "lmod" | |
40 | |
41 def __init__(self, dependency_manager, **kwds): | |
42 # Mapping file management | |
43 self._set_default_mapping_file(kwds) | |
44 self._setup_mapping(dependency_manager, **kwds) | |
45 | |
46 # Other attributes | |
47 self.versionless = _string_as_bool(kwds.get('versionless', 'false')) | |
48 self.lmodexec = kwds.get('lmodexec', DEFAULT_LMOD_PATH) | |
49 self.settargexec = kwds.get('settargexec', DEFAULT_SETTARG_PATH) | |
50 self.modulepath = kwds.get('modulepath', DEFAULT_MODULEPATH) | |
51 self.module_checker = AvailModuleChecker(self, self.modulepath) | |
52 | |
53 def _set_default_mapping_file(self, resolver_attributes): | |
54 if 'mapping_files' not in resolver_attributes: | |
55 if exists(DEFAULT_MAPPING_FILE): | |
56 resolver_attributes['mapping_files'] = DEFAULT_MAPPING_FILE | |
57 | |
58 def resolve(self, requirement, **kwds): | |
59 requirement = self._expand_mappings(requirement) | |
60 name, version, type = requirement.name, requirement.version, requirement.type | |
61 | |
62 if type != "package": | |
63 return NullDependency(version=version, name=name) | |
64 | |
65 if self.__has_module(name, version): | |
66 return LmodDependency(self, name, version, exact=True, dependency_resolver=self) | |
67 elif self.versionless and self.__has_module(name, None): | |
68 return LmodDependency(self, name, None, exact=False, dependency_resolver=self) | |
69 | |
70 return NullDependency(version=version, name=name) | |
71 | |
72 def __has_module(self, name, version): | |
73 return self.module_checker.has_module(name, version) | |
74 | |
75 | |
76 class AvailModuleChecker: | |
77 """Parses the output of Lmod 'module avail' command to get the list of available modules.""" | |
78 | |
79 def __init__(self, lmod_dependency_resolver, modulepath): | |
80 self.lmod_dependency_resolver = lmod_dependency_resolver | |
81 self.modulepath = modulepath | |
82 | |
83 def has_module(self, module, version): | |
84 # When version is None (No specific version required by the wrapper -or- versionless is set to 'true'), we only get the list of default modules | |
85 # We get the full list of modules otherwise | |
86 if version is None: | |
87 available_modules = self.__get_list_of_available_modules(True) | |
88 else: | |
89 available_modules = self.__get_list_of_available_modules(False) | |
90 | |
91 # Is the required module in the list of avaialable modules ? | |
92 for module_name, module_version in available_modules: | |
93 names_match = module == module_name | |
94 module_match = names_match and (version is None or module_version == version) | |
95 if module_match: | |
96 return True | |
97 return False | |
98 | |
99 def __get_list_of_available_modules(self, default_version_only=False): | |
100 # Get the results of the "module avail" command in an easy to parse format | |
101 # Note that since "module" is actually a bash function, we are directy executing the underlying executable instead | |
102 raw_output = self.__get_module_avail_command_output(default_version_only).decode("utf-8") | |
103 | |
104 # Parse the result | |
105 for line in StringIO(raw_output): | |
106 # Clean line and discard non-module lines | |
107 line = line and line.strip() | |
108 if not line or line.startswith("/"): | |
109 continue | |
110 | |
111 # Split module lines by / to separate the module name from the module version | |
112 # Module without version are discarded | |
113 module_parts = line.split('/') | |
114 if len(module_parts) == 2: | |
115 yield module_parts[0], module_parts[1] | |
116 | |
117 def __get_module_avail_command_output(self, default_version_only=False): | |
118 # Check if the LMOD executable is available (ie. if both LMOD and the lmod dependency resolver are both setup properly) | |
119 lmodexec = self.lmod_dependency_resolver.lmodexec | |
120 if not exists(lmodexec): | |
121 raise Exception(INVALID_LMOD_PATH_MSG % lmodexec) | |
122 | |
123 # Check if the MODULEPATH environment | |
124 if self.modulepath == "" or self.modulepath is None: | |
125 raise Exception(EMPTY_MODULEPATH_MSG) | |
126 | |
127 # Build command line | |
128 if default_version_only: | |
129 module_avail_command = [lmodexec, '-t', '-d', 'avail'] | |
130 else: | |
131 module_avail_command = [lmodexec, '-t', 'avail'] | |
132 | |
133 # The list of avaialable modules is actually printed on stderr and not stdout for module commands | |
134 return Popen(module_avail_command, stdout=PIPE, stderr=PIPE, env={'MODULEPATH': self.modulepath}, close_fds=True).communicate()[1] | |
135 | |
136 | |
137 class LmodDependency(Dependency): | |
138 """Prepare the commands required to solve the dependency and add them to the script used to run a tool in Galaxy.""" | |
139 | |
140 dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['module_name', 'module_version', 'dependency_resolver'] | |
141 dependency_type = 'lmod' | |
142 | |
143 def __init__(self, lmod_dependency_resolver, module_name, module_version=None, exact=True, dependency_resolver=None): | |
144 self.lmod_dependency_resolver = lmod_dependency_resolver | |
145 self.module_name = module_name | |
146 self.module_version = module_version | |
147 self._exact = exact | |
148 self.dependency_resolver = dependency_resolver | |
149 | |
150 @property | |
151 def name(self): | |
152 return self.module_name | |
153 | |
154 @property | |
155 def version(self): | |
156 return self.module_version | |
157 | |
158 @property | |
159 def exact(self): | |
160 return self._exact | |
161 | |
162 def shell_commands(self): | |
163 # Get the full module name in the form "tool_name/tool_version" | |
164 module_to_load = self.module_name | |
165 if self.module_version: | |
166 module_to_load = f'{self.module_name}/{self.module_version}' | |
167 | |
168 # Build the list of command to add to run script | |
169 # Note that since "module" is actually a bash function, we are directy executing the underlying executable instead | |
170 # - Set the MODULEPATH environment variable | |
171 command = 'MODULEPATH=%s; ' % (self.lmod_dependency_resolver.modulepath) | |
172 command += 'export MODULEPATH; ' | |
173 # - Execute the "module load" command (or rather the "/path/to/lmod load" command) | |
174 command += f'eval `{self.lmod_dependency_resolver.lmodexec} load {module_to_load}` ' | |
175 # - Execute the "settarg" command in addition if needed | |
176 if self.lmod_dependency_resolver.settargexec is not None: | |
177 command += '&& eval `%s -s sh`' % (self.lmod_dependency_resolver.settargexec) | |
178 | |
179 return command | |
180 | |
181 | |
182 def _string_as_bool(value): | |
183 return str(value).lower() == "true" | |
184 | |
185 | |
186 __all__ = ('LmodDependencyResolver', ) |