Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/util/tool_shed/common_util.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 import errno | |
2 import json | |
3 import logging | |
4 import os | |
5 from urllib.parse import urljoin | |
6 | |
7 from routes import url_for | |
8 | |
9 from galaxy import util | |
10 from galaxy.util.tool_shed import encoding_util, xml_util | |
11 | |
12 log = logging.getLogger(__name__) | |
13 | |
14 REPOSITORY_OWNER = 'devteam' | |
15 # not valid for installed Galaxy, fix | |
16 MIGRATE_DIR = os.path.abspath(os.path.join( | |
17 os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'galaxy', 'tool_shed', 'galaxy_install', 'migrate')) | |
18 TOOL_MIGRATION_SCRIPTS_DIR = os.path.join(MIGRATE_DIR, 'scripts') | |
19 TOOL_MIGRATION_VERSIONS_DIR = os.path.join(MIGRATE_DIR, 'versions') | |
20 | |
21 | |
22 def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies): | |
23 if tool_shed_accessible: | |
24 if tool_dependencies: | |
25 for tool_dependency in tool_dependencies: | |
26 if tool_dependency not in all_tool_dependencies: | |
27 all_tool_dependencies.append(tool_dependency) | |
28 return all_tool_dependencies | |
29 | |
30 | |
31 def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number): | |
32 # Get the 000x_tools.xml file associated with the current migrate_tools version number. | |
33 tools_xml_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), | |
34 os.pardir, 'galaxy_install', | |
35 'migrate', 'scripts', | |
36 '%04d_tools.xml' % latest_tool_migration_script_number)) | |
37 # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. | |
38 migrated_tool_configs_dict = {} | |
39 tree, error_message = xml_util.parse_xml(tools_xml_file_path) | |
40 if tree is None: | |
41 return False, {} | |
42 root = tree.getroot() | |
43 tool_shed = root.get('name') | |
44 tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) | |
45 # The default behavior is that the tool shed is down. | |
46 tool_shed_accessible = False | |
47 missing_tool_configs_dict = {} | |
48 if tool_shed_url: | |
49 for elem in root: | |
50 if elem.tag == 'repository': | |
51 repository_dependencies = [] | |
52 all_tool_dependencies = [] | |
53 repository_name = elem.get('name') | |
54 changeset_revision = elem.get('changeset_revision') | |
55 tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app, | |
56 tool_shed_url, | |
57 repository_name, | |
58 REPOSITORY_OWNER, | |
59 changeset_revision) | |
60 if tool_shed_accessible: | |
61 # Accumulate all tool dependencies defined for repository dependencies for display to the user. | |
62 for rd_key, rd_tups in repository_dependencies_dict.items(): | |
63 if rd_key in ['root_key', 'description']: | |
64 continue | |
65 for rd_tup in rd_tups: | |
66 tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ | |
67 parse_repository_dependency_tuple(rd_tup) | |
68 tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, | |
69 tool_shed_url, | |
70 name, | |
71 owner, | |
72 changeset_revision) | |
73 all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) | |
74 tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, | |
75 tool_shed_url, | |
76 repository_name, | |
77 REPOSITORY_OWNER, | |
78 changeset_revision) | |
79 all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) | |
80 for tool_elem in elem.findall('tool'): | |
81 tool_config_file_name = tool_elem.get('file') | |
82 if tool_config_file_name: | |
83 # We currently do nothing with repository dependencies except install them (we do not display repositories that will be | |
84 # installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the | |
85 # future. | |
86 dependencies_dict = dict(tool_dependencies=all_tool_dependencies, | |
87 repository_dependencies=repository_dependencies) | |
88 migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict | |
89 else: | |
90 break | |
91 if tool_shed_accessible: | |
92 # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. | |
93 for tool_panel_config in tool_panel_configs: | |
94 tree, error_message = xml_util.parse_xml(tool_panel_config) | |
95 if tree: | |
96 root = tree.getroot() | |
97 for elem in root: | |
98 if elem.tag == 'tool': | |
99 missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict) | |
100 elif elem.tag == 'section': | |
101 for section_elem in elem: | |
102 if section_elem.tag == 'tool': | |
103 missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict) | |
104 else: | |
105 exception_msg = f'\n\nThe entry for the main Galaxy tool shed at {tool_shed} is missing from the {app.config.tool_sheds_config} file. ' | |
106 exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' | |
107 raise Exception(exception_msg) | |
108 return tool_shed_accessible, missing_tool_configs_dict | |
109 | |
110 | |
111 def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict): | |
112 file_path = elem.get('file', None) | |
113 if file_path: | |
114 name = os.path.basename(file_path) | |
115 for migrated_tool_config in migrated_tool_configs_dict.keys(): | |
116 if migrated_tool_config in [file_path, name]: | |
117 missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config] | |
118 return missing_tool_configs_dict | |
119 | |
120 | |
121 def generate_clone_url_for_installed_repository(app, repository): | |
122 """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" | |
123 tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed)) | |
124 return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)]) | |
125 | |
126 | |
127 def generate_clone_url_for_repository_in_tool_shed(user, repository): | |
128 """Generate the URL for cloning a repository that is in the tool shed.""" | |
129 base_url = url_for('/', qualified=True).rstrip('/') | |
130 if user: | |
131 protocol, base = base_url.split('://') | |
132 username = '%s@' % user.username | |
133 return f'{protocol}://{username}{base}/repos/{repository.user.username}/{repository.name}' | |
134 else: | |
135 return f'{base_url}/repos/{repository.user.username}/{repository.name}' | |
136 | |
137 | |
138 def generate_clone_url_from_repo_info_tup(app, repo_info_tup): | |
139 """Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision.""" | |
140 # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] | |
141 toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ | |
142 parse_repository_dependency_tuple(repo_info_tup) | |
143 tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed) | |
144 # Don't include the changeset_revision in clone urls. | |
145 return util.build_url(tool_shed_url, pathspec=['repos', owner, name]) | |
146 | |
147 | |
148 def get_non_shed_tool_panel_configs(app): | |
149 """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" | |
150 config_filenames = [] | |
151 for config_filename in app.config.tool_configs: | |
152 # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. | |
153 # <toolbox tool_path="database/shed_tools"> | |
154 try: | |
155 tree, error_message = xml_util.parse_xml(config_filename) | |
156 except OSError as exc: | |
157 if (config_filename == app.config.shed_tool_conf and not | |
158 app.config.shed_tool_conf_set and | |
159 exc.errno == errno.ENOENT): | |
160 continue | |
161 raise | |
162 if tree is None: | |
163 continue | |
164 root = tree.getroot() | |
165 tool_path = root.get('tool_path', None) | |
166 if tool_path is None: | |
167 config_filenames.append(config_filename) | |
168 return config_filenames | |
169 | |
170 | |
171 def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): | |
172 repository_dependencies_dict = {} | |
173 tool_shed_accessible = True | |
174 params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) | |
175 pathspec = ['repository', 'get_repository_dependencies'] | |
176 try: | |
177 raw_text = util.url_get(tool_shed_url, auth=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) | |
178 tool_shed_accessible = True | |
179 except Exception as e: | |
180 tool_shed_accessible = False | |
181 log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) | |
182 if tool_shed_accessible: | |
183 if len(raw_text) > 2: | |
184 encoded_text = json.loads(util.unicodify(raw_text)) | |
185 repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) | |
186 return tool_shed_accessible, repository_dependencies_dict | |
187 | |
188 | |
189 def get_protocol_from_tool_shed_url(tool_shed_url): | |
190 """Return the protocol from the received tool_shed_url if it exists.""" | |
191 try: | |
192 if tool_shed_url.find('://') > 0: | |
193 return tool_shed_url.split('://')[0].lower() | |
194 except Exception: | |
195 # We receive a lot of calls here where the tool_shed_url is None. The container_util uses | |
196 # that value when creating a header row. If the tool_shed_url is not None, we have a problem. | |
197 if tool_shed_url is not None: | |
198 log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url)) | |
199 # Default to HTTP protocol. | |
200 return 'http' | |
201 | |
202 | |
203 def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): | |
204 tool_dependencies = [] | |
205 tool_shed_accessible = True | |
206 params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) | |
207 pathspec = ['repository', 'get_tool_dependencies'] | |
208 try: | |
209 text = util.url_get(tool_shed_url, auth=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) | |
210 tool_shed_accessible = True | |
211 except Exception as e: | |
212 tool_shed_accessible = False | |
213 log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) | |
214 if tool_shed_accessible: | |
215 if text: | |
216 tool_dependencies_dict = encoding_util.tool_shed_decode(text) | |
217 for requirements_dict in tool_dependencies_dict.values(): | |
218 tool_dependency_name = requirements_dict['name'] | |
219 tool_dependency_version = requirements_dict['version'] | |
220 tool_dependency_type = requirements_dict['type'] | |
221 tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type)) | |
222 return tool_shed_accessible, tool_dependencies | |
223 | |
224 | |
225 def get_tool_shed_repository_ids(as_string=False, **kwd): | |
226 tsrid = kwd.get('tool_shed_repository_id', None) | |
227 tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None)) | |
228 if not tsridslist: | |
229 tsridslist = util.listify(kwd.get('id', None)) | |
230 if tsridslist is not None: | |
231 if tsrid is not None and tsrid not in tsridslist: | |
232 tsridslist.append(tsrid) | |
233 if as_string: | |
234 return ','.join(tsridslist) | |
235 return tsridslist | |
236 else: | |
237 tsridslist = util.listify(kwd.get('ordered_tsr_ids', None)) | |
238 if tsridslist is not None: | |
239 if as_string: | |
240 return ','.join(tsridslist) | |
241 return tsridslist | |
242 if as_string: | |
243 return '' | |
244 return [] | |
245 | |
246 | |
247 def get_tool_shed_url_from_tool_shed_registry(app, tool_shed): | |
248 """ | |
249 The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is | |
250 something like: http://toolshed.g2.bx.psu.edu/ | |
251 """ | |
252 cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed) | |
253 for shed_url in app.tool_shed_registry.tool_sheds.values(): | |
254 if shed_url.find(cleaned_tool_shed) >= 0: | |
255 if shed_url.endswith('/'): | |
256 shed_url = shed_url.rstrip('/') | |
257 return shed_url | |
258 # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. | |
259 return None | |
260 | |
261 | |
262 def get_tool_shed_repository_url(app, tool_shed, owner, name): | |
263 tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) | |
264 if tool_shed_url: | |
265 # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate | |
266 # the last part of a URL if it does not end with a forward slash. | |
267 tool_shed_url = '%s/' % tool_shed_url | |
268 return urljoin(tool_shed_url, f'view/{owner}/{name}') | |
269 return tool_shed_url | |
270 | |
271 | |
272 def get_user_by_username(app, username): | |
273 """Get a user from the database by username.""" | |
274 sa_session = app.model.context.current | |
275 try: | |
276 user = sa_session.query(app.model.User) \ | |
277 .filter(app.model.User.table.c.username == username) \ | |
278 .one() | |
279 return user | |
280 except Exception: | |
281 return None | |
282 | |
283 | |
284 def handle_galaxy_url(trans, **kwd): | |
285 galaxy_url = kwd.get('galaxy_url', None) | |
286 if galaxy_url: | |
287 trans.set_cookie(galaxy_url, name='toolshedgalaxyurl') | |
288 else: | |
289 galaxy_url = trans.get_cookie(name='toolshedgalaxyurl') | |
290 return galaxy_url | |
291 | |
292 | |
293 def handle_tool_shed_url_protocol(app, shed_url): | |
294 """Handle secure and insecure HTTP protocol since they may change over time.""" | |
295 try: | |
296 if app.name == 'galaxy': | |
297 url = remove_protocol_from_tool_shed_url(shed_url) | |
298 tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url) | |
299 else: | |
300 tool_shed_url = str(url_for('/', qualified=True)).rstrip('/') | |
301 return tool_shed_url | |
302 except Exception: | |
303 # We receive a lot of calls here where the tool_shed_url is None. The container_util uses | |
304 # that value when creating a header row. If the tool_shed_url is not None, we have a problem. | |
305 if shed_url is not None: | |
306 log.exception("Handled exception removing protocol from URL %s", str(shed_url)) | |
307 return shed_url | |
308 | |
309 | |
310 def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False): | |
311 # Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the | |
312 # repository containing the repository_dependency definition. | |
313 prior_installation_required = 'False' | |
314 only_if_compiling_contained_td = 'False' | |
315 if contains_error: | |
316 if len(repository_dependency_tuple) == 5: | |
317 tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple | |
318 elif len(repository_dependency_tuple) == 6: | |
319 tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple | |
320 elif len(repository_dependency_tuple) == 7: | |
321 tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ | |
322 repository_dependency_tuple | |
323 return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error | |
324 else: | |
325 if len(repository_dependency_tuple) == 4: | |
326 tool_shed, name, owner, changeset_revision = repository_dependency_tuple | |
327 elif len(repository_dependency_tuple) == 5: | |
328 tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple | |
329 elif len(repository_dependency_tuple) == 6: | |
330 tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple | |
331 return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td | |
332 | |
333 | |
334 def remove_port_from_tool_shed_url(tool_shed_url): | |
335 """Return a partial Tool Shed URL, eliminating the port if it exists.""" | |
336 try: | |
337 if tool_shed_url.find(':') > 0: | |
338 # Eliminate the port, if any, since it will result in an invalid directory name. | |
339 new_tool_shed_url = tool_shed_url.split(':')[0] | |
340 else: | |
341 new_tool_shed_url = tool_shed_url | |
342 return new_tool_shed_url.rstrip('/') | |
343 except Exception: | |
344 # We receive a lot of calls here where the tool_shed_url is None. The container_util uses | |
345 # that value when creating a header row. If the tool_shed_url is not None, we have a problem. | |
346 if tool_shed_url is not None: | |
347 log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url)) | |
348 return tool_shed_url | |
349 | |
350 | |
351 def remove_protocol_and_port_from_tool_shed_url(tool_shed_url): | |
352 """Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists.""" | |
353 tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) | |
354 tool_shed = remove_port_from_tool_shed_url(tool_shed) | |
355 return tool_shed | |
356 | |
357 | |
358 def remove_protocol_and_user_from_clone_url(repository_clone_url): | |
359 """Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists.""" | |
360 if repository_clone_url.find('@') > 0: | |
361 # We have an url that includes an authenticated user, something like: | |
362 # http://test@bx.psu.edu:9009/repos/some_username/column | |
363 items = repository_clone_url.split('@') | |
364 tmp_url = items[1] | |
365 elif repository_clone_url.find('//') > 0: | |
366 # We have an url that includes only a protocol, something like: | |
367 # http://bx.psu.edu:9009/repos/some_username/column | |
368 items = repository_clone_url.split('//') | |
369 tmp_url = items[1] | |
370 else: | |
371 tmp_url = repository_clone_url | |
372 return tmp_url.rstrip('/') | |
373 | |
374 | |
375 def remove_protocol_from_tool_shed_url(tool_shed_url): | |
376 """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" | |
377 return util.remove_protocol_from_url(tool_shed_url) |