Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/verify/script.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 #!/usr/bin/env python | |
2 | |
3 import argparse | |
4 import datetime as dt | |
5 import json | |
6 import logging | |
7 import os | |
8 import sys | |
9 import tempfile | |
10 from collections import namedtuple | |
11 from concurrent.futures import thread, ThreadPoolExecutor | |
12 | |
13 import yaml | |
14 | |
15 from galaxy.tool_util.verify.interactor import ( | |
16 DictClientTestConfig, | |
17 GalaxyInteractorApi, | |
18 verify_tool, | |
19 ) | |
20 | |
21 DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance.""" | |
22 DEFAULT_SUITE_NAME = "Galaxy Tool Tests" | |
23 ALL_TESTS = -1 | |
24 ALL_TOOLS = "*" | |
25 ALL_VERSION = "*" | |
26 LATEST_VERSION = None | |
27 | |
28 | |
29 TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"]) | |
30 TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"]) | |
31 | |
32 | |
33 class Results: | |
34 | |
35 def __init__(self, default_suitename, test_json, append=False, galaxy_url=None): | |
36 self.test_json = test_json or "-" | |
37 self.galaxy_url = galaxy_url | |
38 test_results = [] | |
39 test_exceptions = [] | |
40 suitename = default_suitename | |
41 if append: | |
42 assert test_json != "-" | |
43 with open(test_json) as f: | |
44 previous_results = json.load(f) | |
45 test_results = previous_results["tests"] | |
46 if "suitename" in previous_results: | |
47 suitename = previous_results["suitename"] | |
48 self.test_results = test_results | |
49 self.test_exceptions = test_exceptions | |
50 self.suitename = suitename | |
51 | |
52 def register_result(self, result): | |
53 self.test_results.append(result) | |
54 | |
55 def register_exception(self, test_exception): | |
56 self.test_exceptions.append(test_exception) | |
57 | |
58 def already_successful(self, test_reference): | |
59 test_data = self._previous_test_data(test_reference) | |
60 if test_data: | |
61 if 'status' in test_data and test_data['status'] == 'success': | |
62 return True | |
63 | |
64 return False | |
65 | |
66 def already_executed(self, test_reference): | |
67 test_data = self._previous_test_data(test_reference) | |
68 if test_data: | |
69 if 'status' in test_data and test_data['status'] != 'skipped': | |
70 return True | |
71 | |
72 return False | |
73 | |
74 def _previous_test_data(self, test_reference): | |
75 test_id = _test_id_for_reference(test_reference) | |
76 for test_result in self.test_results: | |
77 if test_result.get('id') != test_id: | |
78 continue | |
79 | |
80 has_data = test_result.get('has_data', False) | |
81 if has_data: | |
82 test_data = test_result.get("data", {}) | |
83 return test_data | |
84 | |
85 return None | |
86 | |
87 def write(self): | |
88 tests = sorted(self.test_results, key=lambda el: el['id']) | |
89 n_passed, n_failures, n_skips = 0, 0, 0 | |
90 n_errors = len([e for e in self.test_exceptions if not e.was_recorded]) | |
91 for test in tests: | |
92 has_data = test.get('has_data', False) | |
93 if has_data: | |
94 test_data = test.get("data", {}) | |
95 if 'status' not in test_data: | |
96 raise Exception(f"Test result data {test_data} doesn't contain a status key.") | |
97 status = test_data['status'] | |
98 if status == "success": | |
99 n_passed += 1 | |
100 elif status == "error": | |
101 n_errors += 1 | |
102 elif status == "skip": | |
103 n_skips += 1 | |
104 elif status == "failure": | |
105 n_failures += 1 | |
106 report_obj = { | |
107 'version': '0.1', | |
108 'suitename': self.suitename, | |
109 'results': { | |
110 'total': n_passed + n_failures + n_skips + n_errors, | |
111 'errors': n_errors, | |
112 'failures': n_failures, | |
113 'skips': n_skips, | |
114 }, | |
115 'tests': tests, | |
116 } | |
117 if self.galaxy_url: | |
118 report_obj['galaxy_url'] = self.galaxy_url | |
119 if self.test_json == "-": | |
120 print(json.dumps(report_obj)) | |
121 else: | |
122 with open(self.test_json, "w") as f: | |
123 json.dump(report_obj, f) | |
124 | |
125 def info_message(self): | |
126 messages = [] | |
127 passed_tests = self._tests_with_status('success') | |
128 messages.append("Passed tool tests ({}): {}".format( | |
129 len(passed_tests), | |
130 [t["id"] for t in passed_tests] | |
131 )) | |
132 failed_tests = self._tests_with_status('failure') | |
133 messages.append("Failed tool tests ({}): {}".format( | |
134 len(failed_tests), | |
135 [t["id"] for t in failed_tests] | |
136 )) | |
137 skiped_tests = self._tests_with_status('skip') | |
138 messages.append("Skipped tool tests ({}): {}".format( | |
139 len(skiped_tests), | |
140 [t["id"] for t in skiped_tests] | |
141 )) | |
142 errored_tests = self._tests_with_status('error') | |
143 messages.append("Errored tool tests ({}): {}".format( | |
144 len(errored_tests), | |
145 [t["id"] for t in errored_tests] | |
146 )) | |
147 return "\n".join(messages) | |
148 | |
149 @property | |
150 def success_count(self): | |
151 self._tests_with_status('success') | |
152 | |
153 @property | |
154 def skip_count(self): | |
155 self._tests_with_status('skip') | |
156 | |
157 @property | |
158 def error_count(self): | |
159 return self._tests_with_status('error') + len(self.test_exceptions) | |
160 | |
161 @property | |
162 def failure_count(self): | |
163 return self._tests_with_status('failure') | |
164 | |
165 def _tests_with_status(self, status): | |
166 return [t for t in self.test_results if t.get("data", {}).get("status") == status] | |
167 | |
168 | |
169 def test_tools( | |
170 galaxy_interactor, | |
171 test_references, | |
172 results, | |
173 log=None, | |
174 parallel_tests=1, | |
175 history_per_test_case=False, | |
176 no_history_cleanup=False, | |
177 publish_history=False, | |
178 retries=0, | |
179 verify_kwds=None, | |
180 ): | |
181 """Run through tool tests and write report. | |
182 | |
183 Refactor this into Galaxy in 21.01. | |
184 """ | |
185 verify_kwds = (verify_kwds or {}).copy() | |
186 tool_test_start = dt.datetime.now() | |
187 history_created = False | |
188 if history_per_test_case: | |
189 test_history = None | |
190 else: | |
191 history_created = True | |
192 test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}", publish_history=publish_history) | |
193 verify_kwds.update({ | |
194 "no_history_cleanup": no_history_cleanup, | |
195 "test_history": test_history, | |
196 }) | |
197 with ThreadPoolExecutor(max_workers=parallel_tests) as executor: | |
198 try: | |
199 for test_reference in test_references: | |
200 _test_tool( | |
201 executor=executor, | |
202 test_reference=test_reference, | |
203 results=results, | |
204 galaxy_interactor=galaxy_interactor, | |
205 log=log, | |
206 retries=retries, | |
207 verify_kwds=verify_kwds, | |
208 publish_history=publish_history, | |
209 ) | |
210 finally: | |
211 # Always write report, even if test was cancelled. | |
212 try: | |
213 executor.shutdown(wait=True) | |
214 except KeyboardInterrupt: | |
215 executor._threads.clear() | |
216 thread._threads_queues.clear() | |
217 results.write() | |
218 if log: | |
219 if results.test_json == "-": | |
220 destination = 'standard output' | |
221 else: | |
222 destination = os.path.abspath(results.test_json) | |
223 log.info(f"Report written to '{destination}'") | |
224 log.info(results.info_message()) | |
225 log.info("Total tool test time: {}".format(dt.datetime.now() - tool_test_start)) | |
226 if history_created and not no_history_cleanup: | |
227 galaxy_interactor.delete_history(test_history) | |
228 | |
229 | |
230 def _test_id_for_reference(test_reference): | |
231 tool_id = test_reference.tool_id | |
232 tool_version = test_reference.tool_version | |
233 test_index = test_reference.test_index | |
234 | |
235 if tool_version and tool_id.endswith("/" + tool_version): | |
236 tool_id = tool_id[:-len("/" + tool_version)] | |
237 | |
238 label_base = tool_id | |
239 if tool_version: | |
240 label_base += "/" + str(tool_version) | |
241 | |
242 test_id = label_base + "-" + str(test_index) | |
243 return test_id | |
244 | |
245 | |
246 def _test_tool( | |
247 executor, | |
248 test_reference, | |
249 results, | |
250 galaxy_interactor, | |
251 log, | |
252 retries, | |
253 publish_history, | |
254 verify_kwds, | |
255 ): | |
256 tool_id = test_reference.tool_id | |
257 tool_version = test_reference.tool_version | |
258 test_index = test_reference.test_index | |
259 # If given a tool_id with a version suffix, strip it off so we can treat tool_version | |
260 # correctly at least in client_test_config. | |
261 if tool_version and tool_id.endswith("/" + tool_version): | |
262 tool_id = tool_id[:-len("/" + tool_version)] | |
263 | |
264 test_id = _test_id_for_reference(test_reference) | |
265 | |
266 def run_test(): | |
267 run_retries = retries | |
268 job_data = None | |
269 job_exception = None | |
270 | |
271 def register(job_data_): | |
272 nonlocal job_data | |
273 job_data = job_data_ | |
274 | |
275 try: | |
276 while run_retries >= 0: | |
277 job_exception = None | |
278 try: | |
279 if log: | |
280 log.info("Executing test '%s'", test_id) | |
281 verify_tool( | |
282 tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, | |
283 register_job_data=register, publish_history=publish_history, **verify_kwds | |
284 ) | |
285 if log: | |
286 log.info("Test '%s' passed", test_id) | |
287 break | |
288 except Exception as e: | |
289 if log: | |
290 log.warning("Test '%s' failed", test_id, exc_info=True) | |
291 | |
292 job_exception = e | |
293 run_retries -= 1 | |
294 finally: | |
295 if job_data is not None: | |
296 results.register_result({ | |
297 "id": test_id, | |
298 "has_data": True, | |
299 "data": job_data, | |
300 }) | |
301 if job_exception is not None: | |
302 was_recorded = job_data is not None | |
303 test_exception = TestException(tool_id, job_exception, was_recorded) | |
304 results.register_exception(test_exception) | |
305 | |
306 executor.submit(run_test) | |
307 | |
308 | |
309 def build_case_references( | |
310 galaxy_interactor, | |
311 tool_id=ALL_TOOLS, | |
312 tool_version=LATEST_VERSION, | |
313 test_index=ALL_TESTS, | |
314 page_size=0, | |
315 page_number=0, | |
316 test_filters=None, | |
317 log=None, | |
318 ): | |
319 test_references = [] | |
320 if tool_id == ALL_TOOLS: | |
321 tests_summary = galaxy_interactor.get_tests_summary() | |
322 for tool_id, tool_versions_dict in tests_summary.items(): | |
323 for tool_version, summary in tool_versions_dict.items(): | |
324 for test_index in range(summary["count"]): | |
325 test_reference = TestReference(tool_id, tool_version, test_index) | |
326 test_references.append(test_reference) | |
327 else: | |
328 assert tool_id | |
329 tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {} | |
330 for i, tool_test_dict in enumerate(tool_test_dicts): | |
331 this_tool_version = tool_test_dict.get("tool_version", tool_version) | |
332 this_test_index = i | |
333 if test_index == ALL_TESTS or i == test_index: | |
334 test_reference = TestReference(tool_id, this_tool_version, this_test_index) | |
335 test_references.append(test_reference) | |
336 | |
337 if test_filters is not None and len(test_filters) > 0: | |
338 filtered_test_references = [] | |
339 for test_reference in test_references: | |
340 skip_test = False | |
341 for test_filter in test_filters: | |
342 if test_filter(test_reference): | |
343 if log is not None: | |
344 log.debug(f"Filtering test for {test_reference}, skipping") | |
345 skip_test = True | |
346 if not skip_test: | |
347 filtered_test_references.append(test_reference) | |
348 log.info(f"Skipping {len(test_references)-len(filtered_test_references)} out of {len(test_references)} tests.") | |
349 test_references = filtered_test_references | |
350 | |
351 if page_size > 0: | |
352 slice_start = page_size * page_number | |
353 slice_end = page_size * (page_number + 1) | |
354 test_references = test_references[slice_start:slice_end] | |
355 | |
356 return test_references | |
357 | |
358 | |
359 def main(argv=None): | |
360 if argv is None: | |
361 argv = sys.argv[1:] | |
362 | |
363 args = arg_parser().parse_args(argv) | |
364 run_tests(args) | |
365 | |
366 | |
367 def run_tests(args, test_filters=None, log=None): | |
368 # Split out argument parsing so we can quickly build other scripts - such as a script | |
369 # to run all tool tests for a workflow by just passing in a custom test_filters. | |
370 test_filters = test_filters or [] | |
371 log = log or setup_global_logger(__name__, verbose=args.verbose) | |
372 | |
373 client_test_config_path = args.client_test_config | |
374 if client_test_config_path is not None: | |
375 log.debug(f"Reading client config path {client_test_config_path}") | |
376 with open(client_test_config_path) as f: | |
377 client_test_config = yaml.full_load(f) | |
378 else: | |
379 client_test_config = {} | |
380 | |
381 def get_option(key): | |
382 arg_val = getattr(args, key, None) | |
383 if arg_val is None and key in client_test_config: | |
384 val = client_test_config.get(key) | |
385 else: | |
386 val = arg_val | |
387 return val | |
388 | |
389 output_json_path = get_option("output_json") | |
390 galaxy_url = get_option("galaxy_url") | |
391 galaxy_interactor_kwds = { | |
392 "galaxy_url": galaxy_url, | |
393 "master_api_key": get_option("admin_key"), | |
394 "api_key": get_option("key"), | |
395 "keep_outputs_dir": args.output, | |
396 "download_attempts": get_option("download_attempts"), | |
397 "download_sleep": get_option("download_sleep"), | |
398 "test_data": get_option("test_data"), | |
399 } | |
400 tool_id = args.tool_id | |
401 tool_version = args.tool_version | |
402 tools_client_test_config = DictClientTestConfig(client_test_config.get("tools")) | |
403 verbose = args.verbose | |
404 | |
405 galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds) | |
406 results = Results(args.suite_name, output_json_path, append=args.append, galaxy_url=galaxy_url) | |
407 | |
408 skip = args.skip | |
409 if skip == "executed": | |
410 test_filters.append(results.already_executed) | |
411 elif skip == "successful": | |
412 test_filters.append(results.already_successful) | |
413 | |
414 test_references = build_case_references( | |
415 galaxy_interactor, | |
416 tool_id=tool_id, | |
417 tool_version=tool_version, | |
418 test_index=args.test_index, | |
419 page_size=args.page_size, | |
420 page_number=args.page_number, | |
421 test_filters=test_filters, | |
422 log=log, | |
423 ) | |
424 log.debug(f"Built {len(test_references)} test references to executed.") | |
425 verify_kwds = dict( | |
426 client_test_config=tools_client_test_config, | |
427 force_path_paste=args.force_path_paste, | |
428 skip_with_reference_data=not args.with_reference_data, | |
429 quiet=not verbose, | |
430 ) | |
431 test_tools( | |
432 galaxy_interactor, | |
433 test_references, | |
434 results, | |
435 log=log, | |
436 parallel_tests=args.parallel_tests, | |
437 history_per_test_case=args.history_per_test_case, | |
438 no_history_cleanup=args.no_history_cleanup, | |
439 publish_history=get_option("publish_history"), | |
440 verify_kwds=verify_kwds, | |
441 ) | |
442 exceptions = results.test_exceptions | |
443 if exceptions: | |
444 exception = exceptions[0] | |
445 if hasattr(exception, "exception"): | |
446 exception = exception.exception | |
447 raise exception | |
448 | |
449 | |
450 def setup_global_logger(name, log_file=None, verbose=False): | |
451 formatter = logging.Formatter('%(asctime)s %(levelname)-5s - %(message)s') | |
452 console = logging.StreamHandler() | |
453 console.setFormatter(formatter) | |
454 | |
455 logger = logging.getLogger(name) | |
456 logger.setLevel(logging.DEBUG if verbose else logging.INFO) | |
457 logger.addHandler(console) | |
458 | |
459 if not log_file: | |
460 # delete = false is chosen here because it is always nice to have a log file | |
461 # ready if you need to debug. Not having the "if only I had set a log file" | |
462 # moment after the fact. | |
463 temp = tempfile.NamedTemporaryFile(prefix="ephemeris_", delete=False) | |
464 log_file = temp.name | |
465 file_handler = logging.FileHandler(log_file) | |
466 logger.addHandler(file_handler) | |
467 logger.info(f"Storing log file in: {log_file}") | |
468 return logger | |
469 | |
470 | |
471 def arg_parser(): | |
472 parser = argparse.ArgumentParser(description=DESCRIPTION) | |
473 parser.add_argument('-u', '--galaxy-url', default="http://localhost:8080", help='Galaxy URL') | |
474 parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key') | |
475 parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key') | |
476 parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.') | |
477 parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID') | |
478 parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions') | |
479 parser.add_argument('-i', '--test-index', default=ALL_TESTS, type=int, help='Tool Test Index (starting at 0) - by default all tests will run.') | |
480 parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to') | |
481 parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.") | |
482 skip_group = parser.add_mutually_exclusive_group() | |
483 skip_group.add_argument('--skip-previously-executed', dest="skip", default="no", action="store_const", const="executed", help="When used with --append, skip any test previously executed.") | |
484 skip_group.add_argument('--skip-previously-successful', dest="skip", default="no", action="store_const", const="successful", help="When used with --append, skip any test previously executed successfully.") | |
485 parser.add_argument('-j', '--output-json', default=None, help='output metadata json') | |
486 parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.") | |
487 parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing") | |
488 parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output") | |
489 parser.add_argument('--with-reference-data', dest="with_reference_data", default=False, action="store_true") | |
490 parser.add_argument('--skip-with-reference-data', dest="with_reference_data", action="store_false", help="Skip tests the Galaxy server believes use data tables or loc files.") | |
491 history_per_group = parser.add_mutually_exclusive_group() | |
492 history_per_group.add_argument('--history-per-suite', dest="history_per_test_case", default=False, action="store_false", help="Create new history per test suite (all tests in same history).") | |
493 history_per_group.add_argument('--history-per-test-case', dest="history_per_test_case", action="store_true", help="Create new history per test case.") | |
494 parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.") | |
495 parser.add_argument('--publish-history', default=False, action="store_true", help="Publish test history. Useful for CI testing.") | |
496 parser.add_argument('--parallel-tests', default=1, type=int, help="Parallel tests.") | |
497 parser.add_argument('--retries', default=0, type=int, help="Retry failed tests.") | |
498 parser.add_argument('--page-size', default=0, type=int, help="If positive, use pagination and just run one 'page' to tool tests.") | |
499 parser.add_argument('--page-number', default=0, type=int, help="If page size is used, run this 'page' of tests - starts with 0.") | |
500 parser.add_argument('--download-attempts', default=1, type=int, help="Galaxy may return a transient 500 status code for download if test results are written but not yet accessible.") | |
501 parser.add_argument('--download-sleep', default=1, type=int, help="If download attempts is greater than 1, the amount to sleep between download attempts.") | |
502 parser.add_argument('--test-data', action='append', help='Add local test data path to search for missing test data') | |
503 return parser | |
504 | |
505 | |
506 if __name__ == "__main__": | |
507 main() |