annotate toolfactory/galaxy-tool-test @ 103:2ca51677d6d1 draft

Uploaded
author fubar
date Fri, 27 Nov 2020 22:38:43 +0000
parents d4d88d393285
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
99
d4d88d393285 Uploaded
fubar
parents:
diff changeset
1 #!/usr/bin/env python
d4d88d393285 Uploaded
fubar
parents:
diff changeset
2
d4d88d393285 Uploaded
fubar
parents:
diff changeset
3 import argparse
d4d88d393285 Uploaded
fubar
parents:
diff changeset
4 import datetime as dt
d4d88d393285 Uploaded
fubar
parents:
diff changeset
5 import json
d4d88d393285 Uploaded
fubar
parents:
diff changeset
6 import logging
d4d88d393285 Uploaded
fubar
parents:
diff changeset
7 import os
d4d88d393285 Uploaded
fubar
parents:
diff changeset
8 import sys
d4d88d393285 Uploaded
fubar
parents:
diff changeset
9 import tempfile
d4d88d393285 Uploaded
fubar
parents:
diff changeset
10 from collections import namedtuple
d4d88d393285 Uploaded
fubar
parents:
diff changeset
11 from concurrent.futures import thread, ThreadPoolExecutor
d4d88d393285 Uploaded
fubar
parents:
diff changeset
12
d4d88d393285 Uploaded
fubar
parents:
diff changeset
13 import yaml
d4d88d393285 Uploaded
fubar
parents:
diff changeset
14
d4d88d393285 Uploaded
fubar
parents:
diff changeset
15 from galaxy.tool_util.verify.interactor import (
d4d88d393285 Uploaded
fubar
parents:
diff changeset
16 DictClientTestConfig,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
17 GalaxyInteractorApi,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
18 verify_tool,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
19 )
d4d88d393285 Uploaded
fubar
parents:
diff changeset
20
d4d88d393285 Uploaded
fubar
parents:
diff changeset
21 DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
d4d88d393285 Uploaded
fubar
parents:
diff changeset
22 DEFAULT_SUITE_NAME = "Galaxy Tool Tests"
d4d88d393285 Uploaded
fubar
parents:
diff changeset
23 ALL_TESTS = -1
d4d88d393285 Uploaded
fubar
parents:
diff changeset
24 ALL_TOOLS = "*"
d4d88d393285 Uploaded
fubar
parents:
diff changeset
25 ALL_VERSION = "*"
d4d88d393285 Uploaded
fubar
parents:
diff changeset
26 LATEST_VERSION = None
d4d88d393285 Uploaded
fubar
parents:
diff changeset
27
d4d88d393285 Uploaded
fubar
parents:
diff changeset
28
d4d88d393285 Uploaded
fubar
parents:
diff changeset
29 TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"])
d4d88d393285 Uploaded
fubar
parents:
diff changeset
30 TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"])
d4d88d393285 Uploaded
fubar
parents:
diff changeset
31
d4d88d393285 Uploaded
fubar
parents:
diff changeset
32
d4d88d393285 Uploaded
fubar
parents:
diff changeset
33 class Results:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
34
d4d88d393285 Uploaded
fubar
parents:
diff changeset
35 def __init__(self, default_suitename, test_json, append=False):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
36 self.test_json = test_json or "-"
d4d88d393285 Uploaded
fubar
parents:
diff changeset
37 test_results = []
d4d88d393285 Uploaded
fubar
parents:
diff changeset
38 test_exceptions = []
d4d88d393285 Uploaded
fubar
parents:
diff changeset
39 suitename = default_suitename
d4d88d393285 Uploaded
fubar
parents:
diff changeset
40 if append:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
41 assert test_json != "-"
d4d88d393285 Uploaded
fubar
parents:
diff changeset
42 with open(test_json) as f:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
43 previous_results = json.load(f)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
44 test_results = previous_results["tests"]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
45 if "suitename" in previous_results:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
46 suitename = previous_results["suitename"]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
47 self.test_results = test_results
d4d88d393285 Uploaded
fubar
parents:
diff changeset
48 self.test_exceptions = test_exceptions
d4d88d393285 Uploaded
fubar
parents:
diff changeset
49 self.suitename = suitename
d4d88d393285 Uploaded
fubar
parents:
diff changeset
50
d4d88d393285 Uploaded
fubar
parents:
diff changeset
51 def register_result(self, result):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
52 self.test_results.append(result)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
53
d4d88d393285 Uploaded
fubar
parents:
diff changeset
54 def register_exception(self, test_exception):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
55 self.test_exceptions.append(test_exception)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
56
d4d88d393285 Uploaded
fubar
parents:
diff changeset
57 def already_successful(self, test_reference):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
58 test_id = _test_id_for_reference(test_reference)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
59 for test_result in self.test_results:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
60 if test_result.get('id') != test_id:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
61 continue
d4d88d393285 Uploaded
fubar
parents:
diff changeset
62
d4d88d393285 Uploaded
fubar
parents:
diff changeset
63 has_data = test_result.get('has_data', False)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
64 if has_data:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
65 test_data = test_result.get("data", {})
d4d88d393285 Uploaded
fubar
parents:
diff changeset
66 if 'status' in test_data and test_data['status'] == 'success':
d4d88d393285 Uploaded
fubar
parents:
diff changeset
67 return True
d4d88d393285 Uploaded
fubar
parents:
diff changeset
68
d4d88d393285 Uploaded
fubar
parents:
diff changeset
69 return False
d4d88d393285 Uploaded
fubar
parents:
diff changeset
70
d4d88d393285 Uploaded
fubar
parents:
diff changeset
71 def write(self):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
72 tests = sorted(self.test_results, key=lambda el: el['id'])
d4d88d393285 Uploaded
fubar
parents:
diff changeset
73 n_passed, n_failures, n_skips = 0, 0, 0
d4d88d393285 Uploaded
fubar
parents:
diff changeset
74 n_errors = len([e for e in self.test_exceptions if not e.was_recorded])
d4d88d393285 Uploaded
fubar
parents:
diff changeset
75 for test in tests:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
76 has_data = test.get('has_data', False)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
77 if has_data:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
78 test_data = test.get("data", {})
d4d88d393285 Uploaded
fubar
parents:
diff changeset
79 if 'status' not in test_data:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
80 raise Exception(f"Test result data {test_data} doesn't contain a status key.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
81 status = test_data['status']
d4d88d393285 Uploaded
fubar
parents:
diff changeset
82 if status == "success":
d4d88d393285 Uploaded
fubar
parents:
diff changeset
83 n_passed += 1
d4d88d393285 Uploaded
fubar
parents:
diff changeset
84 elif status == "error":
d4d88d393285 Uploaded
fubar
parents:
diff changeset
85 n_errors += 1
d4d88d393285 Uploaded
fubar
parents:
diff changeset
86 elif status == "skip":
d4d88d393285 Uploaded
fubar
parents:
diff changeset
87 n_skips += 1
d4d88d393285 Uploaded
fubar
parents:
diff changeset
88 elif status == "failure":
d4d88d393285 Uploaded
fubar
parents:
diff changeset
89 n_failures += 1
d4d88d393285 Uploaded
fubar
parents:
diff changeset
90 report_obj = {
d4d88d393285 Uploaded
fubar
parents:
diff changeset
91 'version': '0.1',
d4d88d393285 Uploaded
fubar
parents:
diff changeset
92 'suitename': self.suitename,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
93 'results': {
d4d88d393285 Uploaded
fubar
parents:
diff changeset
94 'total': n_passed + n_failures + n_skips + n_errors,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
95 'errors': n_errors,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
96 'failures': n_failures,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
97 'skips': n_skips,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
98 },
d4d88d393285 Uploaded
fubar
parents:
diff changeset
99 'tests': tests,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
100 }
d4d88d393285 Uploaded
fubar
parents:
diff changeset
101 if self.test_json == "-":
d4d88d393285 Uploaded
fubar
parents:
diff changeset
102 print(json.dumps(report_obj))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
103 else:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
104 with open(self.test_json, "w") as f:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
105 json.dump(report_obj, f)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
106
d4d88d393285 Uploaded
fubar
parents:
diff changeset
107 def info_message(self):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
108 messages = []
d4d88d393285 Uploaded
fubar
parents:
diff changeset
109 passed_tests = self._tests_with_status('success')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
110 messages.append("Passed tool tests ({}): {}".format(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
111 len(passed_tests),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
112 [t["id"] for t in passed_tests]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
113 ))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
114 failed_tests = self._tests_with_status('failure')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
115 messages.append("Failed tool tests ({}): {}".format(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
116 len(failed_tests),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
117 [t["id"] for t in failed_tests]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
118 ))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
119 skiped_tests = self._tests_with_status('skip')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
120 messages.append("Skipped tool tests ({}): {}".format(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
121 len(skiped_tests),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
122 [t["id"] for t in skiped_tests]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
123 ))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
124 errored_tests = self._tests_with_status('error')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
125 messages.append("Errored tool tests ({}): {}".format(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
126 len(errored_tests),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
127 [t["id"] for t in errored_tests]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
128 ))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
129 return "\n".join(messages)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
130
d4d88d393285 Uploaded
fubar
parents:
diff changeset
131 @property
d4d88d393285 Uploaded
fubar
parents:
diff changeset
132 def success_count(self):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
133 self._tests_with_status('success')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
134
d4d88d393285 Uploaded
fubar
parents:
diff changeset
135 @property
d4d88d393285 Uploaded
fubar
parents:
diff changeset
136 def skip_count(self):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
137 self._tests_with_status('skip')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
138
d4d88d393285 Uploaded
fubar
parents:
diff changeset
139 @property
d4d88d393285 Uploaded
fubar
parents:
diff changeset
140 def error_count(self):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
141 return self._tests_with_status('error') + len(self.test_exceptions)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
142
d4d88d393285 Uploaded
fubar
parents:
diff changeset
143 @property
d4d88d393285 Uploaded
fubar
parents:
diff changeset
144 def failure_count(self):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
145 return self._tests_with_status('failure')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
146
d4d88d393285 Uploaded
fubar
parents:
diff changeset
147 def _tests_with_status(self, status):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
148 return [t for t in self.test_results if t.get("data", {}).get("status") == status]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
149
d4d88d393285 Uploaded
fubar
parents:
diff changeset
150
d4d88d393285 Uploaded
fubar
parents:
diff changeset
151 def test_tools(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
152 galaxy_interactor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
153 test_references,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
154 results,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
155 log=None,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
156 parallel_tests=1,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
157 history_per_test_case=False,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
158 no_history_cleanup=False,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
159 retries=0,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
160 verify_kwds=None,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
161 ):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
162 """Run through tool tests and write report.
d4d88d393285 Uploaded
fubar
parents:
diff changeset
163
d4d88d393285 Uploaded
fubar
parents:
diff changeset
164 Refactor this into Galaxy in 21.01.
d4d88d393285 Uploaded
fubar
parents:
diff changeset
165 """
d4d88d393285 Uploaded
fubar
parents:
diff changeset
166 verify_kwds = (verify_kwds or {}).copy()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
167 tool_test_start = dt.datetime.now()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
168 history_created = False
d4d88d393285 Uploaded
fubar
parents:
diff changeset
169 if history_per_test_case:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
170 test_history = None
d4d88d393285 Uploaded
fubar
parents:
diff changeset
171 else:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
172 history_created = True
d4d88d393285 Uploaded
fubar
parents:
diff changeset
173 test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
174 verify_kwds.update({
d4d88d393285 Uploaded
fubar
parents:
diff changeset
175 "no_history_cleanup": no_history_cleanup,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
176 "test_history": test_history,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
177 })
d4d88d393285 Uploaded
fubar
parents:
diff changeset
178 with ThreadPoolExecutor(max_workers=parallel_tests) as executor:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
179 try:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
180 for test_reference in test_references:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
181 _test_tool(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
182 executor=executor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
183 test_reference=test_reference,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
184 results=results,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
185 galaxy_interactor=galaxy_interactor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
186 log=log,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
187 retries=retries,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
188 verify_kwds=verify_kwds,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
189 )
d4d88d393285 Uploaded
fubar
parents:
diff changeset
190 finally:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
191 # Always write report, even if test was cancelled.
d4d88d393285 Uploaded
fubar
parents:
diff changeset
192 try:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
193 executor.shutdown(wait=True)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
194 except KeyboardInterrupt:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
195 executor._threads.clear()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
196 thread._threads_queues.clear()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
197 results.write()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
198 if log:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
199 log.info("Report written to '%s'", os.path.abspath(results.test_json))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
200 log.info(results.info_message())
d4d88d393285 Uploaded
fubar
parents:
diff changeset
201 log.info("Total tool test time: {}".format(dt.datetime.now() - tool_test_start))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
202 if history_created and not no_history_cleanup:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
203 galaxy_interactor.delete_history(test_history)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
204
d4d88d393285 Uploaded
fubar
parents:
diff changeset
205
d4d88d393285 Uploaded
fubar
parents:
diff changeset
206 def _test_id_for_reference(test_reference):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
207 tool_id = test_reference.tool_id
d4d88d393285 Uploaded
fubar
parents:
diff changeset
208 tool_version = test_reference.tool_version
d4d88d393285 Uploaded
fubar
parents:
diff changeset
209 test_index = test_reference.test_index
d4d88d393285 Uploaded
fubar
parents:
diff changeset
210
d4d88d393285 Uploaded
fubar
parents:
diff changeset
211 if tool_version and tool_id.endswith("/" + tool_version):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
212 tool_id = tool_id[:-len("/" + tool_version)]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
213
d4d88d393285 Uploaded
fubar
parents:
diff changeset
214 label_base = tool_id
d4d88d393285 Uploaded
fubar
parents:
diff changeset
215 if tool_version:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
216 label_base += "/" + str(tool_version)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
217
d4d88d393285 Uploaded
fubar
parents:
diff changeset
218 test_id = label_base + "-" + str(test_index)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
219 return test_id
d4d88d393285 Uploaded
fubar
parents:
diff changeset
220
d4d88d393285 Uploaded
fubar
parents:
diff changeset
221
d4d88d393285 Uploaded
fubar
parents:
diff changeset
222 def _test_tool(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
223 executor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
224 test_reference,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
225 results,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
226 galaxy_interactor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
227 log,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
228 retries,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
229 verify_kwds,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
230 ):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
231 tool_id = test_reference.tool_id
d4d88d393285 Uploaded
fubar
parents:
diff changeset
232 tool_version = test_reference.tool_version
d4d88d393285 Uploaded
fubar
parents:
diff changeset
233 test_index = test_reference.test_index
d4d88d393285 Uploaded
fubar
parents:
diff changeset
234 # If given a tool_id with a version suffix, strip it off so we can treat tool_version
d4d88d393285 Uploaded
fubar
parents:
diff changeset
235 # correctly at least in client_test_config.
d4d88d393285 Uploaded
fubar
parents:
diff changeset
236 if tool_version and tool_id.endswith("/" + tool_version):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
237 tool_id = tool_id[:-len("/" + tool_version)]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
238
d4d88d393285 Uploaded
fubar
parents:
diff changeset
239 test_id = _test_id_for_reference(test_reference)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
240
d4d88d393285 Uploaded
fubar
parents:
diff changeset
241 def run_test():
d4d88d393285 Uploaded
fubar
parents:
diff changeset
242 run_retries = retries
d4d88d393285 Uploaded
fubar
parents:
diff changeset
243 job_data = None
d4d88d393285 Uploaded
fubar
parents:
diff changeset
244 job_exception = None
d4d88d393285 Uploaded
fubar
parents:
diff changeset
245
d4d88d393285 Uploaded
fubar
parents:
diff changeset
246 def register(job_data_):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
247 nonlocal job_data
d4d88d393285 Uploaded
fubar
parents:
diff changeset
248 job_data = job_data_
d4d88d393285 Uploaded
fubar
parents:
diff changeset
249
d4d88d393285 Uploaded
fubar
parents:
diff changeset
250 try:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
251 while run_retries >= 0:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
252 job_exception = None
d4d88d393285 Uploaded
fubar
parents:
diff changeset
253 try:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
254 if log:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
255 log.info("Executing test '%s'", test_id)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
256 verify_tool(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
257 tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
258 register_job_data=register, **verify_kwds
d4d88d393285 Uploaded
fubar
parents:
diff changeset
259 )
d4d88d393285 Uploaded
fubar
parents:
diff changeset
260 if log:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
261 log.info("Test '%s' passed", test_id)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
262 break
d4d88d393285 Uploaded
fubar
parents:
diff changeset
263 except Exception as e:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
264 if log:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
265 log.warning("Test '%s' failed", test_id, exc_info=True)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
266
d4d88d393285 Uploaded
fubar
parents:
diff changeset
267 job_exception = e
d4d88d393285 Uploaded
fubar
parents:
diff changeset
268 run_retries -= 1
d4d88d393285 Uploaded
fubar
parents:
diff changeset
269 finally:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
270 if job_data is not None:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
271 results.register_result({
d4d88d393285 Uploaded
fubar
parents:
diff changeset
272 "id": test_id,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
273 "has_data": True,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
274 "data": job_data,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
275 })
d4d88d393285 Uploaded
fubar
parents:
diff changeset
276 if job_exception is not None:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
277 was_recorded = job_data is not None
d4d88d393285 Uploaded
fubar
parents:
diff changeset
278 test_exception = TestException(tool_id, job_exception, was_recorded)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
279 results.register_exception(test_exception)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
280
d4d88d393285 Uploaded
fubar
parents:
diff changeset
281 executor.submit(run_test)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
282
d4d88d393285 Uploaded
fubar
parents:
diff changeset
283
d4d88d393285 Uploaded
fubar
parents:
diff changeset
284 def build_case_references(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
285 galaxy_interactor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
286 tool_id=ALL_TOOLS,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
287 tool_version=LATEST_VERSION,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
288 test_index=ALL_TESTS,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
289 page_size=0,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
290 page_number=0,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
291 check_against=None,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
292 log=None,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
293 ):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
294 test_references = []
d4d88d393285 Uploaded
fubar
parents:
diff changeset
295 if tool_id == ALL_TOOLS:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
296 tests_summary = galaxy_interactor.get_tests_summary()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
297 for tool_id, tool_versions_dict in tests_summary.items():
d4d88d393285 Uploaded
fubar
parents:
diff changeset
298 for tool_version, summary in tool_versions_dict.items():
d4d88d393285 Uploaded
fubar
parents:
diff changeset
299 for test_index in range(summary["count"]):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
300 test_reference = TestReference(tool_id, tool_version, test_index)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
301 test_references.append(test_reference)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
302 else:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
303 assert tool_id
d4d88d393285 Uploaded
fubar
parents:
diff changeset
304 tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {}
d4d88d393285 Uploaded
fubar
parents:
diff changeset
305 for i, tool_test_dict in enumerate(tool_test_dicts):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
306 this_tool_version = tool_test_dict.get("tool_version", tool_version)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
307 this_test_index = i
d4d88d393285 Uploaded
fubar
parents:
diff changeset
308 if test_index == ALL_TESTS or i == test_index:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
309 test_reference = TestReference(tool_id, this_tool_version, this_test_index)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
310 test_references.append(test_reference)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
311
d4d88d393285 Uploaded
fubar
parents:
diff changeset
312 if check_against:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
313 filtered_test_references = []
d4d88d393285 Uploaded
fubar
parents:
diff changeset
314 for test_reference in test_references:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
315 if check_against.already_successful(test_reference):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
316 if log is not None:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
317 log.debug(f"Found successful test for {test_reference}, skipping")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
318 continue
d4d88d393285 Uploaded
fubar
parents:
diff changeset
319 filtered_test_references.append(test_reference)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
320 log.info(f"Skipping {len(test_references)-len(filtered_test_references)} out of {len(test_references)} tests.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
321 test_references = filtered_test_references
d4d88d393285 Uploaded
fubar
parents:
diff changeset
322
d4d88d393285 Uploaded
fubar
parents:
diff changeset
323 if page_size > 0:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
324 slice_start = page_size * page_number
d4d88d393285 Uploaded
fubar
parents:
diff changeset
325 slice_end = page_size * (page_number + 1)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
326 test_references = test_references[slice_start:slice_end]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
327
d4d88d393285 Uploaded
fubar
parents:
diff changeset
328 return test_references
d4d88d393285 Uploaded
fubar
parents:
diff changeset
329
d4d88d393285 Uploaded
fubar
parents:
diff changeset
330
d4d88d393285 Uploaded
fubar
parents:
diff changeset
331 def main(argv=None):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
332 if argv is None:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
333 argv = sys.argv[1:]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
334
d4d88d393285 Uploaded
fubar
parents:
diff changeset
335 args = _arg_parser().parse_args(argv)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
336 log = setup_global_logger(__name__, verbose=args.verbose)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
337 client_test_config_path = args.client_test_config
d4d88d393285 Uploaded
fubar
parents:
diff changeset
338 if client_test_config_path is not None:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
339 log.debug(f"Reading client config path {client_test_config_path}")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
340 with open(client_test_config_path) as f:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
341 client_test_config = yaml.full_load(f)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
342 else:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
343 client_test_config = {}
d4d88d393285 Uploaded
fubar
parents:
diff changeset
344
d4d88d393285 Uploaded
fubar
parents:
diff changeset
345 def get_option(key):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
346 arg_val = getattr(args, key, None)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
347 if arg_val is None and key in client_test_config:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
348 val = client_test_config.get(key)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
349 else:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
350 val = arg_val
d4d88d393285 Uploaded
fubar
parents:
diff changeset
351 return val
d4d88d393285 Uploaded
fubar
parents:
diff changeset
352
d4d88d393285 Uploaded
fubar
parents:
diff changeset
353 output_json_path = get_option("output_json")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
354 galaxy_interactor_kwds = {
d4d88d393285 Uploaded
fubar
parents:
diff changeset
355 "galaxy_url": get_option("galaxy_url"),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
356 "master_api_key": get_option("admin_key"),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
357 "api_key": get_option("key"),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
358 "keep_outputs_dir": args.output,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
359 "download_attempts": get_option("download_attempts"),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
360 "download_sleep": get_option("download_sleep"),
d4d88d393285 Uploaded
fubar
parents:
diff changeset
361 }
d4d88d393285 Uploaded
fubar
parents:
diff changeset
362 tool_id = args.tool_id
d4d88d393285 Uploaded
fubar
parents:
diff changeset
363 tool_version = args.tool_version
d4d88d393285 Uploaded
fubar
parents:
diff changeset
364 tools_client_test_config = DictClientTestConfig(client_test_config.get("tools"))
d4d88d393285 Uploaded
fubar
parents:
diff changeset
365 verbose = args.verbose
d4d88d393285 Uploaded
fubar
parents:
diff changeset
366
d4d88d393285 Uploaded
fubar
parents:
diff changeset
367 galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
368 results = Results(args.suite_name, output_json_path, append=args.append)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
369 check_against = None if not args.skip_successful else results
d4d88d393285 Uploaded
fubar
parents:
diff changeset
370 test_references = build_case_references(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
371 galaxy_interactor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
372 tool_id=tool_id,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
373 tool_version=tool_version,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
374 test_index=args.test_index,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
375 page_size=args.page_size,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
376 page_number=args.page_number,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
377 check_against=check_against,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
378 log=log,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
379 )
d4d88d393285 Uploaded
fubar
parents:
diff changeset
380 log.debug(f"Built {len(test_references)} test references to executed.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
381 verify_kwds = dict(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
382 client_test_config=tools_client_test_config,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
383 force_path_paste=args.force_path_paste,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
384 skip_with_reference_data=not args.with_reference_data,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
385 quiet=not verbose,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
386 )
d4d88d393285 Uploaded
fubar
parents:
diff changeset
387 test_tools(
d4d88d393285 Uploaded
fubar
parents:
diff changeset
388 galaxy_interactor,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
389 test_references,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
390 results,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
391 log=log,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
392 parallel_tests=args.parallel_tests,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
393 history_per_test_case=args.history_per_test_case,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
394 no_history_cleanup=args.no_history_cleanup,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
395 verify_kwds=verify_kwds,
d4d88d393285 Uploaded
fubar
parents:
diff changeset
396 )
d4d88d393285 Uploaded
fubar
parents:
diff changeset
397 exceptions = results.test_exceptions
d4d88d393285 Uploaded
fubar
parents:
diff changeset
398 if exceptions:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
399 exception = exceptions[0]
d4d88d393285 Uploaded
fubar
parents:
diff changeset
400 if hasattr(exception, "exception"):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
401 exception = exception.exception
d4d88d393285 Uploaded
fubar
parents:
diff changeset
402 raise exception
d4d88d393285 Uploaded
fubar
parents:
diff changeset
403
d4d88d393285 Uploaded
fubar
parents:
diff changeset
404
d4d88d393285 Uploaded
fubar
parents:
diff changeset
405 def setup_global_logger(name, log_file=None, verbose=False):
d4d88d393285 Uploaded
fubar
parents:
diff changeset
406 formatter = logging.Formatter('%(asctime)s %(levelname)-5s - %(message)s')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
407 console = logging.StreamHandler()
d4d88d393285 Uploaded
fubar
parents:
diff changeset
408 console.setFormatter(formatter)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
409
d4d88d393285 Uploaded
fubar
parents:
diff changeset
410 logger = logging.getLogger(name)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
411 logger.setLevel(logging.DEBUG if verbose else logging.INFO)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
412 logger.addHandler(console)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
413
d4d88d393285 Uploaded
fubar
parents:
diff changeset
414 if not log_file:
d4d88d393285 Uploaded
fubar
parents:
diff changeset
415 # delete = false is chosen here because it is always nice to have a log file
d4d88d393285 Uploaded
fubar
parents:
diff changeset
416 # ready if you need to debug. Not having the "if only I had set a log file"
d4d88d393285 Uploaded
fubar
parents:
diff changeset
417 # moment after the fact.
d4d88d393285 Uploaded
fubar
parents:
diff changeset
418 temp = tempfile.NamedTemporaryFile(prefix="ephemeris_", delete=False)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
419 log_file = temp.name
d4d88d393285 Uploaded
fubar
parents:
diff changeset
420 file_handler = logging.FileHandler(log_file)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
421 logger.addHandler(file_handler)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
422 logger.info(f"Storing log file in: {log_file}")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
423 return logger
d4d88d393285 Uploaded
fubar
parents:
diff changeset
424
d4d88d393285 Uploaded
fubar
parents:
diff changeset
425
d4d88d393285 Uploaded
fubar
parents:
diff changeset
426 def _arg_parser():
d4d88d393285 Uploaded
fubar
parents:
diff changeset
427 parser = argparse.ArgumentParser(description=DESCRIPTION)
d4d88d393285 Uploaded
fubar
parents:
diff changeset
428 parser.add_argument('-u', '--galaxy-url', default="http://localhost:8080", help='Galaxy URL')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
429 parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
430 parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
431 parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
432 parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
433 parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
434 parser.add_argument('-i', '--test-index', default=ALL_TESTS, type=int, help='Tool Test Index (starting at 0) - by default all tests will run.')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
435 parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
436 parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
437 parser.add_argument('--skip-successful', default=False, action="store_true", help="When used with --append, skip previously run successful tests.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
438 parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
d4d88d393285 Uploaded
fubar
parents:
diff changeset
439 parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
440 parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
441 parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
442 parser.add_argument('--with-reference-data', dest="with_reference_data", default=False, action="store_true")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
443 parser.add_argument('--skip-with-reference-data', dest="with_reference_data", action="store_false", help="Skip tests the Galaxy server believes use data tables or loc files.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
444 parser.add_argument('--history-per-suite', dest="history_per_test_case", default=False, action="store_false", help="Create new history per test suite (all tests in same history).")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
445 parser.add_argument('--history-per-test-case', dest="history_per_test_case", action="store_true", help="Create new history per test case.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
446 parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
447 parser.add_argument('--parallel-tests', default=1, type=int, help="Parallel tests.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
448 parser.add_argument('--retries', default=0, type=int, help="Retry failed tests.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
449 parser.add_argument('--page-size', default=0, type=int, help="If positive, use pagination and just run one 'page' to tool tests.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
450 parser.add_argument('--page-number', default=0, type=int, help="If page size is used, run this 'page' of tests - starts with 0.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
451 parser.add_argument('--download-attempts', default=1, type=int, help="Galaxy may return a transient 500 status code for download if test results are written but not yet accessible.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
452 parser.add_argument('--download-sleep', default=1, type=int, help="If download attempts is greater than 1, the amount to sleep between download attempts.")
d4d88d393285 Uploaded
fubar
parents:
diff changeset
453 return parser
d4d88d393285 Uploaded
fubar
parents:
diff changeset
454
d4d88d393285 Uploaded
fubar
parents:
diff changeset
455
d4d88d393285 Uploaded
fubar
parents:
diff changeset
456 if __name__ == "__main__":
d4d88d393285 Uploaded
fubar
parents:
diff changeset
457 main()