Mercurial > repos > shellac > sam_consensus_v3
comparison env/lib/python3.9/site-packages/galaxy/tool_util/output_checker.py @ 0:4f3585e2f14b draft default tip
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author | shellac |
---|---|
date | Mon, 22 Mar 2021 18:12:50 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:4f3585e2f14b |
---|---|
1 import re | |
2 from logging import getLogger | |
3 | |
4 from galaxy.tool_util.parser.stdio import StdioErrorLevel | |
5 from galaxy.util import unicodify | |
6 from galaxy.util.bunch import Bunch | |
7 | |
8 log = getLogger(__name__) | |
9 | |
10 DETECTED_JOB_STATE = Bunch( | |
11 OK='ok', | |
12 OUT_OF_MEMORY_ERROR='oom_error', | |
13 GENERIC_ERROR='generic_error', | |
14 ) | |
15 | |
16 ERROR_PEEK_SIZE = 2000 | |
17 | |
18 | |
19 def check_output_regex(job_id_tag, regex, stream, stream_name, job_messages, max_error_level): | |
20 """ | |
21 check a single regex against a stream | |
22 | |
23 regex the regex to check | |
24 stream the stream to search in | |
25 job_messages a list where the descriptions of the detected regexes can be appended | |
26 max_error_level the maximum error level that has been detected so far | |
27 returns the max of the error_level of the regex and the given max_error_level | |
28 """ | |
29 regex_match = re.search(regex.match, stream, re.IGNORECASE) | |
30 if regex_match: | |
31 reason = __regex_err_msg(regex_match, stream_name, regex) | |
32 job_messages.append(reason) | |
33 return max(max_error_level, regex.error_level) | |
34 return max_error_level | |
35 | |
36 | |
37 def check_output(stdio_regexes, stdio_exit_codes, stdout, stderr, tool_exit_code, job_id_tag): | |
38 """ | |
39 Check the output of a tool - given the stdout, stderr, and the tool's | |
40 exit code, return DETECTED_JOB_STATE.OK if the tool exited succesfully or | |
41 error type otherwise. No exceptions should be thrown. If this code encounters | |
42 an exception, it returns OK so that the workflow can continue; | |
43 otherwise, a bug in this code could halt workflow progress. | |
44 | |
45 Note that, if the tool did not define any exit code handling or | |
46 any stdio/stderr handling, then it reverts back to previous behavior: | |
47 if stderr contains anything, then False is returned. | |
48 """ | |
49 # By default, the tool succeeded. This covers the case where the code | |
50 # has a bug but the tool was ok, and it lets a workflow continue. | |
51 state = DETECTED_JOB_STATE.OK | |
52 | |
53 stdout = unicodify(stdout, strip_null=True) | |
54 stderr = unicodify(stderr, strip_null=True) | |
55 | |
56 # messages (descriptions of the detected exit_code and regexes) | |
57 # to be prepended to the stdout/stderr after all exit code and regex tests | |
58 # are done (otherwise added messages are searched again). | |
59 # messages are added it the order of detection | |
60 | |
61 # If job is failed, track why. | |
62 job_messages = [] | |
63 | |
64 try: | |
65 # Check exit codes and match regular expressions against stdout and | |
66 # stderr if this tool was configured to do so. | |
67 # If there is a regular expression for scanning stdout/stderr, | |
68 # then we assume that the tool writer overwrote the default | |
69 # behavior of just setting an error if there is *anything* on | |
70 # stderr. | |
71 if len(stdio_regexes) > 0 or len(stdio_exit_codes) > 0: | |
72 # Check the exit code ranges in the order in which | |
73 # they were specified. Each exit_code is a StdioExitCode | |
74 # that includes an applicable range. If the exit code was in | |
75 # that range, then apply the error level and add a message. | |
76 # If we've reached a fatal error rule, then stop. | |
77 max_error_level = StdioErrorLevel.NO_ERROR | |
78 if tool_exit_code is not None: | |
79 for stdio_exit_code in stdio_exit_codes: | |
80 if (tool_exit_code >= stdio_exit_code.range_start and | |
81 tool_exit_code <= stdio_exit_code.range_end): | |
82 # Tack on a generic description of the code | |
83 # plus a specific code description. For example, | |
84 # this might prepend "Job 42: Warning (Out of Memory)\n". | |
85 code_desc = stdio_exit_code.desc | |
86 if None is code_desc: | |
87 code_desc = "" | |
88 desc = "%s: Exit code %d (%s)" % ( | |
89 StdioErrorLevel.desc(stdio_exit_code.error_level), | |
90 tool_exit_code, | |
91 code_desc) | |
92 reason = { | |
93 'type': 'exit_code', | |
94 'desc': desc, | |
95 'exit_code': tool_exit_code, | |
96 'code_desc': code_desc, | |
97 'error_level': stdio_exit_code.error_level, | |
98 } | |
99 job_messages.append(reason) | |
100 max_error_level = max(max_error_level, | |
101 stdio_exit_code.error_level) | |
102 if max_error_level >= StdioErrorLevel.MAX: | |
103 break | |
104 | |
105 if max_error_level < StdioErrorLevel.FATAL_OOM: | |
106 # We'll examine every regex. Each regex specifies whether | |
107 # it is to be run on stdout, stderr, or both. (It is | |
108 # possible for neither stdout nor stderr to be scanned, | |
109 # but those regexes won't be used.) We record the highest | |
110 # error level, which are currently "warning" and "fatal". | |
111 # If fatal, then we set the job's state to ERROR. | |
112 # If warning, then we still set the job's state to OK | |
113 # but include a message. We'll do this if we haven't seen | |
114 # a fatal error yet | |
115 for regex in stdio_regexes: | |
116 # If ( this regex should be matched against stdout ) | |
117 # - Run the regex's match pattern against stdout | |
118 # - If it matched, then determine the error level. | |
119 # o If it was fatal, then we're done - break. | |
120 if regex.stderr_match: | |
121 max_error_level = check_output_regex(job_id_tag, regex, stderr, 'stderr', job_messages, max_error_level) | |
122 if max_error_level >= StdioErrorLevel.MAX: | |
123 break | |
124 | |
125 if regex.stdout_match: | |
126 max_error_level = check_output_regex(job_id_tag, regex, stdout, 'stdout', job_messages, max_error_level) | |
127 if max_error_level >= StdioErrorLevel.MAX: | |
128 break | |
129 | |
130 # If we encountered a fatal error, then we'll need to set the | |
131 # job state accordingly. Otherwise the job is ok: | |
132 if max_error_level == StdioErrorLevel.FATAL_OOM: | |
133 state = DETECTED_JOB_STATE.OUT_OF_MEMORY_ERROR | |
134 elif max_error_level >= StdioErrorLevel.FATAL: | |
135 reason = '' | |
136 if job_messages: | |
137 reason = f" Reasons are {job_messages}" | |
138 log.info(f"Job error detected, failing job.{reason}") | |
139 state = DETECTED_JOB_STATE.GENERIC_ERROR | |
140 | |
141 # When there are no regular expressions and no exit codes to check, | |
142 # default to the previous behavior: when there's anything on stderr | |
143 # the job has an error, and the job is ok otherwise. | |
144 else: | |
145 # TODO: Add in the tool and job id: | |
146 # log.debug( "Tool did not define exit code or stdio handling; " | |
147 # + "checking stderr for success" ) | |
148 if stderr: | |
149 state = DETECTED_JOB_STATE.GENERIC_ERROR | |
150 peek = stderr[0:ERROR_PEEK_SIZE] if stderr else "" | |
151 log.info(f"Job failed because of contents in the standard error stream: [{peek}]") | |
152 except Exception: | |
153 log.exception("Job state check encountered unexpected exception; assuming execution successful") | |
154 | |
155 return state, stdout, stderr, job_messages | |
156 | |
157 | |
158 def __regex_err_msg(match, stream, regex): | |
159 """ | |
160 Return a message about the match on tool output using the given | |
161 ToolStdioRegex regex object. The regex_match is a MatchObject | |
162 that will contain the string matched on. | |
163 """ | |
164 # Get the description for the error level: | |
165 desc = StdioErrorLevel.desc(regex.error_level) + ": " | |
166 mstart = match.start() | |
167 mend = match.end() | |
168 if mend - mstart > 256: | |
169 match_str = match.string[mstart : mstart + 256] + "..." | |
170 else: | |
171 match_str = match.string[mstart: mend] | |
172 | |
173 # If there's a description for the regular expression, then use it. | |
174 # Otherwise, we'll take the first 256 characters of the match. | |
175 if regex.desc is not None: | |
176 desc += regex.desc | |
177 else: | |
178 desc += "Matched on %s" % match_str | |
179 return { | |
180 "type": "regex", | |
181 "stream": stream, | |
182 "desc": desc, | |
183 "code_desc": regex.desc, | |
184 "match": match_str, | |
185 "error_level": regex.error_level, | |
186 } |