Mercurial > repos > shellac > sam_consensus_v3

comparison env/lib/python3.9/site-packages/schema_salad/validate.py @ 0:4f3585e2f14b draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author shellac
date Mon, 22 Mar 2021 18:12:50 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:4f3585e2f14b
1 import logging
2 import pprint
3 from typing import Any, List, MutableMapping, MutableSequence, Optional, Set
4 from urllib.parse import urlsplit
5
6 from . import avro
7 from .avro.schema import Schema # pylint: disable=no-name-in-module, import-error
8 from .exceptions import (
9 ClassValidationException,
10 SchemaSaladException,
11 ValidationException,
12 )
13 from .sourceline import SourceLine
14
15 _logger = logging.getLogger("salad")
16
17
18 def validate(
19 expected_schema: Schema,
20 datum: Any,
21 identifiers: Optional[List[str]] = None,
22 strict: bool = False,
23 foreign_properties: Optional[Set[str]] = None,
24 ) -> bool:
25 if not identifiers:
26 identifiers = []
27 if not foreign_properties:
28 foreign_properties = set()
29 return validate_ex(
30 expected_schema,
31 datum,
32 identifiers,
33 strict=strict,
34 foreign_properties=foreign_properties,
35 raise_ex=False,
36 )
37
38
39 INT_MIN_VALUE = -(1 << 31)
40 INT_MAX_VALUE = (1 << 31) - 1
41 LONG_MIN_VALUE = -(1 << 63)
42 LONG_MAX_VALUE = (1 << 63) - 1
43
44
45 def friendly(v): # type: (Any) -> Any
46 if isinstance(v, avro.schema.NamedSchema):
47 return v.name
48 if isinstance(v, avro.schema.ArraySchema):
49 return "array of <{}>".format(friendly(v.items))
50 elif isinstance(v, avro.schema.PrimitiveSchema):
51 return v.type
52 elif isinstance(v, avro.schema.UnionSchema):
53 return " or ".join([friendly(s) for s in v.schemas])
54 else:
55 return v
56
57
58 def vpformat(datum): # type: (Any) -> str
59 a = pprint.pformat(datum)
60 if len(a) > 160:
61 a = a[0:160] + "[...]"
62 return a
63
64
65 def validate_ex(
66 expected_schema: Schema,
67 datum, # type: Any
68 identifiers=None, # type: Optional[List[str]]
69 strict=False, # type: bool
70 foreign_properties=None, # type: Optional[Set[str]]
71 raise_ex=True, # type: bool
72 strict_foreign_properties=False, # type: bool
73 logger=_logger, # type: logging.Logger
74 skip_foreign_properties=False, # type: bool
75 ):
76 # type: (...) -> bool
77 """Determine if a python datum is an instance of a schema."""
78
79 if not identifiers:
80 identifiers = []
81
82 if not foreign_properties:
83 foreign_properties = set()
84
85 schema_type = expected_schema.type
86
87 if schema_type == "null":
88 if datum is None:
89 return True
90 if raise_ex:
91 raise ValidationException("the value is not null")
92 return False
93 elif schema_type == "boolean":
94 if isinstance(datum, bool):
95 return True
96 if raise_ex:
97 raise ValidationException("the value is not boolean")
98 return False
99 elif schema_type == "string":
100 if isinstance(datum, str):
101 return True
102 if isinstance(datum, bytes):
103 return True
104 if raise_ex:
105 raise ValidationException("the value is not string")
106 return False
107 elif schema_type == "int":
108 if isinstance(datum, int) and INT_MIN_VALUE <= datum <= INT_MAX_VALUE:
109 return True
110 if raise_ex:
111 raise ValidationException("`{}` is not int".format(vpformat(datum)))
112 return False
113 elif schema_type == "long":
114 if (isinstance(datum, int)) and LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE:
115 return True
116 if raise_ex:
117 raise ValidationException(
118 "the value `{}` is not long".format(vpformat(datum))
119 )
120 return False
121 elif schema_type in ["float", "double"]:
122 if isinstance(datum, int) or isinstance(datum, float):
123 return True
124 if raise_ex:
125 raise ValidationException(
126 "the value `{}` is not float or double".format(vpformat(datum))
127 )
128 return False
129 elif isinstance(expected_schema, avro.schema.EnumSchema):
130 if expected_schema.name == "Any":
131 if datum is not None:
132 return True
133 if raise_ex:
134 raise ValidationException("'Any' type must be non-null")
135 return False
136 if not isinstance(datum, str):
137 if raise_ex:
138 raise ValidationException(
139 "value is a {} but expected a string".format(type(datum).__name__)
140 )
141 return False
142 if expected_schema.name == "Expression":
143 if "$(" in datum or "${" in datum:
144 return True
145 if raise_ex:
146 raise ValidationException(
147 "value `{}` does not contain an expression in the form $() or ${{}}".format(
148 datum
149 )
150 )
151 return False
152 if datum in expected_schema.symbols:
153 return True
154 else:
155 if raise_ex:
156 raise ValidationException(
157 "the value {} is not a valid {}, expected {}{}".format(
158 vpformat(datum),
159 expected_schema.name,
160 "one of " if len(expected_schema.symbols) > 1 else "",
161 "'" + "', '".join(expected_schema.symbols) + "'",
162 )
163 )
164 return False
165 elif isinstance(expected_schema, avro.schema.ArraySchema):
166 if isinstance(datum, MutableSequence):
167 for i, d in enumerate(datum):
168 try:
169 sl = SourceLine(datum, i, ValidationException)
170 if not validate_ex(
171 expected_schema.items,
172 d,
173 identifiers,
174 strict=strict,
175 foreign_properties=foreign_properties,
176 raise_ex=raise_ex,
177 strict_foreign_properties=strict_foreign_properties,
178 logger=logger,
179 skip_foreign_properties=skip_foreign_properties,
180 ):
181 return False
182 except ValidationException as v:
183 if raise_ex:
184 raise ValidationException("item is invalid because", sl, [v])
185 return False
186 return True
187 else:
188 if raise_ex:
189 raise ValidationException(
190 "the value {} is not a list, expected list of {}".format(
191 vpformat(datum), friendly(expected_schema.items)
192 )
193 )
194 return False
195 elif isinstance(expected_schema, avro.schema.UnionSchema):
196 for s in expected_schema.schemas:
197 if validate_ex(
198 s,
199 datum,
200 identifiers,
201 strict=strict,
202 raise_ex=False,
203 strict_foreign_properties=strict_foreign_properties,
204 logger=logger,
205 skip_foreign_properties=skip_foreign_properties,
206 ):
207 return True
208
209 if not raise_ex:
210 return False
211
212 errors = [] # type: List[SchemaSaladException]
213 checked = []
214 for s in expected_schema.schemas:
215 if isinstance(datum, MutableSequence) and not isinstance(
216 s, avro.schema.ArraySchema
217 ):
218 continue
219 elif isinstance(datum, MutableMapping) and not isinstance(
220 s, avro.schema.RecordSchema
221 ):
222 continue
223 elif isinstance(datum, (bool, int, float, str)) and isinstance(
224 s, (avro.schema.ArraySchema, avro.schema.RecordSchema)
225 ):
226 continue
227 elif datum is not None and s.type == "null":
228 continue
229
230 checked.append(s)
231 try:
232 validate_ex(
233 s,
234 datum,
235 identifiers,
236 strict=strict,
237 foreign_properties=foreign_properties,
238 raise_ex=True,
239 strict_foreign_properties=strict_foreign_properties,
240 logger=logger,
241 skip_foreign_properties=skip_foreign_properties,
242 )
243 except ClassValidationException:
244 raise
245 except ValidationException as e:
246 errors.append(e)
247 if bool(errors):
248 raise ValidationException(
249 "",
250 None,
251 [
252 ValidationException(
253 "tried {} but".format(friendly(check)), None, [err]
254 )
255 for (check, err) in zip(checked, errors)
256 ],
257 "-",
258 )
259 else:
260 raise ValidationException(
261 "value is a {}, expected {}".format(
262 type(datum).__name__, friendly(expected_schema)
263 )
264 )
265
266 elif isinstance(expected_schema, avro.schema.RecordSchema):
267 if not isinstance(datum, MutableMapping):
268 if raise_ex:
269 raise ValidationException("is not a dict")
270 else:
271 return False
272
273 classmatch = None
274 for f in expected_schema.fields:
275 if f.name in ("class",):
276 d = datum.get(f.name)
277 if not d:
278 if raise_ex:
279 raise ValidationException(f"Missing '{f.name}' field")
280 else:
281 return False
282 if expected_schema.name != d:
283 if raise_ex:
284 raise ValidationException(
285 "Expected class '{}' but this is '{}'".format(
286 expected_schema.name, d
287 )
288 )
289 else:
290 return False
291 classmatch = d
292 break
293
294 errors = []
295 for f in expected_schema.fields:
296 if f.name in ("class",):
297 continue
298
299 if f.name in datum:
300 fieldval = datum[f.name]
301 else:
302 try:
303 fieldval = f.default
304 except KeyError:
305 fieldval = None
306
307 try:
308 sl = SourceLine(datum, f.name, str)
309 if not validate_ex(
310 f.type,
311 fieldval,
312 identifiers,
313 strict=strict,
314 foreign_properties=foreign_properties,
315 raise_ex=raise_ex,
316 strict_foreign_properties=strict_foreign_properties,
317 logger=logger,
318 skip_foreign_properties=skip_foreign_properties,
319 ):
320 return False
321 except ValidationException as v:
322 if f.name not in datum:
323 errors.append(
324 ValidationException(f"missing required field `{f.name}`")
325 )
326 else:
327 errors.append(
328 ValidationException(
329 f"the `{f.name}` field is not valid because",
330 sl,
331 [v],
332 )
333 )
334
335 for d in datum:
336 found = False
337 for f in expected_schema.fields:
338 if d == f.name:
339 found = True
340 if not found:
341 sl = SourceLine(datum, d, str)
342 if d is None:
343 err = ValidationException("mapping with implicit null key", sl)
344 if strict:
345 errors.append(err)
346 else:
347 logger.warning(err.as_warning())
348 continue
349 if (
350 d not in identifiers
351 and d not in foreign_properties
352 and d[0] not in ("@", "$")
353 ):
354 if (
355 (d not in identifiers and strict)
356 and (
357 d not in foreign_properties
358 and strict_foreign_properties
359 and not skip_foreign_properties
360 )
361 and not raise_ex
362 ):
363 return False
364 split = urlsplit(d)
365 if split.scheme:
366 if not skip_foreign_properties:
367 err = ValidationException(
368 "unrecognized extension field `{}`{}.{}".format(
369 d,
370 " and strict_foreign_properties checking is enabled"
371 if strict_foreign_properties
372 else "",
373 "\nForeign properties from $schemas:\n {}".format(
374 "\n ".join(sorted(foreign_properties))
375 )
376 if len(foreign_properties) > 0
377 else "",
378 ),
379 sl,
380 )
381 if strict_foreign_properties:
382 errors.append(err)
383 elif len(foreign_properties) > 0:
384 logger.warning(err.as_warning())
385 else:
386 err = ValidationException(
387 "invalid field `{}`, expected one of: {}".format(
388 d,
389 ", ".join(
390 f"'{fn.name}'" for fn in expected_schema.fields
391 ),
392 ),
393 sl,
394 )
395 if strict:
396 errors.append(err)
397 else:
398 logger.warning(err.as_warning())
399
400 if bool(errors):
401 if raise_ex:
402 if classmatch:
403 raise ClassValidationException("", None, errors, "*")
404 else:
405 raise ValidationException("", None, errors, "*")
406 else:
407 return False
408 else:
409 return True
410 if raise_ex:
411 raise ValidationException(f"Unrecognized schema_type {schema_type}")
412 else:
413 return False