pyfields/validate_n_convert.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/python-pyfields>
3 #
4 # License: 3-clause BSD, <https://github.com/smarie/python-pyfields/blob/master/LICENSE>
5 import sys
6 from collections import OrderedDict
7
8 from valid8 import Validator, failure_raiser, ValidationError, ValidationFailure
9 from valid8.base import getfullargspec as v8_getfullargspec, get_callable_name, is_mini_lambda
10 from valid8.common_syntax import FunctionDefinitionError, make_validation_func_callables
11 from valid8.composition import _and_
12 from valid8.entry_points import _add_none_handler
13 from valid8.utils.signature_tools import IsBuiltInError
14 from valid8.validation_lib import instance_of
15
16 try: # python 3.5+
17 # noinspection PyUnresolvedReferences
18 from typing import Callable, Type, Any, TypeVar, Union, Iterable, Tuple, Mapping, Optional, Dict, Literal
19 # from valid8.common_syntax import ValidationFuncs
20 use_type_hints = sys.version_info > (3, 0)
21 except ImportError:
22 use_type_hints = False
23
24
25 if use_type_hints:
26 T = TypeVar('T')
27
28 # ------------- validator type hints -----------
29 # 1. the lowest-level user or 3d party-provided validation functions
30 ValidationFunc = Union[Callable[[Any], Any],
31 Callable[[Any, Any], Any],
32 Callable[[Any, Any, Any], Any]]
33 """A validation function is a callable with signature (val), (obj, val) or (obj, field, val), returning `True`
34 or `None` in case of success"""
35
36 try:
37 # noinspection PyUnresolvedReferences
38 from mini_lambda import y
39 ValidationFuncOrLambda = Union[ValidationFunc, type(y)]
40 except ImportError:
41 ValidationFuncOrLambda = ValidationFunc
42
43 # 2. the syntax to optionally transform them into failure raisers by providing a tuple
44 ValidatorDef = Union[ValidationFuncOrLambda,
45 Tuple[ValidationFuncOrLambda, str],
46 Tuple[ValidationFuncOrLambda, Type[ValidationFailure]],
47 Tuple[ValidationFuncOrLambda, str, Type[ValidationFailure]]
48 ]
49 """A validator is a validation function together with optional error message and error type"""
50
51 # 3. the syntax to describe several validation functions at once
52 VFDefinitionElement = Union[str, Type[ValidationFailure], ValidationFuncOrLambda]
53 """This type represents one of the elements that can define a checker: help msg, failure type, callable"""
54
55 OneOrSeveralVFDefinitions = Union[ValidatorDef,
56 Iterable[ValidatorDef],
57 Mapping[VFDefinitionElement, Union[VFDefinitionElement,
58 Tuple[VFDefinitionElement, ...]]]]
59 """Several validators can be provided as a singleton, iterable, or dict-like. In that case the value can be a
60 single variable or a tuple, and it will be combined with the key to form the validator. So you can use any of
61 the elements defining a validators as the key."""
62
63 # shortcut name used everywhere. Less explicit
64 Validators = OneOrSeveralVFDefinitions
65
66
67 class FieldValidator(Validator):
68 """
69 Represents a `Validator` responsible to validate a `field`
70 """
71 __slots__ = '__weakref__', 'validated_field', 'base_validation_funcs'
72
73 def __init__(self,
- F821 Undefined name 'DescriptorField'
75 validators, # type: Validators
76 **kwargs
77 ):
78 """
79
80 :param validated_field: the field being validated.
81 :param validators: the base validation function or list of base validation functions to use. A callable, a
82 tuple(callable, help_msg_str), a tuple(callable, failure_type), tuple(callable, help_msg_str, failure_type)
83 or a list of several such elements. A dict can also be used.
84 Tuples indicate an implicit `failure_raiser`.
85 [mini_lambda](https://smarie.github.io/python-mini-lambda/) expressions can be used instead
86 of callables, they will be transformed to functions automatically.
87 :param error_type: a subclass of ValidationError to raise in case of validation failure. By default a
88 ValidationError will be raised with the provided help_msg
89 :param help_msg: an optional help message to be used in the raised error in case of validation failure.
90 :param none_policy: describes how None values should be handled. See `NonePolicy` for the various possibilities.
91 Default is `NonePolicy.VALIDATE`, meaning that None values will be treated exactly like other values and
92 follow the same validation process.
93 :param kw_context_args: optional contextual information to store in the exception, and that may be also used
94 to format the help message
95 """
96 # store this additional info about the function been validated
97 self.validated_field = validated_field
98
99 try: # dict ?
100 validators.keys()
101 except (AttributeError, FunctionDefinitionError): # FunctionDefinitionError when mini_lambda
102 if isinstance(validators, tuple):
103 # single tuple
104 validators = (validators,)
105 else:
106 try: # iterable
107 iter(validators)
108 except (TypeError, FunctionDefinitionError): # FunctionDefinitionError when mini_lambda
109 # single
110 validators = (validators,)
111 else:
112 # dict
113 validators = (validators,)
114
115 # remember validation funcs so that we can add more later
116 self.base_validation_funcs = validators
117
118 super(FieldValidator, self).__init__(*validators, **kwargs)
119
120 def add_validator(self,
121 validation_func # type: ValidatorDef
122 ):
123 """
124 Adds the provided validation function to the existing list of validation functions
125 :param validation_func:
126 :return:
127 """
128 self.base_validation_funcs = self.base_validation_funcs + (validation_func, )
129
130 # do the same than in super.init, once again. TODO optimize ...
131 validation_funcs = make_validation_func_callables(*self.base_validation_funcs,
132 callable_creator=self.get_callables_creator())
133 main_val_func = _and_(validation_funcs)
134 self.main_function = _add_none_handler(main_val_func, none_policy=self.none_policy)
135
136 def get_callables_creator(self):
137 def make_validator_callable(validation_callable, # type: ValidationFunc
138 help_msg=None, # type: str
139 failure_type=None, # type: Type[ValidationFailure]
140 **kw_context_args):
141 """
142
143 :param validation_callable:
144 :param help_msg: custom help message for failures to raise
145 :param failure_type: type of failures to raise
146 :param kw_context_args: contextual arguments for failures to raise
147 :return:
148 """
149 if is_mini_lambda(validation_callable):
150 validation_callable = validation_callable.as_function()
151
152 # support several cases for the validation function signature
153 # `f(val)`, `f(obj, val)` or `f(obj, field, val)`
154 # the validation function has two or three (or more but optional) arguments.
155 # valid8 requires only 1.
156 try:
157 args, varargs, varkwargs, defaults = v8_getfullargspec(validation_callable, skip_bound_arg=True)[0:4]
158
159 nb_args = len(args) if args is not None else 0
160 nbvarargs = 1 if varargs is not None else 0
161 # nbkwargs = 1 if varkwargs is not None else 0
162 # nbdefaults = len(defaults) if defaults is not None else 0
163 except IsBuiltInError:
164 # built-ins: TypeError: <built-in function isinstance> is not a Python function
165 # assume signature with a single positional argument
166 nb_args = 1
167 nbvarargs = 0
168 # nbkwargs = 0
169 # nbdefaults = 0
170
171 if nb_args == 0 and nbvarargs == 0:
172 raise ValueError(
173 "validation function should accept 1, 2, or 3 arguments at least. `f(val)`, `f(obj, val)` or "
174 "`f(obj, field, val)`")
175 elif nb_args == 1 or (nb_args == 0 and nbvarargs >= 1): # varargs default to one argument (compliance with old mini lambda) # noqa
176 # `f(val)`
177 def new_validation_callable(val, **ctx):
178 return validation_callable(val)
179 elif nb_args == 2:
180 # `f(obj, val)`
181 def new_validation_callable(val, **ctx):
182 return validation_callable(ctx['obj'], val)
183 else:
184 # `f(obj, field, val, *opt_args, **ctx)`
185 def new_validation_callable(val, **ctx):
186 # note: field is available both from **ctx and self. Use the "fastest" way
187 return validation_callable(ctx['obj'], self.validated_field, val)
188
189 # preserve the name
190 new_validation_callable.__name__ = get_callable_name(validation_callable)
191
192 return failure_raiser(new_validation_callable, help_msg=help_msg, failure_type=failure_type,
193 **kw_context_args)
194
195 return make_validator_callable
196
197 def get_additional_info_for_repr(self):
198 # type: (...) -> str
199 return 'validated_field=%s' % self.validated_field.qualname
200
201 def _get_name_for_errors(self,
202 name # type: str
203 ):
204 # type: (...) -> str
205 """ override this so that qualname is only called if an error is raised, not before """
206 return self.validated_field.qualname
207
208 def assert_valid(self,
209 obj, # type: Any
210 value, # type: Any
211 error_type=None, # type: Type[ValidationError]
212 help_msg=None, # type: str
213 **ctx):
214 # do not use qualname here so as to save time.
215 super(FieldValidator, self).assert_valid(self.validated_field.name, value,
216 error_type=error_type, help_msg=help_msg,
217 # context info contains obj and field
218 obj=obj, field=self.validated_field, **ctx)
219
220
221 # --------------- converters
222 supported_syntax = 'a Converter, a conversion callable, a tuple(validation_callable, conversion_callable), ' \
223 'a tuple(valid_type, conversion_callable), or a list of several such elements. ' \
224 'A special string \'*\' can be used to denote that all values are accepted.' \
225 'Dicts are supported too, in which case the key is the validation callable or the valid type.' \
226 '[mini_lambda](https://smarie.github.io/python-mini-lambda/) expressions can be used instead of ' \
227 'callables, they will be transformed to functions automatically.'
228
229
230 class Converter(object):
231 """
232 A converter to be used in `field`s.
233 """
234 __slots__ = ('name', )
235
236 def __init__(self, name=None):
237 self.name = name
238
239 def __str__(self):
240 if self.name is not None:
241 return self.name
242 else:
243 return self.__class__.__name__
244
245 def accepts(self, obj, field, value):
246 # type: (...) -> Optional[bool]
247 """
248 Should return `True` or `None` in case the provided value can be converted.
249
250 :param obj:
251 :param field:
252 :param value:
253 :return:
254 """
255 pass
256
257 def convert(self, obj, field, value):
258 # type: (...) -> Any
259 """
260 Converts the provided `value`. This method is only called when `accepts()` has returned `True`.
261 Implementors can dynamically declare that they are not able to convert the given value, by raising an Exception.
262
263 Returning `None` means that the `value` converts to `None`.
264
265 :param obj:
266 :param field:
267 :param value:
268 :return:
269 """
270 raise NotImplementedError()
271
272 @classmethod
273 def create_from_fun(cls,
274 converter_fun, # type: ConverterFuncOrLambda
275 validation_fun=None # type: ValidationFuncOrLambda
276 ):
277 # type: (...) -> Converter
278 """
279 Creates an instance of `Converter` where the `accepts` method is bound to the provided `validation_fun` and the
280 `convert` method bound to the provided `converter_fun`.
281
282 If these methods have less than 3 parameters, the mapping is done acccordingly.
283
284 :param converter_fun:
285 :param validation_fun:
286 :return:
287 """
288 # Mandatory conversion callable
289 if is_mini_lambda(converter_fun):
290 is_mini = True
291 converter_fun = converter_fun.as_function()
292 else:
293 is_mini = False
294 converter_fun_3params = make_3params_callable(converter_fun, is_mini_lambda=is_mini)
295
296 # Optional acceptance callable
297 if validation_fun is not None:
298 if is_mini_lambda(validation_fun):
299 is_mini = True
300 validation_fun = validation_fun.as_function()
301 else:
302 is_mini = False
303 validation_fun_3params = make_3params_callable(validation_fun, is_mini_lambda=is_mini)
304 else:
305 validation_fun_3params = None
306
307 # Finally create the converter instance
308 return ConverterWithFuncs(name=converter_fun_3params.__name__,
309 accepts_fun=validation_fun_3params,
310 convert_fun=converter_fun_3params)
311
312
313 # noinspection PyAbstractClass
314 class ConverterWithFuncs(Converter):
315 """
316 Represents a converter for which the `accepts` and `convert` methods can be provided in the constructor.
317 """
318 __slots__ = ('accepts', 'convert')
319
320 def __init__(self, convert_fun, name=None, accepts_fun=None):
321 # call super to set the name
322 super(ConverterWithFuncs, self).__init__(name=name)
323
324 # use the convert method
325 self.convert = convert_fun
326
327 # use the accepts method if provided, otherwise use parent's
328 if accepts_fun is not None:
329 self.accepts = accepts_fun
330 else:
331 # use parent method - bind it on this instance
332 self.accepts = Converter.accepts.__get__(self, ConverterWithFuncs)
333
334
335 if use_type_hints:
336 # --------------converter type hints
337 # 1. the lowest-level user or 3d party-provided validation functions
338 ConverterFunc = Union[Callable[[Any], Any],
339 Callable[[Any, Any], Any],
340 Callable[[Any, Any, Any], Any]]
341 """A converter function is a callable with signature (val), (obj, val) or (obj, field, val), returning the
342 converted value in case of success"""
343
344 try:
345 # noinspection PyUnresolvedReferences
346 from mini_lambda import y
347
348 ConverterFuncOrLambda = Union[ConverterFunc, type(y)]
349 except ImportError:
350 ConverterFuncOrLambda = ConverterFunc
351
352 # 2. the syntax to optionally transform them into Converter by providing a tuple
353 ValidType = Type
354 # noinspection PyUnboundLocalVariable
355 ConverterFuncDefinition = Union[Converter,
356 ConverterFuncOrLambda,
357 Tuple[ValidationFuncOrLambda, ConverterFuncOrLambda],
358 Tuple[ValidType, ConverterFuncOrLambda]]
359
360 TypeDef = Union[Type, Tuple[Type, ...], Literal['*'], str] # todo remove str whe pycharm understands Literal
361 OneOrSeveralConverterDefinitions = Union[Converter,
362 ConverterFuncOrLambda,
363 Iterable[Tuple[TypeDef, ConverterFuncOrLambda]],
364 Mapping[TypeDef, ConverterFuncOrLambda]]
365 Converters = OneOrSeveralConverterDefinitions
366
367
368 def make_3params_callable(f, # type: Union[ValidationFunc, ConverterFunc]
369 is_mini_lambda=False # type: bool
370 ):
- F821 Undefined name 'Field'
372 """
373 Transforms the provided validation or conversion callable into a callable with 3 arguments (obj, field, val).
374
375 :param f:
376 :param is_mini_lambda: a boolean indicating if the function comes from a mini lambda. In which case we know the
377 signature has one param only (x)
378 :return:
379 """
380 # support several cases for the function signature
381 # `f(val)`, `f(obj, val)` or `f(obj, field, val)`
382 if is_mini_lambda:
383 nbargs = 1
384 nbvarargs = 0
385 # nbkwargs = 0
386 # nbdefaults = 0
387 else:
388 try:
389 args, varargs, varkwargs, defaults = v8_getfullargspec(f, skip_bound_arg=True)[0:4]
390 nbargs = len(args) if args is not None else 0
391 nbvarargs = 1 if varargs is not None else 0
392 # nbkwargs = 1 if varkwargs is not None else 0
393 # nbdefaults = len(defaults) if defaults is not None else 0
394 except IsBuiltInError:
395 # built-ins: TypeError: <built-in function isinstance> is not a Python function
396 # assume signature with a single positional argument
397 nbargs = 1
398 nbvarargs = 0
399 # nbkwargs = 0
400 # nbdefaults = 0
401
402 if nbargs == 0 and nbvarargs == 0:
403 raise ValueError(
404 "validation or converter function should accept 1, 2, or 3 arguments at least. `f(val)`, `f(obj, val)` or "
405 "`f(obj, field, val)`")
406 elif nbargs == 1 or (
407 nbargs == 0 and nbvarargs >= 1): # varargs default to one argument (compliance with old mini lambda)
408 # `f(val)`
409 def new_f_with_3_args(obj, field, value):
410 return f(value)
411
412 elif nbargs == 2:
413 # `f(obj, val)`
414 def new_f_with_3_args(obj, field, value):
415 return f(obj, value)
416
417 else:
418 # `f(obj, field, val, *opt_args, **ctx)`
419 new_f_with_3_args = f
420
421 # preserve the name
422 new_f_with_3_args.__name__ = get_callable_name(f)
423
424 return new_f_with_3_args
425
426
427 JOKER_STR = '*'
428 """String used in converter definition dict entries or tuples, to indicate that the converter accepts everything"""
429
430
431 def make_converter(converter_def # type: ConverterFuncDefinition
432 ):
433 # type: (...) -> Converter
434 """
435 Makes a `Converter` from the provided converter object. Supported formats:
436
437 - a `Converter`
438 - a `<conversion_callable>` with possible signatures `(value)`, `(obj, value)`, `(obj, field, value)`.
439 - a tuple `(<validation_callable>, <conversion_callable>)`
440 - a tuple `(<valid_type>, <conversion_callable>)`
441
442 If no name is provided and a `<conversion_callable>` is present, the callable name will be used as the converter
443 name.
444
445 The name of the conversion callable will be used in that case
446
447 :param converter_def:
448 :return:
449 """
450 try:
451 nb_elts = len(converter_def)
452 except (TypeError, FunctionDefinitionError):
453 # -- single element
454 # handle the special case of a LambdaExpression: automatically convert to a function
455 if not is_mini_lambda(converter_def):
456 if isinstance(converter_def, Converter):
457 # already a converter
458 return converter_def
459 elif not callable(converter_def):
460 raise ValueError('base converter function(s) not compliant with the allowed syntax. Base validation'
461 ' function(s) can be %s Found %s.' % (supported_syntax, converter_def))
462 # single element.
463 return Converter.create_from_fun(converter_def)
464 else:
465 # -- a tuple
466 if nb_elts == 1:
467 converter_fun, validation_fun = converter_def[0], None
468 elif nb_elts == 2:
469 validation_fun, converter_fun = converter_def
470 if validation_fun is not None:
471 if isinstance(validation_fun, type):
472 # a type can be provided to denote accept "instances of <type>"
473 validation_fun = instance_of(validation_fun)
474 elif validation_fun == JOKER_STR:
475 validation_fun = None
476 else:
477 if not is_mini_lambda(validation_fun) and not callable(validation_fun):
478 raise ValueError('base converter function(s) not compliant with the allowed syntax. Validator '
479 'is incorrect. Base converter function(s) can be %s Found %s.'
480 % (supported_syntax, converter_def))
481 else:
482 raise ValueError(
483 'tuple in converter_fun definition should have length 1, or 2. Found: %s' % (converter_def,))
484
485 # check that the definition is valid
486 if not is_mini_lambda(converter_fun) and not callable(converter_fun):
487 raise ValueError('base converter function(s) not compliant with the allowed syntax. Base converter'
488 ' function(s) can be %s Found %s.' % (supported_syntax, converter_def))
489
490 # finally create the failure raising callable
491 return Converter.create_from_fun(converter_fun, validation_fun)
492
493
494 def make_converters_list(converters # type: OneOrSeveralConverterDefinitions
495 ):
496 # type: (...) -> Tuple[Converter, ...]
497 """
498 Creates a tuple of converters from the provided `converters`. The following things are supported:
499
500 - a single item. This can be a `Converter`, a `<converter_callable>`, a tuple
501 `(<acceptance_callable>, <converter_callable>)` or a tuple `(<accepted_type>, <converter_callable>)`.
502 `<accepted_type>` can also contain `None` or `'*'`, both mean "anything".
503
504 - a list of such items
505
506 - a dictionary-like of `<acceptance>: <converter_callable>`, where `<acceptance>` can be an `<acceptance_callable>`
507 or an `<accepted_type>`.
508
509 :param converters:
510 :return:
511 """
512 # support a single tuple
513 if isinstance(converters, tuple):
514 converters = [converters]
515
516 try:
517 # mapping ?
518 c_items = iter(converters.items())
519 except (AttributeError, FunctionDefinitionError):
520 try:
521 # iterable ?
522 c_iter = iter(converters)
523 except (TypeError, FunctionDefinitionError):
524 # single converter: create a tuple manually
525 all_converters = (make_converter(converters),)
526 else:
527 # iterable
528 all_converters = tuple(make_converter(c) for c in c_iter)
529 else:
530 # mapping: assume that each entry is {validation_fun: converter_fun}
531 all_converters = tuple(make_converter((k, v)) for k, v in c_items)
532
533 if len(all_converters) == 0:
534 raise ValueError("No converters provided")
535 else:
536 return all_converters
537
538
- F821 Undefined name 'Field'
540 value, # type: Any
541 obj=None # type: Any
542 ):
543 # type: (...) -> Tuple[Any, DetailedConversionResults]
544 """
545 Utility method to debug conversion issues.
546 Instead of just returning the converted value, it also returns conversion details.
547
548 In case conversion can not be made, a `ConversionError` is raised.
549
550 Inspired by the `getversion` library.
551
552 :param obj:
553 :param field:
554 :param value:
555 :return:
556 """
557 errors = OrderedDict()
558
559 for conv in field.converters:
560 try:
561 # check if converter accepts this ?
562 accepted = conv.accepts(obj, field, value)
563 except Exception as e:
564 # error in acceptance test
565 errors[conv] = "Acceptance test: ERROR [%s] %s" % (e.__class__.__name__, e)
566 else:
567 if accepted is not None and not accepted:
568 # acceptance failed
569 errors[conv] = "Acceptance test: REJECTED (returned %s)" % accepted
570 else:
571 # accepted! (None or True truth value)
572 try:
573 # apply converter
574 converted_value = conv.convert(obj, field, value)
575 except Exception as e:
576 errors[conv] = "Acceptance test: SUCCESS (returned %s). Conversion: ERROR [%s] %s" \
577 % (accepted, e.__class__.__name__, e)
578 else:
579 # conversion success !
580 errors[conv] = "Acceptance test: SUCCESS (returned %s). Conversion: SUCCESS -> %s" \
581 % (accepted, converted_value)
582 return converted_value, DetailedConversionResults(value, field, obj, errors, conv, converted_value)
583
584 raise ConversionError(value_to_convert=value, field=field, obj=obj, err_dct=errors)
585
586
587 class ConversionError(Exception):
588 """
589 Final exception Raised by `trace_convert` when a value cannot be converted successfully
590 """
591 __slots__ = 'value_to_convert', 'field', 'obj', 'err_dct'
592
593 def __init__(self, value_to_convert, field, obj, err_dct):
594 self.value_to_convert = value_to_convert
595 self.field = field
596 self.obj = obj
597 self.err_dct = err_dct
598 super(ConversionError, self).__init__()
599
600 def __str__(self):
601 return "Unable to convert value %r. Results:\n%s" \
602 % (self.value_to_convert, err_dct_to_str(self.err_dct))
603
604
605 def err_dct_to_str(err_dct # type: Dict[Converter, str]
606 ):
607 # type: (...) -> str
608 msg = ""
609 for converter, err in err_dct.items():
610 msg += " - Converter '%s': %s\n" % (converter, err)
611
612 return msg
613
614
615 class DetailedConversionResults(object):
616 """
617 Returned by `trace_convert` for detailed results about which converter failed before the winning one.
618 """
619 __slots__ = 'value_to_convert', 'field', 'obj', 'err_dct', 'winning_converter', 'converted_value'
620
621 def __init__(self, value_to_convert, field, obj, err_dct, winning_converter, converted_value):
622 self.value_to_convert = value_to_convert
623 self.field = field
624 self.obj = obj
625 self.err_dct = err_dct
626 self.winning_converter = winning_converter
627 self.converted_value = converted_value
628
629 def __str__(self):
630 return "Value %r successfully converted to %r using converter '%s', after the following attempts:\n%s"\
631 % (self.value_to_convert, self.converted_value, self.winning_converter, err_dct_to_str(self.err_dct))