Coverage for pyfields/validate_n_convert.py: 81%
224 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-11-06 16:35 +0000
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/python-pyfields>
3#
4# License: 3-clause BSD, <https://github.com/smarie/python-pyfields/blob/master/LICENSE>
5import sys
6from collections import OrderedDict
8from valid8 import Validator, failure_raiser, ValidationError, ValidationFailure
9from valid8.base import getfullargspec as v8_getfullargspec, get_callable_name, is_mini_lambda
10from valid8.common_syntax import FunctionDefinitionError, make_validation_func_callables
11from valid8.composition import _and_
12from valid8.entry_points import _add_none_handler
13from valid8.utils.signature_tools import IsBuiltInError
14from valid8.validation_lib import instance_of
16try: # python 3.5+
17 # noinspection PyUnresolvedReferences
18 from typing import Callable, Type, Any, TypeVar, Union, Iterable, Tuple, Mapping, Optional, Dict, Literal
19 # from valid8.common_syntax import ValidationFuncs
20 use_type_hints = sys.version_info > (3, 0)
21except ImportError:
22 use_type_hints = False
25if use_type_hints: 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true
26 T = TypeVar('T')
28 # ------------- validator type hints -----------
29 # 1. the lowest-level user or 3d party-provided validation functions
30 ValidationFunc = Union[Callable[[Any], Any],
31 Callable[[Any, Any], Any],
32 Callable[[Any, Any, Any], Any]]
33 """A validation function is a callable with signature (val), (obj, val) or (obj, field, val), returning `True`
34 or `None` in case of success"""
36 try:
37 # noinspection PyUnresolvedReferences
38 from mini_lambda import y
39 ValidationFuncOrLambda = Union[ValidationFunc, type(y)]
40 except ImportError:
41 ValidationFuncOrLambda = ValidationFunc
43 # 2. the syntax to optionally transform them into failure raisers by providing a tuple
44 ValidatorDef = Union[ValidationFuncOrLambda,
45 Tuple[ValidationFuncOrLambda, str],
46 Tuple[ValidationFuncOrLambda, Type[ValidationFailure]],
47 Tuple[ValidationFuncOrLambda, str, Type[ValidationFailure]]
48 ]
49 """A validator is a validation function together with optional error message and error type"""
51 # 3. the syntax to describe several validation functions at once
52 VFDefinitionElement = Union[str, Type[ValidationFailure], ValidationFuncOrLambda]
53 """This type represents one of the elements that can define a checker: help msg, failure type, callable"""
55 OneOrSeveralVFDefinitions = Union[ValidatorDef,
56 Iterable[ValidatorDef],
57 Mapping[VFDefinitionElement, Union[VFDefinitionElement,
58 Tuple[VFDefinitionElement, ...]]]]
59 """Several validators can be provided as a singleton, iterable, or dict-like. In that case the value can be a
60 single variable or a tuple, and it will be combined with the key to form the validator. So you can use any of
61 the elements defining a validators as the key."""
63 # shortcut name used everywhere. Less explicit
64 Validators = OneOrSeveralVFDefinitions
67class FieldValidator(Validator):
68 """
69 Represents a `Validator` responsible to validate a `field`
70 """
71 __slots__ = '__weakref__', 'validated_field', 'base_validation_funcs'
73 def __init__(self,
74 validated_field, # type: 'DescriptorField'
75 validators, # type: Validators
76 **kwargs
77 ):
78 """
80 :param validated_field: the field being validated.
81 :param validators: the base validation function or list of base validation functions to use. A callable, a
82 tuple(callable, help_msg_str), a tuple(callable, failure_type), tuple(callable, help_msg_str, failure_type)
83 or a list of several such elements. A dict can also be used.
84 Tuples indicate an implicit `failure_raiser`.
85 [mini_lambda](https://smarie.github.io/python-mini-lambda/) expressions can be used instead
86 of callables, they will be transformed to functions automatically.
87 :param error_type: a subclass of ValidationError to raise in case of validation failure. By default a
88 ValidationError will be raised with the provided help_msg
89 :param help_msg: an optional help message to be used in the raised error in case of validation failure.
90 :param none_policy: describes how None values should be handled. See `NonePolicy` for the various possibilities.
91 Default is `NonePolicy.VALIDATE`, meaning that None values will be treated exactly like other values and
92 follow the same validation process.
93 :param kw_context_args: optional contextual information to store in the exception, and that may be also used
94 to format the help message
95 """
96 # store this additional info about the function been validated
97 self.validated_field = validated_field
99 try: # dict ?
100 validators.keys()
101 except (AttributeError, FunctionDefinitionError): # FunctionDefinitionError when mini_lambda
102 if isinstance(validators, tuple): 102 ↛ 104line 102 didn't jump to line 104, because the condition on line 102 was never true
103 # single tuple
104 validators = (validators,)
105 else:
106 try: # iterable
107 iter(validators)
108 except (TypeError, FunctionDefinitionError): # FunctionDefinitionError when mini_lambda
109 # single
110 validators = (validators,)
111 else:
112 # dict
113 validators = (validators,)
115 # remember validation funcs so that we can add more later
116 self.base_validation_funcs = validators
118 super(FieldValidator, self).__init__(*validators, **kwargs)
120 def add_validator(self,
121 validation_func # type: ValidatorDef
122 ):
123 """
124 Adds the provided validation function to the existing list of validation functions
125 :param validation_func:
126 :return:
127 """
128 self.base_validation_funcs = self.base_validation_funcs + (validation_func, )
130 # do the same than in super.init, once again. TODO optimize ...
131 validation_funcs = make_validation_func_callables(*self.base_validation_funcs,
132 callable_creator=self.get_callables_creator())
133 main_val_func = _and_(validation_funcs)
134 self.main_function = _add_none_handler(main_val_func, none_policy=self.none_policy)
136 def get_callables_creator(self):
137 def make_validator_callable(validation_callable, # type: ValidationFunc
138 help_msg=None, # type: str
139 failure_type=None, # type: Type[ValidationFailure]
140 **kw_context_args):
141 """
143 :param validation_callable:
144 :param help_msg: custom help message for failures to raise
145 :param failure_type: type of failures to raise
146 :param kw_context_args: contextual arguments for failures to raise
147 :return:
148 """
149 if is_mini_lambda(validation_callable):
150 validation_callable = validation_callable.as_function()
152 # support several cases for the validation function signature
153 # `f(val)`, `f(obj, val)` or `f(obj, field, val)`
154 # the validation function has two or three (or more but optional) arguments.
155 # valid8 requires only 1.
156 try:
157 args, varargs, varkwargs, defaults = v8_getfullargspec(validation_callable, skip_bound_arg=True)[0:4]
159 nb_args = len(args) if args is not None else 0
160 nbvarargs = 1 if varargs is not None else 0
161 # nbkwargs = 1 if varkwargs is not None else 0
162 # nbdefaults = len(defaults) if defaults is not None else 0
163 except IsBuiltInError:
164 # built-ins: TypeError: <built-in function isinstance> is not a Python function
165 # assume signature with a single positional argument
166 nb_args = 1
167 nbvarargs = 0
168 # nbkwargs = 0
169 # nbdefaults = 0
171 if nb_args == 0 and nbvarargs == 0: 171 ↛ 172line 171 didn't jump to line 172, because the condition on line 171 was never true
172 raise ValueError(
173 "validation function should accept 1, 2, or 3 arguments at least. `f(val)`, `f(obj, val)` or "
174 "`f(obj, field, val)`")
175 elif nb_args == 1 or (nb_args == 0 and nbvarargs >= 1): # varargs default to one argument (compliance with old mini lambda) # noqa
176 # `f(val)`
177 def new_validation_callable(val, **ctx):
178 return validation_callable(val)
179 elif nb_args == 2:
180 # `f(obj, val)`
181 def new_validation_callable(val, **ctx):
182 return validation_callable(ctx['obj'], val)
183 else:
184 # `f(obj, field, val, *opt_args, **ctx)`
185 def new_validation_callable(val, **ctx):
186 # note: field is available both from **ctx and self. Use the "fastest" way
187 return validation_callable(ctx['obj'], self.validated_field, val)
189 # preserve the name
190 new_validation_callable.__name__ = get_callable_name(validation_callable)
192 return failure_raiser(new_validation_callable, help_msg=help_msg, failure_type=failure_type,
193 **kw_context_args)
195 return make_validator_callable
197 def get_additional_info_for_repr(self):
198 # type: (...) -> str
199 return 'validated_field=%s' % self.validated_field.qualname
201 def _get_name_for_errors(self,
202 name # type: str
203 ):
204 # type: (...) -> str
205 """ override this so that qualname is only called if an error is raised, not before """
206 return self.validated_field.qualname
208 def assert_valid(self,
209 obj, # type: Any
210 value, # type: Any
211 error_type=None, # type: Type[ValidationError]
212 help_msg=None, # type: str
213 **ctx):
214 # do not use qualname here so as to save time.
215 super(FieldValidator, self).assert_valid(self.validated_field.name, value,
216 error_type=error_type, help_msg=help_msg,
217 # context info contains obj and field
218 obj=obj, field=self.validated_field, **ctx)
221# --------------- converters
222supported_syntax = 'a Converter, a conversion callable, a tuple(validation_callable, conversion_callable), ' \
223 'a tuple(valid_type, conversion_callable), or a list of several such elements. ' \
224 'A special string \'*\' can be used to denote that all values are accepted.' \
225 'Dicts are supported too, in which case the key is the validation callable or the valid type.' \
226 '[mini_lambda](https://smarie.github.io/python-mini-lambda/) expressions can be used instead of ' \
227 'callables, they will be transformed to functions automatically.'
230class Converter(object):
231 """
232 A converter to be used in `field`s.
233 """
234 __slots__ = ('name', )
236 def __init__(self, name=None):
237 self.name = name
239 def __str__(self):
240 if self.name is not None:
241 return self.name
242 else:
243 return self.__class__.__name__
245 def accepts(self, obj, field, value):
246 # type: (...) -> Optional[bool]
247 """
248 Should return `True` or `None` in case the provided value can be converted.
250 :param obj:
251 :param field:
252 :param value:
253 :return:
254 """
255 pass
257 def convert(self, obj, field, value):
258 # type: (...) -> Any
259 """
260 Converts the provided `value`. This method is only called when `accepts()` has returned `True`.
261 Implementors can dynamically declare that they are not able to convert the given value, by raising an Exception.
263 Returning `None` means that the `value` converts to `None`.
265 :param obj:
266 :param field:
267 :param value:
268 :return:
269 """
270 raise NotImplementedError()
272 @classmethod
273 def create_from_fun(cls,
274 converter_fun, # type: ConverterFuncOrLambda
275 validation_fun=None # type: ValidationFuncOrLambda
276 ):
277 # type: (...) -> Converter
278 """
279 Creates an instance of `Converter` where the `accepts` method is bound to the provided `validation_fun` and the
280 `convert` method bound to the provided `converter_fun`.
282 If these methods have less than 3 parameters, the mapping is done acccordingly.
284 :param converter_fun:
285 :param validation_fun:
286 :return:
287 """
288 # Mandatory conversion callable
289 if is_mini_lambda(converter_fun): 289 ↛ 290line 289 didn't jump to line 290, because the condition on line 289 was never true
290 is_mini = True
291 converter_fun = converter_fun.as_function()
292 else:
293 is_mini = False
294 converter_fun_3params = make_3params_callable(converter_fun, is_mini_lambda=is_mini)
296 # Optional acceptance callable
297 if validation_fun is not None:
298 if is_mini_lambda(validation_fun): 298 ↛ 299line 298 didn't jump to line 299, because the condition on line 298 was never true
299 is_mini = True
300 validation_fun = validation_fun.as_function()
301 else:
302 is_mini = False
303 validation_fun_3params = make_3params_callable(validation_fun, is_mini_lambda=is_mini)
304 else:
305 validation_fun_3params = None
307 # Finally create the converter instance
308 return ConverterWithFuncs(name=converter_fun_3params.__name__,
309 accepts_fun=validation_fun_3params,
310 convert_fun=converter_fun_3params)
313# noinspection PyAbstractClass
314class ConverterWithFuncs(Converter):
315 """
316 Represents a converter for which the `accepts` and `convert` methods can be provided in the constructor.
317 """
318 __slots__ = ('accepts', 'convert')
320 def __init__(self, convert_fun, name=None, accepts_fun=None):
321 # call super to set the name
322 super(ConverterWithFuncs, self).__init__(name=name)
324 # use the convert method
325 self.convert = convert_fun
327 # use the accepts method if provided, otherwise use parent's
328 if accepts_fun is not None:
329 self.accepts = accepts_fun
330 else:
331 # use parent method - bind it on this instance
332 self.accepts = Converter.accepts.__get__(self, ConverterWithFuncs)
335if use_type_hints: 335 ↛ 338line 335 didn't jump to line 338, because the condition on line 335 was never true
336 # --------------converter type hints
337 # 1. the lowest-level user or 3d party-provided validation functions
338 ConverterFunc = Union[Callable[[Any], Any],
339 Callable[[Any, Any], Any],
340 Callable[[Any, Any, Any], Any]]
341 """A converter function is a callable with signature (val), (obj, val) or (obj, field, val), returning the
342 converted value in case of success"""
344 try:
345 # noinspection PyUnresolvedReferences
346 from mini_lambda import y
348 ConverterFuncOrLambda = Union[ConverterFunc, type(y)]
349 except ImportError:
350 ConverterFuncOrLambda = ConverterFunc
352 # 2. the syntax to optionally transform them into Converter by providing a tuple
353 ValidType = Type
354 # noinspection PyUnboundLocalVariable
355 ConverterFuncDefinition = Union[Converter,
356 ConverterFuncOrLambda,
357 Tuple[ValidationFuncOrLambda, ConverterFuncOrLambda],
358 Tuple[ValidType, ConverterFuncOrLambda]]
360 TypeDef = Union[Type, Tuple[Type, ...], Literal['*'], str] # todo remove str whe pycharm understands Literal
361 OneOrSeveralConverterDefinitions = Union[Converter,
362 ConverterFuncOrLambda,
363 Iterable[Tuple[TypeDef, ConverterFuncOrLambda]],
364 Mapping[TypeDef, ConverterFuncOrLambda]]
365 Converters = OneOrSeveralConverterDefinitions
368def make_3params_callable(f, # type: Union[ValidationFunc, ConverterFunc]
369 is_mini_lambda=False # type: bool
370 ):
371 # type: (...) -> Callable[[Any, 'Field', Any], Any]
372 """
373 Transforms the provided validation or conversion callable into a callable with 3 arguments (obj, field, val).
375 :param f:
376 :param is_mini_lambda: a boolean indicating if the function comes from a mini lambda. In which case we know the
377 signature has one param only (x)
378 :return:
379 """
380 # support several cases for the function signature
381 # `f(val)`, `f(obj, val)` or `f(obj, field, val)`
382 if is_mini_lambda: 382 ↛ 383line 382 didn't jump to line 383, because the condition on line 382 was never true
383 nbargs = 1
384 nbvarargs = 0
385 # nbkwargs = 0
386 # nbdefaults = 0
387 else:
388 try:
389 args, varargs, varkwargs, defaults = v8_getfullargspec(f, skip_bound_arg=True)[0:4]
390 nbargs = len(args) if args is not None else 0
391 nbvarargs = 1 if varargs is not None else 0
392 # nbkwargs = 1 if varkwargs is not None else 0
393 # nbdefaults = len(defaults) if defaults is not None else 0
394 except IsBuiltInError:
395 # built-ins: TypeError: <built-in function isinstance> is not a Python function
396 # assume signature with a single positional argument
397 nbargs = 1
398 nbvarargs = 0
399 # nbkwargs = 0
400 # nbdefaults = 0
402 if nbargs == 0 and nbvarargs == 0: 402 ↛ 403line 402 didn't jump to line 403, because the condition on line 402 was never true
403 raise ValueError(
404 "validation or converter function should accept 1, 2, or 3 arguments at least. `f(val)`, `f(obj, val)` or "
405 "`f(obj, field, val)`")
406 elif nbargs == 1 or (
407 nbargs == 0 and nbvarargs >= 1): # varargs default to one argument (compliance with old mini lambda)
408 # `f(val)`
409 def new_f_with_3_args(obj, field, value):
410 return f(value)
412 elif nbargs == 2:
413 # `f(obj, val)`
414 def new_f_with_3_args(obj, field, value):
415 return f(obj, value)
417 else:
418 # `f(obj, field, val, *opt_args, **ctx)`
419 new_f_with_3_args = f
421 # preserve the name
422 new_f_with_3_args.__name__ = get_callable_name(f)
424 return new_f_with_3_args
427JOKER_STR = '*'
428"""String used in converter definition dict entries or tuples, to indicate that the converter accepts everything"""
431def make_converter(converter_def # type: ConverterFuncDefinition
432 ):
433 # type: (...) -> Converter
434 """
435 Makes a `Converter` from the provided converter object. Supported formats:
437 - a `Converter`
438 - a `<conversion_callable>` with possible signatures `(value)`, `(obj, value)`, `(obj, field, value)`.
439 - a tuple `(<validation_callable>, <conversion_callable>)`
440 - a tuple `(<valid_type>, <conversion_callable>)`
442 If no name is provided and a `<conversion_callable>` is present, the callable name will be used as the converter
443 name.
445 The name of the conversion callable will be used in that case
447 :param converter_def:
448 :return:
449 """
450 try:
451 nb_elts = len(converter_def)
452 except (TypeError, FunctionDefinitionError):
453 # -- single element
454 # handle the special case of a LambdaExpression: automatically convert to a function
455 if not is_mini_lambda(converter_def): 455 ↛ 463line 455 didn't jump to line 463, because the condition on line 455 was never false
456 if isinstance(converter_def, Converter):
457 # already a converter
458 return converter_def
459 elif not callable(converter_def): 459 ↛ 460line 459 didn't jump to line 460, because the condition on line 459 was never true
460 raise ValueError('base converter function(s) not compliant with the allowed syntax. Base validation'
461 ' function(s) can be %s Found %s.' % (supported_syntax, converter_def))
462 # single element.
463 return Converter.create_from_fun(converter_def)
464 else:
465 # -- a tuple
466 if nb_elts == 1: 466 ↛ 467line 466 didn't jump to line 467, because the condition on line 466 was never true
467 converter_fun, validation_fun = converter_def[0], None
468 elif nb_elts == 2: 468 ↛ 482line 468 didn't jump to line 482, because the condition on line 468 was never false
469 validation_fun, converter_fun = converter_def
470 if validation_fun is not None:
471 if isinstance(validation_fun, type):
472 # a type can be provided to denote accept "instances of <type>"
473 validation_fun = instance_of(validation_fun)
474 elif validation_fun == JOKER_STR:
475 validation_fun = None
476 else:
477 if not is_mini_lambda(validation_fun) and not callable(validation_fun): 477 ↛ 478line 477 didn't jump to line 478, because the condition on line 477 was never true
478 raise ValueError('base converter function(s) not compliant with the allowed syntax. Validator '
479 'is incorrect. Base converter function(s) can be %s Found %s.'
480 % (supported_syntax, converter_def))
481 else:
482 raise ValueError(
483 'tuple in converter_fun definition should have length 1, or 2. Found: %s' % (converter_def,))
485 # check that the definition is valid
486 if not is_mini_lambda(converter_fun) and not callable(converter_fun): 486 ↛ 487line 486 didn't jump to line 487, because the condition on line 486 was never true
487 raise ValueError('base converter function(s) not compliant with the allowed syntax. Base converter'
488 ' function(s) can be %s Found %s.' % (supported_syntax, converter_def))
490 # finally create the failure raising callable
491 return Converter.create_from_fun(converter_fun, validation_fun)
494def make_converters_list(converters # type: OneOrSeveralConverterDefinitions
495 ):
496 # type: (...) -> Tuple[Converter, ...]
497 """
498 Creates a tuple of converters from the provided `converters`. The following things are supported:
500 - a single item. This can be a `Converter`, a `<converter_callable>`, a tuple
501 `(<acceptance_callable>, <converter_callable>)` or a tuple `(<accepted_type>, <converter_callable>)`.
502 `<accepted_type>` can also contain `None` or `'*'`, both mean "anything".
504 - a list of such items
506 - a dictionary-like of `<acceptance>: <converter_callable>`, where `<acceptance>` can be an `<acceptance_callable>`
507 or an `<accepted_type>`.
509 :param converters:
510 :return:
511 """
512 # support a single tuple
513 if isinstance(converters, tuple):
514 converters = [converters]
516 try:
517 # mapping ?
518 c_items = iter(converters.items())
519 except (AttributeError, FunctionDefinitionError):
520 try:
521 # iterable ?
522 c_iter = iter(converters)
523 except (TypeError, FunctionDefinitionError):
524 # single converter: create a tuple manually
525 all_converters = (make_converter(converters),)
526 else:
527 # iterable
528 all_converters = tuple(make_converter(c) for c in c_iter)
529 else:
530 # mapping: assume that each entry is {validation_fun: converter_fun}
531 all_converters = tuple(make_converter((k, v)) for k, v in c_items)
533 if len(all_converters) == 0: 533 ↛ 534line 533 didn't jump to line 534, because the condition on line 533 was never true
534 raise ValueError("No converters provided")
535 else:
536 return all_converters
539def trace_convert(field, # type: 'Field'
540 value, # type: Any
541 obj=None # type: Any
542 ):
543 # type: (...) -> Tuple[Any, DetailedConversionResults]
544 """
545 Utility method to debug conversion issues.
546 Instead of just returning the converted value, it also returns conversion details.
548 In case conversion can not be made, a `ConversionError` is raised.
550 Inspired by the `getversion` library.
552 :param obj:
553 :param field:
554 :param value:
555 :return:
556 """
557 errors = OrderedDict()
559 for conv in field.converters:
560 try:
561 # check if converter accepts this ?
562 accepted = conv.accepts(obj, field, value)
563 except Exception as e:
564 # error in acceptance test
565 errors[conv] = "Acceptance test: ERROR [%s] %s" % (e.__class__.__name__, e)
566 else:
567 if accepted is not None and not accepted:
568 # acceptance failed
569 errors[conv] = "Acceptance test: REJECTED (returned %s)" % accepted
570 else:
571 # accepted! (None or True truth value)
572 try:
573 # apply converter
574 converted_value = conv.convert(obj, field, value)
575 except Exception as e:
576 errors[conv] = "Acceptance test: SUCCESS (returned %s). Conversion: ERROR [%s] %s" \
577 % (accepted, e.__class__.__name__, e)
578 else:
579 # conversion success !
580 errors[conv] = "Acceptance test: SUCCESS (returned %s). Conversion: SUCCESS -> %s" \
581 % (accepted, converted_value)
582 return converted_value, DetailedConversionResults(value, field, obj, errors, conv, converted_value)
584 raise ConversionError(value_to_convert=value, field=field, obj=obj, err_dct=errors)
587class ConversionError(Exception):
588 """
589 Final exception Raised by `trace_convert` when a value cannot be converted successfully
590 """
591 __slots__ = 'value_to_convert', 'field', 'obj', 'err_dct'
593 def __init__(self, value_to_convert, field, obj, err_dct):
594 self.value_to_convert = value_to_convert
595 self.field = field
596 self.obj = obj
597 self.err_dct = err_dct
598 super(ConversionError, self).__init__()
600 def __str__(self):
601 return "Unable to convert value %r. Results:\n%s" \
602 % (self.value_to_convert, err_dct_to_str(self.err_dct))
605def err_dct_to_str(err_dct # type: Dict[Converter, str]
606 ):
607 # type: (...) -> str
608 msg = ""
609 for converter, err in err_dct.items():
610 msg += " - Converter '%s': %s\n" % (converter, err)
612 return msg
615class DetailedConversionResults(object):
616 """
617 Returned by `trace_convert` for detailed results about which converter failed before the winning one.
618 """
619 __slots__ = 'value_to_convert', 'field', 'obj', 'err_dct', 'winning_converter', 'converted_value'
621 def __init__(self, value_to_convert, field, obj, err_dct, winning_converter, converted_value):
622 self.value_to_convert = value_to_convert
623 self.field = field
624 self.obj = obj
625 self.err_dct = err_dct
626 self.winning_converter = winning_converter
627 self.converted_value = converted_value
629 def __str__(self):
630 return "Value %r successfully converted to %r using converter '%s', after the following attempts:\n%s"\
631 % (self.value_to_convert, self.converted_value, self.winning_converter, err_dct_to_str(self.err_dct))