pyfields/autofields_.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/python-pyfields>
3 #
4 # License: 3-clause BSD, <https://github.com/smarie/python-pyfields/blob/master/LICENSE>
5 import sys
6 from copy import deepcopy
7 from inspect import isdatadescriptor, ismethoddescriptor
8
9 try:
10 from typing import Union, Callable, Type, Any, TypeVar, Tuple, Iterable
11 DecoratedClass = TypeVar("DecoratedClass", bound=Type[Any])
12 except ImportError:
13 pass
14
15
16 from .core import Field, field
17 from .init_makers import make_init as mkinit
18 from .helpers import copy_value, get_fields
19
20
21 PY36 = sys.version_info >= (3, 6)
22 DEFAULT_EXCLUDED = ('_abc_impl',)
23
24
25 def _make_init(cls):
26 """Utility method used in autofields and autoclass to create the constructor based on the class fields"""
27 if "__init__" not in cls.__dict__:
28 new_init = mkinit()
29 cls.__init__ = new_init
30 # attach explicitly to the class so that the descriptor is correctly completed.
31 new_init.__set_name__(cls, '__init__')
32
33
34 def autofields(check_types=False, # type: Union[bool, DecoratedClass]
35 include_upper=False, # type: bool
36 include_dunder=False, # type: bool
37 exclude=DEFAULT_EXCLUDED, # type: Iterable[str]
38 make_init=True, # type: bool
39 ):
40 # type: (...) -> Union[Callable[[DecoratedClass], DecoratedClass], DecoratedClass]
41 """
42 Decorator to automatically create fields and constructor on a class.
43
44 When a class is decorated with `@autofields`, all of its members are automatically transformed to fields.
45 More precisely: members that only contain a type annotation become mandatory fields, while members that contain a
46 value (with or without type annotation) become optional fields with a `copy_value` default_factory.
47
48 By default, the following members are NOT transformed into fields:
49
50 * members with upper-case names. This is because this kind of name formatting usually denotes class constants. They
51 can be transformed to fields by setting `include_upper=True`.
52 * members with dunder-like names. They can be included using `include_dunder=True`. Note that reserved python
53 dunder names such as `__name__`, `__setattr__`, etc. can not be transformed to fields, even when
54 `include_dunder=True`.
55 * members that are classes or methods defined in the class (that is, where their `.__name__` is the same name than
56 the member name).
57 * members that are already fields. Therefore you can continue to use `field()` on certain members explicitly if
58 you need to add custom validators, converters, etc.
59
60 All created fields have their `type_hint` filled with the type hint associated with the member, and have
61 `check_type=False` by default. This can be changed by setting `check_types=True`.
62
63 Finally, in addition, an init method (constructor) is generated for the class, using `make_init()`. This may be
64 disabled by setting `make_init=False`..
65
66 >>> import sys, pytest
67 >>> if sys.version_info < (3, 6): pytest.skip("doctest skipped for python < 3.6")
68 ...
69 >>> @autofields
70 ... class Pocket:
71 ... SENTENCE = "hello world" # uppercase: not a field
72 ... size: int # mandatory field
73 ... items = [] # optional - default value will be a factory
74 ...
75 >>> p = Pocket(size=10)
76 >>> p.items
77 []
78 >>> Pocket(size=10, SENTENCE="hello")
79 Traceback (most recent call last):
80 ...
81 TypeError: __init__() got an unexpected keyword argument 'SENTENCE'
82
83
84 :param check_types: boolean flag (default: `False`) indicating the value of `check_type` for created fields. Note
85 that the type hint of each created field is copied from the type hint of the member it originates from.
86 :param include_upper: boolean flag (default: `False`) indicating whether upper-case class members should be also
87 transformed to fields (usually such names are reserved for class constants, not for fields).
88 :param include_dunder: boolean flag (default: `False`) indicating whether dunder-named class members should be also
89 transformed to fields. Note that even if you set this to True, members with reserved python dunder names will
90 not be transformed. See `is_reserved_dunder` for the list of reserved names.
91 :param exclude: a tuple of field names that should be excluded from automatic creation. By default this is set to
92 `DEFAULT_EXCLUDED`, which eliminates fields created by `ABC`.
93 :param make_init: boolean flag (default: `True`) indicating whether a constructor should be created for the class if
94 no `__init__` method is present. Such constructor will be created using `__init__ = make_init()`.
95 :return:
96 """
97 def _autofields(cls):
98 NO_DEFAULT = object()
99
100 try:
101 # Are type hints present ?
102 # note: since this attribute can be inherited, we get the own attribute only
103 # cls_annotations = cls.__annotations__
104 cls_annotations = getownattr(cls, "__annotations__")
105 except AttributeError:
106 # No type hints: shortcut. note: do not return a generator since we'll modify __dict__ in the loop after
107 members_defs = tuple((k, None, v) for k, v in cls.__dict__.items())
108 else:
109 # Fill the list of potential fields definitions
110 members_defs = []
111 cls_dict = cls.__dict__
112
113 if not PY36:
114 # Is this even possible ? does not seem so. Raising an error until this is reported
115 raise ValueError("Unsupported case: `__annotations__` is present while python is < 3.6 - please report")
116 # # dont care about the order, it is not preserved
117 # # -- fields with type hint
118 # for member_name, type_hint in cls_annotations.items():
119 # members_defs.append((member_name, type_hint, cls_dict.get(member_name, NO_DEFAULT)))
120 #
121 # # -- fields without type hint
122 # members_with_type = set(cls_annotations.keys())
123 # for member_name, default_value in cls_dict.items():
124 # if member_name not in members_with_type:
125 # members_defs.append((member_name, None, default_value))
126 #
127 else:
128 # create a list of members with consistent order
129 members_with_type_and_value = set(cls_annotations.keys()).intersection(cls_dict.keys())
130
131 in_types = [name for name in cls_annotations if name in members_with_type_and_value]
132 in_values = [name for name in cls_dict if name in members_with_type_and_value]
- S101 Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
134
135 def t_gen():
136 """ generator used to fill the definitions for members only in annotations dict """
137 next_stop_name = yield
138 for _name, _type_hint in cls_annotations.items():
139 if _name != next_stop_name:
140 members_defs.append((_name, _type_hint, NO_DEFAULT))
141 else:
142 next_stop_name = yield
143
144 def v_gen():
145 """ generator used to fill the definitions for members only in the values dict """
146 next_stop_name, next_stop_type_hint = yield
147 for _name, _default_value in cls_dict.items():
148 if _name != next_stop_name:
149 members_defs.append((_name, None, _default_value))
150 else:
151 members_defs.append((_name, next_stop_type_hint, _default_value))
152 next_stop_name, next_stop_type_hint = yield
153
154 types_gen = t_gen()
155 types_gen.send(None)
156 values_gen = v_gen()
157 values_gen.send(None)
158 for common_name in in_types:
159 types_gen.send(common_name)
160 values_gen.send((common_name, cls_annotations[common_name]))
161 # last one
162 try:
163 types_gen.send(None)
164 except StopIteration:
165 pass
166 try:
167 values_gen.send((None, None))
168 except StopIteration:
169 pass
170
171 # Main loop : for each member, possibly create a field()
172 for member_name, type_hint, default_value in members_defs:
173 if member_name in exclude:
174 # excluded explicitly
175 continue
176 elif not include_upper and member_name == member_name.upper():
177 # excluded uppercase
178 continue
179 elif (include_dunder and is_reserved_dunder(member_name)) \
180 or is_dunder(member_name):
181 # excluded dunder
182 continue
183 elif isinstance(default_value, Field):
184 # already a field, no need to create
185 # but in order to preserve relative order with generated fields, detach and attach again
186 try:
187 delattr(cls, member_name)
188 except AttributeError:
189 pass
190 setattr(cls, member_name, default_value)
191 continue
192 elif isinstance(default_value, property) or isdatadescriptor(default_value) \
193 or ismethoddescriptor(default_value):
194 # a property or a data or non-data descriptor > exclude
195 continue
196 elif (isinstance(default_value, type) or callable(default_value)) \
197 and getattr(default_value, '__name__', None) == member_name:
198 # a function/class defined in the class > exclude
199 continue
200 else:
201 # Create a field !!
202 need_to_check_type = check_types and (type_hint is not None)
203 if default_value is NO_DEFAULT:
204 # mandatory field
205 new_field = field(check_type=need_to_check_type)
206 else:
207 # optional field : copy the default value by default
208 try:
209 # autocheck: make sure that we will be able to create copies later
210 deepcopy(default_value)
211 except Exception as e:
212 raise ValueError("The provided default value for field %r=%r can not be deep-copied: "
213 "caught error %r" % (member_name, default_value, e))
214 new_field = field(check_type=need_to_check_type,
215 default_factory=copy_value(default_value, autocheck=False))
216
217 # Attach the newly created field to the class. Delete attr first so that order is preserved
218 # even if one of them had only an annotation.
219 try:
220 delattr(cls, member_name)
221 except AttributeError:
222 pass
223 setattr(cls, member_name, new_field)
224 new_field.set_as_cls_member(cls, member_name, type_hint=type_hint)
225
226 # Finally, make init if not already explicitly present
227 if make_init:
228 _make_init(cls)
229
230 return cls
231 # end of _autofields(cls)
232
233 # Main logic of autofield(**kwargs)
234 if check_types is not True and check_types is not False and isinstance(check_types, type):
235 # called without arguments @autofields: check_types is the decorated class
- S101 Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
- S101 Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
238 # use the parameter and use the correct check_types default value now
239 _cls = check_types
240 check_types = False # <-- important: variable is in the local context of _autofields
241 return _autofields(cls=_cls)
242 else:
243 # called with arguments @autofields(...): return the decorator
244 return _autofields
245
246
247 def is_dunder(name):
248 return len(name) >= 4 and name.startswith('__') and name.endswith('__')
249
250
251 def is_reserved_dunder(name):
252 return name in ('__doc__', '__name__', '__qualname__', '__module__', '__code__', '__globals__',
253 '__dict__', '__closure__', '__annotations__') # '__defaults__', '__kwdefaults__')
254
255
256 _dict, _hash = dict, hash
257 """Aliases for autoclass body"""
258
259
260 def autoclass(
261 # --- autofields
262 fields=True, # type: Union[bool, DecoratedClass]
263 typecheck=False, # type: bool
264 # --- constructor
265 init=True, # type: bool
266 # --- class methods
267 dict=True, # type: bool
268 dict_public_only=True, # type: bool
269 repr=True, # type: bool
270 repr_curly_mode=False, # type: bool
271 repr_public_only=True, # type: bool
272 eq=True, # type: bool
273 eq_public_only=False, # type: bool
274 hash=True, # type: bool
275 hash_public_only=False, # type: bool
276 # --- advanced
277 af_include_upper=False, # type: bool
278 af_include_dunder=False, # type: bool
279 af_exclude=DEFAULT_EXCLUDED, # type: Iterable[str]
280 ac_include=None, # type: Union[str, Tuple[str]]
281 ac_exclude=None, # type: Union[str, Tuple[str]]
282 ):
283 """
284 A decorator to automate many things at once for your class.
285
286 First if `fields=True` (default) it executes `@autofields` to generate fields from attribute defined at class
287 level.
288
289 - you can include attributes with dunder names or uppercase names with `af_include_dunder` and
290 `af_include_upper` respectively
291 - you can enable type checking on all fields at once by setting `check_types=True`
292 - the constructor is not generated at this stage
293
294 Then it generates methods for the class:
295
296 - if `init=True` (default) it generates the constructor based on all fields present, using `make_init()`.
297 - if `dict=True` (default) it generates `to_dict` and `from_dict` methods. Only public fields are represented in
298 `to_dict`, you can change this with `dict_public_only=False`.
299 - if `repr=True` (default) it generates a `__repr__` method. Only public fields are represented, you can change
300 this with `repr_public_only=False`.
301 - if `eq=True` (default) it generates an `__eq__` method, so that instances can be compared to other instances and
302 to dicts. All fields are compared by default, you can change this with `eq_public_only=True`.
303 - if `hash=True` (default) it generates an `__hash__` method, so that instances can be inserted in sets or dict
304 keys. All fields are hashed by default, you can change this with `hash_public_only=True`.
305
306 You can specify an explicit list of fields to include or exclude in the dict/repr/eq/hash methods with the
307 `ac_include` and `ac_exclude` parameters.
308
309 Note that this decorator is similar to the [autoclass library](https://smarie.github.io/python-autoclass/) but is
310 reimplemented here. In particular the parameter names and dictionary behaviour are different.
311
312 :param fields: boolean flag (default: True) indicating whether to create fields automatically. See `@autofields`
313 for details
314 :param typecheck: boolean flag (default: False) used when fields=True indicating the value of `check_type`
315 for created fields. Note that the type hint of each created field is copied from the type hint of the member it
316 originates from.
317 :param init: boolean flag (default: True) indicating whether a constructor should be created for the class if
318 no `__init__` method is already present. Such constructor will be created using `__init__ = make_init()`.
319 This is the same behaviour than `make_init` in `@autofields`. Note that this is *not* automatically disabled if
320 you set `fields=False`.
321 :param dict: a boolean to automatically create `cls.from_dict(dct)` and `obj.to_dict()` methods on the class
322 (default: True).
323 :param dict_public_only: a boolean (default: True) to indicate if only public fields should be
324 exposed in the dictionary view created by `to_dict` when `dict=True`.
325 :param repr: a boolean (default: True) to indicate if `__repr__` and `__str__` should be created for the class if
326 not explicitly present.
327 :param repr_curly_mode: a boolean (default: False) to turn on an alternate string representation when `repr=True`,
328 using curly braces.
329 :param repr_public_only: a boolean (default: True) to indicate if only public fields should be
330 exposed in the string representation when `repr=True`.
331 :param eq: a boolean (default: True) to indicate if `__eq__` should be created for the class if not explicitly
332 present.
333 :param eq_public_only: a boolean (default: False) to indicate if only public fields should be
334 compared in the equality method created when `eq=True`.
335 :param hash: a boolean (default: True) to indicate if `__hash__` should be created for the class if not explicitly
336 present.
337 :param hash_public_only: a boolean (default: False) to indicate if only public fields should be
338 hashed in the hash method created when `hash=True`.
339 :param af_include_upper: boolean flag (default: False) used when autofields=True indicating whether
340 upper-case class members should be also transformed to fields (usually such names are reserved for class
341 constants, not for fields).
342 :param af_include_dunder: boolean flag (default: False) used when autofields=True indicating whether
343 dunder-named class members should be also transformed to fields. Note that even if you set this to True,
344 members with reserved python dunder names will not be transformed. See `is_reserved_dunder` for the list of
345 reserved names.
346 :param af_exclude: a tuple of explicit attribute names to exclude from automatic fields creation. See
347 `@autofields(exclude=...)` for details.
348 :param ac_include: a tuple of explicit attribute names to include in dict/repr/eq/hash (None means all)
349 :param ac_exclude: a tuple of explicit attribute names to exclude in dict/repr/eq/hash. In such case,
350 include should be None.
351 :return:
352 """
353 if not fields and (af_include_dunder or af_include_upper or typecheck):
354 raise ValueError("Not able to set af_include_dunder or af_include_upper or typecheck when fields=False")
355
356 # switch between args and actual symbols for readability
357 dict_on = dict
358 dict = _dict
359 hash_on = hash
360 hash = _hash
361
362 # Create the decorator function
363 def _apply_decorator(cls):
364
365 # create fields automatically
366 if fields:
367 cls = autofields(check_types=typecheck, include_upper=af_include_upper,
368 exclude=af_exclude, include_dunder=af_include_dunder, make_init=False)(cls)
369
370 # make init if not already explicitly present
371 if init:
372 _make_init(cls)
373
374 # list all fields
375 all_pyfields = get_fields(cls)
376 if len(all_pyfields) == 0:
377 raise ValueError("No fields detected on class %s (including inherited ones)" % cls)
378
379 # filter selected
380 all_names = tuple(f.name for f in all_pyfields)
381 selected_names = filter_names(all_names, include=ac_include, exclude=ac_exclude, caller="@autoclass")
382 public_selected_names = tuple(n for n in selected_names if not n.startswith('_'))
383
384 # to/from dict
385 if dict_on:
386 dict_names = public_selected_names if dict_public_only else selected_names
387 if "to_dict" not in cls.__dict__:
388
389 def to_dict(self):
390 """ Generated by @pyfields.autoclass based on the class fields """
391 return {n: getattr(self, n) for n in dict_names}
392
393 cls.to_dict = to_dict
394 if "from_dict" not in cls.__dict__:
395
396 def from_dict(cls, dct):
397 """ Generated by @pyfields.autoclass """
398 return cls(**dct)
399
400 cls.from_dict = classmethod(from_dict)
401
402 # __str__ and __repr__
403 if repr:
404 repr_names = public_selected_names if repr_public_only else selected_names
405 if not repr_curly_mode: # default
406
407 def __repr__(self):
408 """ Generated by @pyfields.autoclass based on the class fields """
409 return '%s(%s)' % (self.__class__.__name__,
410 ', '.join('%s=%r' % (k, getattr(self, k)) for k in repr_names))
411 else:
412 def __repr__(self):
413 """ Generated by @pyfields.autoclass based on the class fields """
414 return '%s(**{%s})' % (self.__class__.__name__,
415 ', '.join('%r: %r' % (k, getattr(self, k)) for k in repr_names))
416
417 if "__repr__" not in cls.__dict__:
418 cls.__repr__ = __repr__
419 if "__str__" not in cls.__dict__:
420 cls.__str__ = __repr__
421
422 # __eq__
423 if eq:
424 eq_names = public_selected_names if eq_public_only else selected_names
425
426 def __eq__(self, other):
427 """ Generated by @pyfields.autoclass based on the class fields """
428 if isinstance(other, dict):
429 # comparison with dicts only when a to_dict method is available
430 try:
431 _self_to_dict = self.to_dict
432 except AttributeError:
433 return False
434 else:
435 return _self_to_dict() == other
436 elif isinstance(self, other.__class__):
437 # comparison with objects of the same class or a parent
438 try:
439 for att_name in eq_names:
440 if getattr(self, att_name) != getattr(other, att_name):
441 return False
442 except AttributeError:
443 return False
444 else:
445 return True
446 elif isinstance(other, self.__class__):
447 # other is a subtype: call method on other
448 return other.__eq__(self) # same as NotImplemented ?
449 else:
450 # classes are not related: False
451 return False
452
453 if "__eq__" not in cls.__dict__:
454 cls.__eq__ = __eq__
455
456 # __hash__
457 if hash_on:
458 hash_names = public_selected_names if hash_public_only else selected_names
459
460 def __hash__(self):
461 """ Generated by @autoclass. Implements the __hash__ method by hashing a tuple of field values """
462
463 # note: Should we prepend a unique hash for the class as `attrs` does ?
464 # return hash(tuple([type(self)] + [getattr(self, att_name) for att_name in added]))
465 # > No, it seems more intuitive to not do that.
466 # Warning: the consequence is that instances of subtypes will have the same hash has instance of their
467 # parent class if they have all the same attribute values
468
469 return hash(tuple(getattr(self, att_name) for att_name in hash_names))
470
471 if "__hash__" not in cls.__dict__:
472 cls.__hash__ = __hash__
473
474 return cls
475
476 # Apply: Decorator vs decorator factory logic
477 if isinstance(fields, type):
478 # called without parenthesis: directly apply decorator on first argument
479 cls = fields
480 fields = True # set it back to its default value
481 return _apply_decorator(cls)
482 else:
483 # called with parenthesis: return a decorator function
484 return _apply_decorator
485
486
487 def filter_names(all_names,
488 include=None, # type: Union[str, Tuple[str]]
489 exclude=None, # type: Union[str, Tuple[str]]
490 caller="" # type: str
491 ):
492 # type: (...) -> Iterable[str]
493 """
494 Common validator for include and exclude arguments
495
496 :param all_names:
497 :param include:
498 :param exclude:
499 :param caller:
500 :return:
501 """
502 if include is not None and exclude is not None:
503 raise ValueError("Only one of 'include' or 'exclude' argument should be provided.")
504
505 # check that include/exclude don't contain names that are incorrect
506 selected_names = all_names
507 if include is not None:
508 if exclude is not None:
509 raise ValueError('Only one of \'include\' or \'exclude\' argument should be provided.')
510
511 # get the selected names and check that all names in 'include' are actually valid names
512 included = (include,) if isinstance(include, str) else tuple(include)
513 incorrect = set(included) - set(all_names)
514 if len(incorrect) > 0:
515 raise ValueError("`%s` definition exception: `include` contains %r that is/are "
516 "not part of %r" % (caller, incorrect, all_names))
517 selected_names = included
518
519 elif exclude is not None:
520 excluded_set = {exclude} if isinstance(exclude, str) else set(exclude)
521 incorrect = excluded_set - set(all_names)
522 if len(incorrect) > 0:
523 raise ValueError("`%s` definition exception: exclude contains %r that is/are "
524 "not part of %r" % (caller, incorrect, all_names))
525 selected_names = tuple(n for n in all_names if n not in excluded_set)
526
527 return selected_names
528
529
530 # def method_already_there(cls,
531 # method_name, # type: str
532 # this_class_only=False # type: bool
533 # ):
534 # # type: (...) -> bool
535 # """
536 # Returns True if method `method_name` is already implemented by object_type, that is, its implementation differs
537 # from the one in `object`.
538 #
539 # :param cls:
540 # :param method_name:
541 # :param this_class_only:
542 # :return:
543 # """
544 # if this_class_only:
545 # return method_name in cls.__dict__ # or vars(cls)
546 # else:
547 # method = getattr(cls, method_name, None)
548 # return method is not None and method is not getattr(object, method_name, None)
549
550
551 def getownattr(cls, attrib_name):
552 """
553 Return the value of `cls.<attrib_name>` if it is defined in the class (and not inherited).
554 If the attribute is not present or is inherited, an `AttributeError` is raised.
555
556 >>> class A(object):
557 ... a = 1
558 >>>
559 >>> class B(A):
560 ... pass
561 >>>
562 >>> getownattr(A, 'a')
563 1
564 >>> getownattr(A, 'unknown')
565 Traceback (most recent call last):
566 ...
567 AttributeError: type object 'A' has no attribute 'unknown'
568 >>> getownattr(B, 'a')
569 Traceback (most recent call last):
570 ...
571 AttributeError: type object 'B' has no directly defined attribute 'a'
572
573 """
574 attr = getattr(cls, attrib_name)
575
576 for base_cls in cls.__mro__[1:]:
577 a = getattr(base_cls, attrib_name, None)
578 if attr is a:
579 raise AttributeError("type object %r has no directly defined attribute %r" % (cls.__name__, attrib_name))
580
581 return attr