Coverage for yamlable/main.py: 86%
139 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-07-06 08:57 +0000
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/python-yamlable>
3#
4# License: 3-clause BSD, <https://github.com/smarie/python-yamlable/blob/master/LICENSE>
6try:
7 from collections.abc import Mapping
8except ImportError:
9 from collections import Mapping
11from abc import abstractmethod, ABCMeta
13import six
15try: # python 3.5+
16 from typing import TypeVar, Callable, Iterable, Any, Tuple, Dict, Set, List, Sequence
18 YA = TypeVar('YA', bound='YamlAble')
19 T = TypeVar('T')
20except ImportError:
21 pass
23try: # python 3.5.4+
24 from typing import Type
25except ImportError:
26 pass # normal for old versions of typing
28from yaml import Loader, SafeLoader, Dumper, SafeDumper, MappingNode, ScalarNode, SequenceNode
30from yamlable.base import AbstractYamlObject, read_yaml_node_as_yamlobject, read_yaml_node_as_dict, \
31 read_yaml_node_as_sequence, read_yaml_node_as_scalar
32from yamlable.yaml_objects import YamlObject2
35YAMLABLE_PREFIX = '!yamlable/'
38class AbstractYamlAble(AbstractYamlObject):
39 """
40 The abstract part of YamlAble. It might be useful to inherit if you want to create a super class for several
41 classes, with the same YamlAble behaviour.
42 """
44 @classmethod
45 @abstractmethod
46 def is_yaml_tag_supported(cls,
47 yaml_tag # type: str
48 ):
49 # type: (...) -> bool
50 """
51 Implementing classes should return True if they are able to decode yaml objects with this tag.
52 Note that the associated yaml object tag will be
54 !yamlable/<yaml_tag>
56 :param yaml_tag:
57 :return:
58 """
61class YamlAble(AbstractYamlAble):
62 """
63 A helper class to register a class as able to dump instances to yaml and to load them back from yaml.
65 This class does not rely on the `YAMLObject` class provided in pyyaml, so it provides a bit more flexibility (no
66 metaclass magic).
68 The behaviour is very similar though:
69 - inherit from `YamlAble` or virtually inherit from it using YamlAble.register(cls)
70 - fill the `__yaml_tag_suffix__` either directly or using the `@yaml_info()` decorator
71 - optionally override `__from_yaml_dict__` (class method called during decoding) and/or `__to_yaml_dict__`
72 (instance method called during encoding) if you wish to have control on the process, for example to only dump part
73 of the attributes or perform some custom instance creation. Note that default implementation relies on `vars(self)`
74 for dumping and on `cls(**dct)` for loading.
75 """
76 __yaml_tag_suffix__ = None
77 """ placeholder for a class-wide yaml tag. It will be prefixed with '!yamlable/', stored in `YAMLABLE_PREFIX` """
79 @classmethod
80 def is_yaml_tag_supported(cls,
81 yaml_tag # type: str
82 ):
83 # type: (...) -> bool
84 """
85 Implementing classes should return True if they are able to decode yaml objects with this yaml tag.
86 Default implementation relies on class attribute `__yaml_tag_suffix__` if provided, either manually or through
87 the `@yaml_info` decorator.
89 :param yaml_tag:
90 :return:
91 """
92 if hasattr(cls, '__yaml_tag_suffix__') and cls.__yaml_tag_suffix__ is not None:
93 if '__yaml_tag_suffix__' in cls.__dict__: 93 ↛ 98line 93 didn't jump to line 98, because the condition on line 93 was never false
94 # this is an explicitly configured class (__yaml_tag_suffix__ is set on it), ok
95 return cls.__yaml_tag_suffix__ == yaml_tag
96 else:
97 # this class inherits from the __yaml_tag_suffix__ and does not redefine it, not ok
98 raise TypeError("`__yaml_tag_suffix__` field is not redefined by class {}, cannot inherit from YamlAble"
99 "properly.".format(cls))
101 else:
102 raise NotImplementedError("class {} does not seem to have a non-None '__yaml_tag_suffix__' field. You can "
103 "either create one manually or by decorating your class with @yaml_info. "
104 "Alternately you should override the 'is_yaml_tag_supported' method "
105 "from YamlAble.".format(cls))
108def yaml_info(yaml_tag=None, # type: str
109 yaml_tag_ns=None # type: str
110 ):
111 # type: (...) -> Callable[[Type[YA]], Type[YA]]
112 """
113 A simple class decorator to tag a class with a global yaml tag - that way you do not have to call `YamlAble` super
114 constructor.
116 You can either provide a full yaml tag suffix:
118 ```python
119 @yaml_info("com.example.MyFoo")
120 class Foo(YamlAble):
121 pass
123 print(Foo.__yaml_tag_suffix__) # yields "com.example.MyFoo"
124 ```
126 or simply provide a namespace, that will be appended with '.<class name>' :
128 ```python
129 @yaml_info(yaml_tag_ns="com.example")
130 class Foo(YamlAble):
131 pass
133 print(MyFoo.__yaml_tag_suffix__) # yields "com.example.Foo"
134 ```
136 In both cases, the suffix is appended at the end of the common yamlable prefix:
138 ```python
139 print(Foo().dumps_yaml()) # yields "!yamlable/com.example.Foo {}"
140 ```
142 :param yaml_tag: the complete yaml suffix.
143 :param yaml_tag_ns: the yaml namespace. It will be appended with '.<cls.__name__>'
144 :return:
145 """
146 def f(cls # type: Type[YA]
147 ):
148 # type: (...) -> Type[YA]
149 return yaml_info_decorate(cls, yaml_tag=yaml_tag, yaml_tag_ns=yaml_tag_ns)
150 return f
153def yaml_info_decorate(cls, # type: Type[YA]
154 yaml_tag=None, # type: str
155 yaml_tag_ns=None # type: str
156 ):
157 # type: (...) -> Type[YA]
158 """
159 A simple class decorator to tag a class with a global yaml tag - that way you do not have to call `YamlAble` super
160 constructor.
162 You can either provide a full yaml tag suffix:
164 ```python
165 @yaml_info("com.example.MyFoo")
166 class Foo(YamlAble):
167 pass
169 print(Foo.__yaml_tag_suffix__) # yields "com.example.MyFoo"
170 ```
172 or simply provide a namespace, that will be appended with '.<class name>' :
174 ```python
175 @yaml_info(yaml_tag_ns="com.example")
176 class Foo(YamlAble):
177 pass
179 print(MyFoo.__yaml_tag_suffix__) # yields "com.example.Foo"
180 ```
182 In both cases, the suffix is appended at the end of the common yamlable prefix:
184 ```python
185 print(Foo().dumps_yaml()) # yields "!yamlable/com.example.Foo {}"
186 ```
188 :param cls:
189 :param yaml_tag: the complete yaml suffix.
190 :param yaml_tag_ns: the yaml namespace. It will be appended with '.<cls.__name__>'
191 :return:
192 """
193 if yaml_tag_ns is not None:
194 if yaml_tag is not None: 194 ↛ 195line 194 didn't jump to line 195, because the condition on line 194 was never true
195 raise ValueError("Only one of 'yaml_tag' and 'yaml_tag_ns' should be provided")
197 # create yaml_tag by appending the class name to the namespace
198 yaml_tag = yaml_tag_ns + '.' + cls.__name__
200 elif yaml_tag is None: 200 ↛ 201line 200 didn't jump to line 201, because the condition on line 200 was never true
201 raise ValueError("One non-None `yaml_tag` or `yaml_tag_ns` must be provided.")
203 if issubclass(cls, YamlObject2): 203 ↛ 207line 203 didn't jump to line 207, because the condition on line 203 was never true
204 # Do not support this, because the `YamlObject` metaclass needs the tag to be present BEFORE this decorator is
205 # even called. So if we are here, it means that we are trying to override a yaml_tag that was already registered
206 # with pyyaml. Too late!
207 raise TypeError("This is not supported")
208 # if not yaml_tag.startswith('!'):
209 # raise ValueError("When extending YamlObject2, the `yaml_tag` field should contain the full yaml tag, "
210 # "and should therefore start with !")
211 # cls.yaml_tag = yaml_tag
213 elif issubclass(cls, YamlAble): 213 ↛ 219line 213 didn't jump to line 219, because the condition on line 213 was never false
214 if yaml_tag.startswith('!'): 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true
215 raise ValueError("When extending YamlAble, the `yaml_tag` field should only contain the yaml tag suffix, "
216 "and should therefore NOT start with !")
217 cls.__yaml_tag_suffix__ = yaml_tag # type: ignore
218 else:
219 raise TypeError("classes tagged with @yaml_info should be subclasses of YamlAble or YamlObject2")
221 return cls
224# --------------------------------------Codecs-----------------------------------------------------------
225def decode_yamlable(loader,
226 yaml_tag, # type: str
227 node, # type: MappingNode
228 **kwargs):
229 # type: (...) -> YamlAble
230 """
231 The method used to decode YamlAble object instances
233 :param loader:
234 :param yaml_tag:
235 :param node:
236 :param kwargs:
237 :return:
238 """
239 candidates = _get_all_subclasses(YamlAble)
240 errors = dict()
241 for clazz in candidates:
242 try:
243 if clazz.is_yaml_tag_supported(yaml_tag):
244 return read_yaml_node_as_yamlobject(
245 cls=clazz, loader=loader, node=node, yaml_tag=yaml_tag
246 ) # type: ignore
247 else:
248 errors[clazz.__name__] = "yaml tag %r is not supported." % yaml_tag
249 except Exception as e:
250 errors[clazz.__name__] = e
252 raise TypeError("No YamlAble subclass found able to decode object '!yamlable/" + yaml_tag + "'. Tried classes: "
253 + str(candidates) + ". Caught errors: " + str(errors) + ". "
254 "Please check the value of <cls>.__yaml_tag_suffix__ on these classes. Note that this value may be "
255 "set using @yaml_info() so help(yaml_info) might help too.")
258def encode_yamlable(dumper,
259 obj, # type: YamlAble
260 without_custom_tag=False, # type: bool
261 **kwargs):
262 # type: (...) -> MappingNode
263 """
264 The method used to encode YamlAble object instances
266 :param dumper:
267 :param obj:
268 :param without_custom_tag: if this is set to True, the yaml tag !yamlable/<yaml_tag_suffix> will not be written to
269 the document. Warning: if you do so, decoding the object will not be easy!
270 :param kwargs:
271 :return:
272 """
273 # Convert objects to a dictionary of their representation
274 new_data = obj.__to_yaml_dict__()
276 if without_custom_tag: 276 ↛ 278line 276 didn't jump to line 278, because the condition on line 276 was never true
277 # TODO check that it works
278 return dumper.represent_mapping(None, new_data, flow_style=None)
279 else:
280 # Add the tag information
281 if not hasattr(obj, '__yaml_tag_suffix__') or obj.__yaml_tag_suffix__ is None:
282 raise NotImplementedError("object {} does not seem to have a non-None '__yaml_tag_suffix__' field. You "
283 "can either create one manually or by decorating your class with @yaml_info."
284 "".format(obj))
285 yaml_tag = YAMLABLE_PREFIX + obj.__yaml_tag_suffix__
286 return dumper.represent_mapping(yaml_tag, new_data, flow_style=None)
289try: # PyYaml 5.1+
290 from yaml import FullLoader
291 ALL_PYYAML_LOADERS = (Loader, SafeLoader, FullLoader)
292except ImportError:
293 ALL_PYYAML_LOADERS = (Loader, SafeLoader) # type: ignore
296ALL_PYYAML_DUMPERS = (Dumper, SafeDumper)
299def register_yamlable_codec(loaders=ALL_PYYAML_LOADERS, dumpers=ALL_PYYAML_DUMPERS):
300 # type: (...) -> None
301 """
302 Registers the yamlable encoder and decoder with all pyYaml loaders and dumpers.
304 :param loaders:
305 :param dumpers:
306 :return:
307 """
308 for loader in loaders:
309 loader.add_multi_constructor(YAMLABLE_PREFIX, decode_yamlable)
311 for dumper in dumpers:
312 dumper.add_multi_representer(YamlAble, encode_yamlable)
315# Register the YamlAble encoding and decoding functions
316register_yamlable_codec()
319def _get_all_subclasses(typ, # type: Type[T]
320 recursive=True, # type: bool
321 _memo=None # type: Set[Type[Any]]
322 ):
323 # type: (...) -> Iterable[Type[T]]
324 """
325 Returns all subclasses of `typ`
326 Warning this does not support generic types.
327 See parsyfiles.get_all_subclasses() if one day generic types are needed (commented lines below)
329 :param typ:
330 :param recursive: a boolean indicating whether recursion is needed
331 :param _memo: internal variable used in recursion to avoid exploring subclasses that were already explored
332 :return:
333 """
334 _memo = _memo or set()
336 # if we have collected the subclasses for this already, return
337 if typ in _memo: 337 ↛ 338line 337 didn't jump to line 338, because the condition on line 337 was never true
338 return []
340 # else remember that we have collected them, and collect them
341 _memo.add(typ)
342 # if is_generic_type(typ):
343 # # We now use get_origin() to also find all the concrete subclasses in case the desired type is a generic
344 # sub_list = get_origin(typ).__subclasses__()
345 # else:
346 sub_list = typ.__subclasses__()
348 # recurse
349 result = [] # type: List[Type[T]]
350 for t in sub_list:
351 # only keep the origins in the list
352 # to = get_origin(t) or t
353 to = t
354 # noinspection PyBroadException
355 try:
356 if to is not typ and to not in result and issubclass(to, typ): # is_subtype(to, typ, bound_typevars={}): 356 ↛ 350line 356 didn't jump to line 350, because the condition on line 356 was never false
357 result.append(to)
358 except Exception: # noqa
359 # catching an error with is_subtype(Dict, Dict[str, int], bound_typevars={})
360 pass
362 # recurse
363 if recursive: 363 ↛ 371line 363 didn't jump to line 371, because the condition on line 363 was never false
364 for typpp in sub_list:
365 for t in _get_all_subclasses(typpp, recursive=True, _memo=_memo):
366 # unfortunately we have to check 't not in sub_list' because with generics strange things happen
367 # also is_subtype returns false when the parent is a generic
368 if t not in sub_list and issubclass(t, typ): # is_subtype(t, typ, bound_typevars={}): 368 ↛ 365line 368 didn't jump to line 365, because the condition on line 368 was never false
369 result.append(t)
371 return result
374# ------------------------ Easy codecs ---------------
375class YamlCodec(six.with_metaclass(ABCMeta, object)):
376 """
377 Represents a codec class, able to encode several object types into/from yaml, with potentially different yaml tag
378 ids. It assumes that the objects are written as yaml dictionaries, and that they all have the same yaml tag prefix
380 for example !mycodec/<yaml_tag_suffix>, where 'mycodec' is the yaml prefix associated with this codec.
382 This allows the code to be pre-wired so that it is very easy to implement.
383 - Decoding:
384 - fill get_yaml_prefix
385 - fill is_yaml_tag_supported to declare if a given yaml tag is supported or not
386 - fill from_yaml_dict to create new instances of objects from a dictionary, according to the yaml tag
387 - Encoding:
388 - fill get_known_types
389 - fill the to_yaml_dict
390 """
392 # -------------- decoding
394 @classmethod
395 @abstractmethod
396 def get_yaml_prefix(cls):
397 # type: (...) -> str
398 """
399 Implementors should return the yaml prefix associated tto this codec.
400 :return:
401 """
403 @classmethod
404 def decode(cls, loader,
405 yaml_tag_suffix, # type: str
406 node, # type: MappingNode
407 **kwargs):
408 # type: (...) -> Any
409 """
410 The method used to decode object instances
412 :param loader:
413 :param yaml_tag_suffix:
414 :param node:
415 :param kwargs: keyword arguments coming from pyyaml, not sure what you will find here.
416 :return:
417 """
418 if cls.is_yaml_tag_supported(yaml_tag_suffix): 418 ↛ exitline 418 didn't return from function 'decode', because the condition on line 418 was never false
419 # Note: same as in read_yaml_node_as_yamlobject but different yaml tag handling so code copy
421 if isinstance(node, ScalarNode):
422 constructor_args = read_yaml_node_as_scalar(loader, node)
423 return cls.from_yaml_scalar(yaml_tag_suffix, constructor_args, **kwargs) # type: ignore
425 elif isinstance(node, SequenceNode):
426 constructor_args = read_yaml_node_as_sequence(loader, node)
427 return cls.from_yaml_list(yaml_tag_suffix, constructor_args, **kwargs) # type: ignore
429 elif isinstance(node, MappingNode): 429 ↛ 434line 429 didn't jump to line 434, because the condition on line 429 was never false
430 constructor_args = read_yaml_node_as_dict(loader, node)
431 return cls.from_yaml_dict(yaml_tag_suffix, constructor_args, **kwargs) # type: ignore
433 else:
434 raise TypeError("Unknown type of yaml node: %r. Please report this to `yamlable` project." % type(node))
436 @classmethod
437 @abstractmethod
438 def is_yaml_tag_supported(cls,
439 yaml_tag_suffix # type: str
440 ):
441 # type: (...) -> bool
442 """
443 Implementing classes should return True if they are able to decode yaml objects with this yaml tag.
445 :param yaml_tag_suffix:
446 :return:
447 """
449 @classmethod
450 def from_yaml_scalar(cls,
451 yaml_tag_suffix, # type: str
452 scalar, # type: Any
453 **kwargs):
454 # type: (...) -> Any
455 """
456 Implementing classes should create an object corresponding to the given yaml tag, using the given YAML scalar.
458 :param scalar:
459 :param yaml_tag_suffix:
460 :param kwargs: keyword arguments coming from pyyaml, not sure what you will find here.
461 :return:
462 """
463 raise NotImplementedError("This codec does not support loading objects from scalar. Please override "
464 "`from_yaml_scalar` to support this feature.")
466 @classmethod
467 def from_yaml_list(cls,
468 yaml_tag_suffix, # type: str
469 seq, # type: Sequence[Any]
470 **kwargs):
471 # type: (...) -> Any
472 """
473 Implementing classes should create an object corresponding to the given yaml tag, using the given YAML sequence.
475 :param seq:
476 :param yaml_tag_suffix:
477 :param kwargs: keyword arguments coming from pyyaml, not sure what you will find here.
478 :return:
479 """
480 raise NotImplementedError("This codec does not support loading objects from sequence. Please override "
481 "`from_yaml_list` to support this feature.")
483 @classmethod
484 @abstractmethod
485 def from_yaml_dict(cls,
486 yaml_tag_suffix, # type: str
487 dct, # type: Dict[str, Any]
488 **kwargs):
489 # type: (...) -> Any
490 """
491 Implementing classes should create an object corresponding to the given yaml tag, using the given YAML mapping.
493 :param dct:
494 :param yaml_tag_suffix:
495 :param kwargs: keyword arguments coming from pyyaml, not sure what you will find here.
496 :return:
497 """
499 # --------------- encoding
501 @classmethod
502 @abstractmethod
503 def get_known_types(cls):
504 # type: (...) -> Iterable[Type[Any]]
505 """
506 Implementing classes should return an iterable of known object types.
507 :return:
508 """
510 @classmethod
511 def encode(cls, dumper, obj,
512 without_custom_tag=False, # type: bool
513 **kwargs):
514 # type: (...) -> MappingNode
515 """
516 The method used to encode YamlAble object instances
518 :param dumper:
519 :param obj:
520 :param without_custom_tag: if this is set to True, the yaml tag !yamlable/<yaml_tag_suffix> will not be written
521 to the document. Warning: if you do so, decoding the object will not be easy!
522 :param kwargs: keyword arguments coming from pyyaml, not sure what you will find here.
523 :return:
524 """
525 # Convert objects to a dictionary of their representation
526 yaml_tag_suffix, obj_as_dict = cls.to_yaml_dict(obj)
527 if not isinstance(obj_as_dict, Mapping) or not isinstance(yaml_tag_suffix, str): 527 ↛ 528line 527 didn't jump to line 528, because the condition on line 527 was never true
528 raise TypeError("`to_yaml_dict` did not return correct results. It should return a tuple of "
529 "`yaml_tag_suffix, obj_as_dict`")
531 if without_custom_tag: 531 ↛ 533line 531 didn't jump to line 533, because the condition on line 531 was never true
532 # TODO check that it works
533 return dumper.represent_mapping(None, obj_as_dict, flow_style=None)
534 else:
535 # Add the tag information
536 prefix = cls.get_yaml_prefix()
537 if len(prefix) == 0 or prefix[-1] != '/': 537 ↛ 538line 537 didn't jump to line 538, because the condition on line 537 was never true
538 prefix = prefix + '/'
539 yaml_tag = prefix + yaml_tag_suffix
540 return dumper.represent_mapping(yaml_tag, obj_as_dict, flow_style=None)
542 @classmethod
543 @abstractmethod
544 def to_yaml_dict(cls, obj):
545 # type: (...) -> Tuple[str, Dict[str, Any]]
546 """
547 Implementors should encode the given object as a dictionary and also return the yaml tag that should be used to
548 ensure correct decoding.
550 :param obj:
551 :return: a tuple where the first element is the yaml tag suffix, and the second is the dictionary representing
552 the object
553 """
555 @classmethod
556 def register_with_pyyaml(cls, loaders=ALL_PYYAML_LOADERS, dumpers=ALL_PYYAML_DUMPERS):
557 # type: (...) -> None
558 """
559 Registers this codec with PyYaml, on the provided loaders and dumpers (default: all PyYaml loaders and dumpers).
560 - The encoding part is registered for the object types listed in cls.get_known_types(), in order
561 - The decoding part is registered for the yaml prefix in cls.get_yaml_prefix()
563 :param loaders: the PyYaml loaders to register this codec with. By default all pyyaml loaders are considered
564 (Loader, SafeLoader...)
565 :param dumpers: the PyYaml dumpers to register this codec with. By default all pyyaml loaders are considered
566 (Dumper, SafeDumper...)
567 :return:
568 """
569 for loader in loaders:
570 loader.add_multi_constructor(cls.get_yaml_prefix(), cls.decode)
572 for dumper in dumpers:
573 for t in cls.get_known_types():
574 dumper.add_multi_representer(t, cls.encode)