mkdocs_gallery/backreferences.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/mkdocs-gallery>
3 #
4 # Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5 # License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6 """
7 Backreferences Generator
8 ========================
9
10 Parses example file code in order to keep track of used functions
11 """
12 from __future__ import print_function, unicode_literals
13
14 import ast
15 import codecs
16 import collections
17 import inspect
- F401 'os' imported but unused
19 import re
20 import warnings
21 from html import escape
22 from importlib import import_module
23 from typing import Set
24
25 from . import mkdocs_compatibility
26 from .errors import ExtensionError
27 from .gen_data_model import AllInformation, GalleryScriptResults
28 from .utils import _new_file, _replace_by_new_if_needed
29
30
31 class DummyClass(object):
32 """Dummy class for testing method resolution."""
33
34 def run(self):
35 """Do nothing."""
36 pass
37
38 @property
39 def prop(self):
40 """Property."""
41 return "Property"
42
43
44 class NameFinder(ast.NodeVisitor):
45 """Finds the longest form of variable names and their imports in code.
46
47 Only retains names from imported modules.
48 """
49
50 def __init__(self, global_variables=None):
51 super(NameFinder, self).__init__()
52 self.imported_names = {}
53 self.global_variables = global_variables or {}
54 self.accessed_names = set()
55
56 def visit_Import(self, node, prefix=""):
57 for alias in node.names:
58 local_name = alias.asname or alias.name
59 self.imported_names[local_name] = prefix + alias.name
60
61 def visit_ImportFrom(self, node):
62 self.visit_Import(node, node.module + ".")
63
64 def visit_Name(self, node):
65 self.accessed_names.add(node.id)
66
67 def visit_Attribute(self, node):
68 attrs = []
69 while isinstance(node, ast.Attribute):
70 attrs.append(node.attr)
71 node = node.value
72
73 if isinstance(node, ast.Name):
74 # This is a.b, not e.g. a().b
75 attrs.append(node.id)
76 self.accessed_names.add(".".join(reversed(attrs)))
77 else:
78 # need to get a in a().b
79 self.visit(node)
80
81 def get_mapping(self):
82 options = list()
83 for name in self.accessed_names:
84 local_name_split = name.split(".")
85 # first pass: by global variables and object inspection (preferred)
86 for split_level in range(len(local_name_split)):
87 local_name = ".".join(local_name_split[: split_level + 1])
88 remainder = name[len(local_name) :]
89 if local_name in self.global_variables:
90 obj = self.global_variables[local_name]
91 class_attr, method = False, []
92 if remainder:
93 for level in remainder[1:].split("."):
94 last_obj = obj
95 # determine if it's a property
96 prop = getattr(last_obj.__class__, level, None)
97 if isinstance(prop, property):
98 obj = last_obj
99 class_attr, method = True, [level]
100 break
101 try:
102 obj = getattr(obj, level)
103 except AttributeError:
104 break
105 if inspect.ismethod(obj):
106 obj = last_obj
107 class_attr, method = True, [level]
108 break
109 del remainder
110 is_class = inspect.isclass(obj)
111 if is_class or class_attr:
112 # Traverse all bases
113 classes = [obj if is_class else obj.__class__]
114 offset = 0
115 while offset < len(classes):
116 for base in classes[offset].__bases__:
117 # "object" as a base class is not very useful
118 if base not in classes and base is not object:
119 classes.append(base)
120 offset += 1
121 else:
122 classes = [obj.__class__]
123 for cc in classes:
124 module = inspect.getmodule(cc)
125 if module is not None:
126 module = module.__name__.split(".")
127 class_name = cc.__qualname__
128 # a.b.C.meth could be documented as a.C.meth,
129 # so go down the list
130 for depth in range(len(module), 0, -1):
131 full_name = ".".join(module[:depth] + [class_name] + method)
132 options.append((name, full_name, class_attr, is_class))
133 # second pass: by import (can't resolve as well without doing
134 # some actions like actually importing the modules, so use it
135 # as a last resort)
136 for split_level in range(len(local_name_split)):
137 local_name = ".".join(local_name_split[: split_level + 1])
138 remainder = name[len(local_name) :]
139 if local_name in self.imported_names:
140 full_name = self.imported_names[local_name] + remainder
141 is_class = class_attr = False # can't tell without import
142 options.append((name, full_name, class_attr, is_class))
143 return options
144
145
146 def _from_import(a, b):
147 # imp_line = 'from %s import %s' % (a, b)
148 # scope = dict()
149 # with warnings.catch_warnings(record=True): # swallow warnings
150 # warnings.simplefilter('ignore')
151 # exec(imp_line, scope, scope)
152 # return scope
153 with warnings.catch_warnings(record=True): # swallow warnings
154 warnings.simplefilter("ignore")
155 m = import_module(a)
156 obj = getattr(m, b)
157
158 return obj
159
160
161 def _get_short_module_name(module_name, obj_name):
162 """Get the shortest possible module name."""
163 if "." in obj_name:
164 obj_name, attr = obj_name.split(".")
165 else:
166 attr = None
167 # scope = {}
168 try:
169 # Find out what the real object is supposed to be.
170 imported_obj = _from_import(module_name, obj_name)
171 except Exception: # wrong object
172 return None
173 else:
174 real_obj = imported_obj
175 if attr is not None and not hasattr(real_obj, attr): # wrong class
176 return None # wrong object
177
178 parts = module_name.split(".")
179 short_name = module_name
180 for i in range(len(parts) - 1, 0, -1):
181 short_name = ".".join(parts[:i])
182 # scope = {}
183 try:
184 imported_obj = _from_import(short_name, obj_name)
185 # Ensure shortened object is the same as what we expect.
186 assert real_obj is imported_obj # noqa
187 except Exception: # libraries can throw all sorts of exceptions...
188 # get the last working module name
189 short_name = ".".join(parts[: (i + 1)])
190 break
191 return short_name
192
193
194 _regex = re.compile(r":(?:" r"func(?:tion)?|" r"meth(?:od)?|" r"attr(?:ibute)?|" r"obj(?:ect)?|" r"class):`~?(\S*)`")
195
196
197 def identify_names(script_blocks, global_variables=None, node=""):
198 """Build a codeobj summary by identifying and resolving used names."""
199
200 if node == "": # mostly convenience for testing functions
201 c = "\n".join(txt for kind, txt, _ in script_blocks if kind == "code")
202 node = ast.parse(c)
203
204 # Get matches from the code (AST)
205 finder = NameFinder(global_variables)
206 if node is not None:
207 finder.visit(node)
208 names = list(finder.get_mapping())
209
210 # Get matches from docstring inspection
211 text = "\n".join(txt for kind, txt, _ in script_blocks if kind == "text")
212 names.extend((x, x, False, False) for x in re.findall(_regex, text))
213 example_code_obj = collections.OrderedDict() # order is important
214
215 # Make a list of all guesses, in `_embed_code_links` we will break when we find a match
216 for name, full_name, class_like, is_class in names:
217 if name not in example_code_obj:
218 example_code_obj[name] = list()
219
220 # name is as written in file (e.g. np.asarray)
221 # full_name includes resolved import path (e.g. numpy.asarray)
222 splitted = full_name.rsplit(".", 1 + class_like)
223 if len(splitted) == 1:
224 splitted = ("builtins", splitted[0])
225 elif len(splitted) == 3: # class-like
226 assert class_like # noqa
227 splitted = (splitted[0], ".".join(splitted[1:]))
228 else:
229 assert not class_like # noqa
230
231 module, attribute = splitted
232
233 # get shortened module name
234 module_short = _get_short_module_name(module, attribute)
235 cobj = {
236 "name": attribute,
237 "module": module,
238 "module_short": module_short or module,
239 "is_class": is_class,
240 }
241
242 example_code_obj[name].append(cobj)
243
244 return example_code_obj
245
246
247 # TODO only:: html ?
248 THUMBNAIL_TEMPLATE = """
249 <div class="mkd-glr-thumbcontainer" tooltip="{snippet}">
250 <!--div class="figure align-default" id="id1"-->
251 <img alt="{title}" src="{thumbnail}" />
252 <p class="caption">
253 <span class="caption-text">
254 <a class="reference internal" href="{example_html}">
255 <span class="std std-ref">{title}</span>
256 </a>
257 </span>
258 <!--a class="headerlink" href="#id1" title="Permalink to this image"></a-->
259 </p>
260 <!--/div-->
261 </div>
262 """
263
264 # TODO something specific here ?
265 BACKREF_THUMBNAIL_TEMPLATE = THUMBNAIL_TEMPLATE
266 # + """
267 # .. only:: not html
268 #
269 # * :ref:`mkd_glr_{ref_name}`
270 # """
271
272
273 def _thumbnail_div(script_results: GalleryScriptResults, is_backref: bool = False, check: bool = True):
274 """
275 Generate MD to place a thumbnail in a gallery.
276
277 Parameters
278 ----------
279 script_results : GalleryScriptResults
280 The results from processing a gallery example
281
282 is_backref : bool
283 ?
284
285 check : bool
286 ?
287
288 Returns
289 -------
290 md : str
291 The markdown to integrate in the global gallery readme. Note that this is also the case for subsections.
292 """
293 # Absolute path to the thumbnail
294 if check and not script_results.thumb.exists():
295 # This means we have done something wrong in creating our thumbnail!
296 raise ExtensionError(f"Could not find internal mkdocs-gallery thumbnail file:\n{script_results.thumb}")
297
298 # Relative path to the thumbnail (relative to the gallery, not the subsection)
299 thumb = script_results.thumb_rel_root_gallery
300
301 # Relative path to the html tutorial that will be generated from the md
302 example_html = script_results.script.md_file_rel_root_gallery.with_suffix("")
303
304 template = BACKREF_THUMBNAIL_TEMPLATE if is_backref else THUMBNAIL_TEMPLATE
305 return template.format(
306 snippet=escape(script_results.intro),
307 thumbnail=thumb,
308 title=script_results.script.title,
309 example_html=example_html,
310 )
311
312
313 def _write_backreferences(backrefs: Set, seen_backrefs: Set, script_results: GalleryScriptResults):
314 """
315 Write backreference file including a thumbnail list of examples.
316
317 Parameters
318 ----------
319 backrefs : set
320
321 seen_backrefs : set
322
323 script_results : GalleryScriptResults
324
325 Returns
326 -------
327
328 """
329 all_info = script_results.script.gallery.all_info
330
331 for backref in backrefs:
332 # Get the backref file to use for this module, according to config
333 include_path = _new_file(all_info.get_backreferences_file(backref))
334
335 # Create new or append to existing file
336 seen = backref in seen_backrefs
337 with codecs.open(str(include_path), "a" if seen else "w", encoding="utf-8") as ex_file:
338 # If first ref: write header
339 if not seen:
340 # Be aware that if the number of lines of this heading changes,
341 # the minigallery directive should be modified accordingly
342 heading = "Examples using ``%s``" % backref
343 ex_file.write("\n\n" + heading + "\n")
344 ex_file.write("^" * len(heading) + "\n")
345
346 # Write the thumbnail
347 ex_file.write(_thumbnail_div(script_results, is_backref=True))
348 seen_backrefs.add(backref)
349
350
351 def _finalize_backreferences(seen_backrefs, all_info: AllInformation):
352 """Replace backref files only if necessary."""
353 logger = mkdocs_compatibility.getLogger("mkdocs-gallery")
354 if all_info.gallery_conf["backreferences_dir"] is None:
355 return
356
357 for backref in seen_backrefs:
358 # Get the backref file to use for this module, according to config
359 path = _new_file(all_info.get_backreferences_file(backref))
360 if path.exists():
361 # Simply drop the .new suffix
362 _replace_by_new_if_needed(path, md5_mode="t")
363 else:
364 # No file: warn
365 level = all_info.gallery_conf["log_level"].get("backreference_missing", "warning")
366 func = getattr(logger, level)
367 func("Could not find backreferences file: %s" % (path,))
368 func("The backreferences are likely to be erroneous " "due to file system case insensitivity.")