Coverage for src/mkdocs_gallery/gen_single.py: 85%
521 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-30 08:26 +0000
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/mkdocs-gallery>
3#
4# Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5# License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6"""
7Generator for a single script example in a gallery.
8"""
10from __future__ import absolute_import, division, print_function
12import ast
13import codeop
14import contextlib
15import copy
16import gc
17import importlib
18import os
19import pickle
20import re
21import subprocess
22import sys
23import traceback
24import warnings
25from copy import deepcopy
26from functools import partial
27from io import StringIO
28from pathlib import Path
29from shutil import copyfile
30from textwrap import indent, dedent
31from time import time
32from typing import List, Set, Tuple
34from tqdm import tqdm
36from . import glr_path_static, mkdocs_compatibility
37from .backreferences import _thumbnail_div, _write_backreferences, identify_names
38from .binder import check_binder_conf, gen_binder_md
39from .errors import ExtensionError
40from .gen_data_model import GalleryBase, GalleryScript, GalleryScriptResults
41from .notebook import jupyter_notebook, save_notebook
42from .py_source_parser import remove_config_comments, split_code_and_text_blocks
43from .scrapers import ImageNotFoundError, _find_image_ext, clean_modules, save_figures
44from .utils import _new_file, _replace_by_new_if_needed, optipng, rescale_image, run_async
46logger = mkdocs_compatibility.getLogger("mkdocs-gallery")
49###############################################################################
52class _LoggingTee(object):
53 """A tee object to redirect streams to the logger."""
55 def __init__(self, src_filename: Path):
56 self.logger = logger
57 self.src_filename = src_filename
58 self.logger_buffer = ""
59 self.set_std_and_reset_position()
61 def set_std_and_reset_position(self):
62 if not isinstance(sys.stdout, _LoggingTee):
63 self.origs = (sys.stdout, sys.stderr)
64 sys.stdout = sys.stderr = self
65 self.first_write = True
66 self.output = StringIO()
67 return self
69 def restore_std(self):
70 sys.stdout.flush()
71 sys.stderr.flush()
72 sys.stdout, sys.stderr = self.origs
74 def write(self, data):
75 self.output.write(data)
77 if self.first_write:
78 self.logger.verbose("Output from %s", self.src_filename) # color='brown')
79 self.first_write = False
81 data = self.logger_buffer + data
82 lines = data.splitlines()
83 if data and data[-1] not in "\r\n":
84 # Wait to write last line if it's incomplete. It will write next
85 # time or when the LoggingTee is flushed.
86 self.logger_buffer = lines[-1]
87 lines = lines[:-1]
88 else:
89 self.logger_buffer = ""
91 for line in lines:
92 self.logger.verbose("%s", line)
94 def flush(self):
95 self.output.flush()
96 if self.logger_buffer:
97 self.logger.verbose("%s", self.logger_buffer)
98 self.logger_buffer = ""
100 # When called from a local terminal seaborn needs it in Python3
101 def isatty(self):
102 return self.output.isatty()
104 # When called in gen_single, conveniently use context managing
105 def __enter__(self):
106 return self
108 def __exit__(self, type_, value, tb):
109 self.restore_std()
112###############################################################################
113# The following strings are used when we have several pictures: we use
114# an html div tag that our CSS uses to turn the lists into horizontal
115# lists.
116HLIST_HEADER = """
117{: .mkd-glr-horizontal }
119"""
121HLIST_IMAGE_TEMPLATE = """
122 *
124 .. image:: /%s
125 {: .mkd-glr-multi-img }
126"""
128SINGLE_IMAGE = """
129.. image:: /%s
130 {: .mkd-glr-single-img }
131"""
133# Note: since this seems to be a one-liner, we use inline code. TODO check
134CODE_OUTPUT = """Out:
135{{: .mkd-glr-script-out }}
137```{{.shell .mkd-glr-script-out-disp }}
138{0}
139```
140\n"""
142TIMING_CONTENT = """
143**Total running time of the script:** ({0: .0f} minutes {1: .3f} seconds)
144""" # Strange enough: this CSS class does not actually exist in sphinx-gallery {{: .mkd-glr-timing }}
146# TODO only if html ? .. only:: html
147MKD_GLR_SIG = """\n
148[Gallery generated by mkdocs-gallery](https://smarie.github.io/mkdocs-gallery){: .mkd-glr-signature }
149"""
151# Header used to include raw html from data _repr_html_
152HTML_HEADER = """<div class="output_subarea output_html rendered_html output_result">
153{0}
154</div>
155"""
158def codestr2md(codestr, lang: str = "python", lineno=None, is_exc: bool = False):
159 """Return markdown code block from code string."""
161 # if lineno is not None:
162 # # Sphinx only starts numbering from the first non-empty line.
163 # blank_lines = codestr.count('\n', 0, -len(codestr.lstrip()))
164 # lineno = ' :lineno-start: {0}\n'.format(lineno + blank_lines)
165 # else:
166 # lineno = ''
167 # code_directive = ".. code-block:: {0}\n{1}\n".format(lang, lineno)
168 # indented_block = indent(codestr, ' ' * 4)
169 # return code_directive + indented_block
170 style = " .mkd-glr-script-err-disp" if is_exc else ""
171 if lineno is not None: 171 ↛ 174line 171 didn't jump to line 174 because the condition on line 171 was never true
172 # Sphinx only starts numbering from the first non-empty line. TODO do we need this too ?
173 # blank_lines = codestr.count('\n', 0, -len(codestr.lstrip()))
174 return f'```{{.{lang} {style} linenums="{lineno}"}}\n{codestr}```\n'
175 else:
176 return f"```{{.{lang} {style}}}\n{codestr}```\n"
179def _regroup(x):
180 x = x.groups()
181 return x[0] + x[1].split(".")[-1] + x[2]
184def _sanitize_md(string):
185 """Use regex to remove at least some sphinx directives.
187 TODO is this still needed ?
188 """
189 # :class:`a.b.c <thing here>`, :ref:`abc <thing here>` --> thing here
190 p, e = r"(\s|^):[^:\s]+:`", r"`(\W|$)"
191 string = re.sub(p + r"\S+\s*<([^>`]+)>" + e, r"\1\2\3", string)
192 # :class:`~a.b.c` --> c
193 string = re.sub(p + r"~([^`]+)" + e, _regroup, string)
194 # :class:`a.b.c` --> a.b.c
195 string = re.sub(p + r"([^`]+)" + e, r"\1\2\3", string)
197 # ``whatever thing`` --> whatever thing
198 p = r"(\s|^)`"
199 string = re.sub(p + r"`([^`]+)`" + e, r"\1\2\3", string)
200 # `whatever thing` --> whatever thing
201 string = re.sub(p + r"([^`]+)" + e, r"\1\2\3", string)
202 return string
205# Find RST/Markdown title chars,
206# i.e. lines that consist of (3 or more of the same) 7-bit non-ASCII chars.
207# This conditional is not perfect but should hopefully be good enough.
208RE_3_OR_MORE_NON_ASCII = r"([\W _])\1{3,}" # 3 or more identical chars
210RST_TITLE_MARKER = re.compile(rf"^[ ]*{RE_3_OR_MORE_NON_ASCII}[ ]*$")
211MD_TITLE_MARKER = re.compile(r"^[ ]*[#]+[ ]*(.*)[ ]*$") # One or more starting hash with optional whitespaces before.
212FIRST_NON_MARKER_WITHOUT_HASH = re.compile(rf"^[# ]*(?!{RE_3_OR_MORE_NON_ASCII})[# ]*(.+)", re.MULTILINE)
215def extract_readme_title(file: Path, contents: str) -> str:
216 """Same as `extract_intro_and_title` for the readme files in galleries, but does not return the introduction.
218 Parameters
219 ----------
220 file : Path
221 The readme file path (used for error messages only).
223 contents : str
224 The already parsed readme contents
226 Returns
227 -------
228 title : str
229 The readme title
230 """
231 # Remove html comments.
232 contents = re.sub("(<!--.*?-->)", "", contents, flags=re.DOTALL)
234 match = FIRST_NON_MARKER_WITHOUT_HASH.search(contents)
235 if match is None: 235 ↛ 236line 235 didn't jump to line 236 because the condition on line 235 was never true
236 raise ExtensionError(f"Could not find a title in readme file: {file}")
238 title = match.group(2).strip()
239 return title
242def extract_readme_last_subtitle(file: Path, contents: str) -> str:
243 """Same as `extract_intro_and_title` for the readme files in galleries, but does not return the introduction.
245 Parameters
246 ----------
247 file : Path
248 The readme file path (used for error messages only).
250 contents : str
251 The already parsed readme contents
253 Returns
254 -------
255 last_subtitle : str
256 The readme last title, or None.
257 """
258 paragraphs = extract_paragraphs(contents)
260 # iterate from last paragraph
261 last_subtitle = None
262 for p in reversed(paragraphs): 262 ↛ 283line 262 didn't jump to line 283 because the loop on line 262 didn't complete
263 current_is_good = False
264 for line in reversed(p.splitlines()):
265 if current_is_good: 265 ↛ 266line 265 didn't jump to line 266 because the condition on line 265 was never true
266 last_subtitle = line
267 break
268 # Does this line contain a title ?
269 # - md style
270 md_match = MD_TITLE_MARKER.search(line)
271 if md_match:
272 last_subtitle = md_match.group(1)
273 break
275 # - rst style
276 rst_match = RST_TITLE_MARKER.search(line)
277 if rst_match: 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true
278 current_is_good = True
280 if last_subtitle:
281 break
283 return last_subtitle
286def extract_paragraphs(doc: str) -> List[str]:
287 # lstrip is just in case docstring has a '\n\n' at the beginning
288 paragraphs = doc.lstrip().split("\n\n")
290 # remove comments and other syntax like `.. _link:`
291 paragraphs = [p for p in paragraphs if not p.startswith(".. ") and len(p) > 0]
293 return paragraphs
296def extract_intro_and_title(docstring: str, script: GalleryScript) -> Tuple[str, str]:
297 """Extract and clean the first paragraph of module-level docstring.
299 The title is not saved in the `script` object in this process, users have to do it explicitly.
301 Parameters
302 ----------
303 docstring : str
304 The docstring extracted from the top of the script.
306 script : GalleryScript
307 The script where the docstring was extracted from (used for error messages only).
309 Returns
310 -------
311 title : str
312 The title
314 introduction : str
315 The introduction
316 """
317 # Extract paragraphs from the text
318 paragraphs = extract_paragraphs(docstring)
319 if len(paragraphs) == 0: 319 ↛ 320line 319 didn't jump to line 320 because the condition on line 319 was never true
320 raise ExtensionError(
321 f"Example docstring should have a header for the example title. "
322 f"Please check the example file:\n {script.script_file}\n"
323 )
325 # Title is the first paragraph with any RST/Markdown title chars
326 # removed, i.e. lines that consist of (3 or more of the same) 7-bit
327 # non-ASCII chars.
328 # This conditional is not perfect but should hopefully be good enough.
329 title_paragraph = paragraphs[0]
330 match = FIRST_NON_MARKER_WITHOUT_HASH.search(title_paragraph)
331 if match is None: 331 ↛ 332line 331 didn't jump to line 332 because the condition on line 331 was never true
332 raise ExtensionError(f"Could not find a title in first paragraph:\n{title_paragraph}")
334 title = match.group(2).strip()
336 # Use the title if no other paragraphs are provided
337 intro_paragraph = title if len(paragraphs) < 2 else paragraphs[1]
339 # Concatenate all lines of the first paragraph
340 intro = re.sub("\n", " ", intro_paragraph)
341 intro = _sanitize_md(intro)
343 # Truncate at 95 chars
344 if len(intro) > 95:
345 intro = intro[:95] + "..."
347 return title, intro
350def create_thumb_from_image(script: GalleryScript, src_image_path: Path) -> Path:
351 """Create a thumbnail image from the `src_image_path`.
353 Parameters
354 ----------
355 script : GalleryScript
356 The gallery script.
358 src_image_path : Path
359 The source image path, with some flexibility about the extension.
360 TODO do we actually need this flexibility here ?
362 Returns
363 -------
364 actual_thumb_file : Path
365 The actual thumbnail file generated.
366 """
367 try:
368 # Find the image, with flexibility about the actual extenstion ('png', 'svg', 'jpg', 'gif' are supported)
369 src_image_path, ext = _find_image_ext(src_image_path)
370 except ImageNotFoundError:
371 # The source image does not exist !
372 try:
373 # Does a thumbnail already exist ? with extenstion ('png', 'svg', 'jpg', 'gif')
374 thumb_file, ext = _find_image_ext(script.get_thumbnail_file(".png"))
375 # Yes - let's assume this one will suit the needs
376 return thumb_file
377 except ImageNotFoundError:
378 # Create something to replace the thumbnail
379 default_thumb_path = script.gallery_conf.get("default_thumb_file")
380 if default_thumb_path is None: 380 ↛ 383line 380 didn't jump to line 383 because the condition on line 380 was always true
381 default_thumb_path = os.path.join(glr_path_static(), "no_image.png")
383 src_image_path, ext = _find_image_ext(Path(default_thumb_path))
385 # Now let's create the thumbnail.
386 # - First Make sure the thumb dir exists
387 script.gallery.make_thumb_dir()
389 # - Then create the thum file by copying the src image, possibly rescaling it.
390 thumb_file = script.get_thumbnail_file(ext)
391 if ext in (".svg", ".gif"):
392 # No need to rescale image
393 copyfile(src_image_path, thumb_file)
394 else:
395 # Need to rescale image
396 max_width, max_hegiht = script.gallery_conf["thumbnail_size"]
397 rescale_image(
398 in_file=src_image_path,
399 out_file=thumb_file,
400 max_width=max_width,
401 max_height=max_hegiht,
402 )
403 if "thumbnails" in script.gallery_conf["compress_images"]: 403 ↛ 404line 403 didn't jump to line 404 because the condition on line 403 was never true
404 optipng(thumb_file, script.gallery_conf["compress_images_args"])
406 return thumb_file
409def generate(gallery: GalleryBase, seen_backrefs: Set) -> Tuple[str, str, str, List[GalleryScriptResults]]:
410 """
411 Generate the gallery md for an example directory, including the index.
413 Parameters
414 ----------
415 gallery : GalleryBase
416 The gallery or subgallery to process
418 seen_backrefs : Set
419 Backrefs seen so far.
421 Returns
422 -------
423 title : str
424 The gallery title, that is, the title of the readme file.
426 root_subtitle : str
427 The gallery suptitle that will be used in case the gallery has subsections.
429 index_md : str
430 The markdown to include in the global gallery readme.
432 results : List[GalleryScriptResults]
433 A list of processing results for all scripts in this gallery.
434 """
435 # Read the gallery readme and add it to the index
436 readme_contents = gallery.readme_file.read_text(encoding="utf-8")
437 readme_title = extract_readme_title(gallery.readme_file, readme_contents)
438 if gallery.has_subsections():
439 # parse and try to also extract the last subtitle
440 last_readme_subtitle = extract_readme_last_subtitle(gallery.readme_file, readme_contents)
441 else:
442 # Dont look for the last subtitle
443 last_readme_subtitle = None
445 # Create the destination dir if needed
446 gallery.make_generated_dir()
448 all_thumbnail_entries = []
449 results = []
451 for script in tqdm(gallery.scripts, desc=f"generating gallery for {gallery.generated_dir}... "):
452 # Generate all files related to this example: download file, jupyter notebook, pickle, markdown...
453 script_results = generate_file_md(script=script, seen_backrefs=seen_backrefs)
454 results.append(script_results)
456 # Create the thumbnails-containing div <div class="mkd-glr-thumbcontainer" ...> to place in the readme
457 thumb_div = _thumbnail_div(script_results)
458 all_thumbnail_entries.append(thumb_div)
460 # Write the gallery summary index.md
461 # Note: we write the HTML comment at the bottom instead of the top because having it at the top prevents html
462 # page metadata from mkdocs-material to be processed correctly. See GH#96
463 index_md = f"""{readme_contents}
465{"".join(all_thumbnail_entries)}
466<div class="mkd-glr-clear"></div>
468<!-- {str(gallery.generated_dir_rel_project).replace(os.path.sep, '_')} -->
470"""
471 # Note: the "clear" is to disable floating elements again, now that the gallery section is over.
473 return readme_title, last_readme_subtitle, index_md, results
476def is_failing_example(script: GalleryScript):
477 return script.src_py_file in script.gallery_conf["failing_examples"]
480def handle_exception(exc_info, script: GalleryScript):
481 """Trim and format exception, maybe raise error, etc."""
482 from .gen_gallery import _expected_failing_examples
484 etype, exc, tb = exc_info
485 stack = traceback.extract_tb(tb)
486 # The full traceback will look something like:
487 #
488 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_single.py...
489 # mem_max, _ = gallery_conf['call_memory'](
490 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_galler...
491 # mem, out = memory_usage(func, max_usage=True, retval=True,
492 # File "/home/larsoner/.local/lib/python3.8/site-packages/memory_profi...
493 # returned = f(*args, **kw)
494 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_single.py...
495 # exec(self.code, self.fake_main.__dict__)
496 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/tests/tiny...
497 # raise RuntimeError('some error')
498 # RuntimeError: some error
499 #
500 # But we should trim these to just the relevant trace at the user level,
501 # so we inspect the traceback to find the start and stop points.
502 start = 0
503 stop = len(stack)
504 root = os.path.dirname(__file__) + os.sep
505 for ii, s in enumerate(stack, 1):
506 # Trim our internal stack
507 if s.filename.startswith(root + "gen_gallery.py") and s.name == "call_memory":
508 start = max(ii, start)
509 elif s.filename.startswith(root + "gen_single.py"):
510 # SyntaxError
511 if s.name == "execute_code_block" and ("compile(" in s.line or "save_figures" in s.line): 511 ↛ 512line 511 didn't jump to line 512 because the condition on line 511 was never true
512 start = max(ii, start)
513 # Any other error
514 elif s.name == "__call__":
515 start = max(ii, start)
516 # Our internal input() check
517 elif s.name == "_check_input" and ii == len(stack): 517 ↛ 518line 517 didn't jump to line 518 because the condition on line 517 was never true
518 stop = ii - 1
519 stack = stack[start:stop]
521 formatted_exception = "Traceback (most recent call last):\n" + "".join(
522 traceback.format_list(stack) + traceback.format_exception_only(etype, exc)
523 )
525 src_file = script.src_py_file
526 expected = src_file in _expected_failing_examples(
527 gallery_conf=script.gallery_conf,
528 mkdocs_conf=script.gallery.all_info.mkdocs_conf,
529 )
530 if expected: 530 ↛ 535line 530 didn't jump to line 535 because the condition on line 530 was always true
531 # func, color = logger.info, 'blue'
532 func = logger.info
533 else:
534 # func, color = logger.warning, 'red'
535 func = logger.warning
536 func(f"{src_file} failed to execute correctly: {formatted_exception}") # , color=color)
538 except_md = codestr2md(formatted_exception, lang="pytb", is_exc=True)
540 # Ensure it's marked as our style: this is now already done in codestr2md
541 # except_md = "{: .mkd-glr-script-out }\n\n" + except_md
542 return except_md, formatted_exception
545# Adapted from github.com/python/cpython/blob/3.7/Lib/warnings.py
546def _showwarning(message, category, filename, lineno, file=None, line=None):
547 if file is None:
548 file = sys.stderr
549 if file is None:
550 # sys.stderr is None when run with pythonw.exe:
551 # warnings get lost
552 return
553 text = warnings.formatwarning(message, category, filename, lineno, line)
554 try:
555 file.write(text)
556 except OSError:
557 # the file (probably stderr) is invalid - this warning gets lost.
558 pass
561@contextlib.contextmanager
562def patch_warnings():
563 """Patch warnings.showwarning to actually write out the warning."""
564 # Sphinx or logging or someone is patching warnings, but we want to
565 # capture them, so let's patch over their patch...
566 orig_showwarning = warnings.showwarning
567 try:
568 warnings.showwarning = _showwarning
569 yield
570 finally:
571 warnings.showwarning = orig_showwarning
574class _exec_once(object):
575 """Deal with memory_usage calling functions more than once (argh)."""
577 def __init__(self, code, fake_main):
578 self.code = code
579 self.fake_main = fake_main
580 self.run = False
582 def __call__(self):
583 if not self.run: 583 ↛ exitline 583 didn't return from function '__call__' because the condition on line 583 was always true
584 self.run = True
585 old_main = sys.modules.get("__main__", None)
586 with patch_warnings():
587 sys.modules["__main__"] = self.fake_main
588 try:
589 exec(self.code, self.fake_main.__dict__) # noqa # our purpose is to execute code :)
590 finally:
591 if old_main is not None: 591 ↛ exitline 591 didn't return from function '__call__' because the condition on line 591 was always true
592 sys.modules["__main__"] = old_main
595def _get_memory_base(gallery_conf):
596 """Get the base amount of memory used by running a Python process."""
597 if not gallery_conf["plot_gallery"]:
598 return 0.0
599 # There might be a cleaner way to do this at some point
600 from memory_profiler import memory_usage
602 if sys.platform in ("win32", "darwin"):
603 sleep, timeout = (1, 2)
604 else:
605 sleep, timeout = (0.5, 1)
606 proc = subprocess.Popen(
607 [sys.executable, "-c", "import time, sys; time.sleep(%s); sys.exit(0)" % sleep],
608 close_fds=True,
609 )
610 memories = memory_usage(proc, interval=1e-3, timeout=timeout)
611 kwargs = dict(timeout=timeout) if sys.version_info >= (3, 5) else {}
612 proc.communicate(**kwargs)
613 # On OSX sometimes the last entry can be None
614 memories = [mem for mem in memories if mem is not None] + [0.0]
615 memory_base = max(memories)
616 return memory_base
619def _ast_module():
620 """Get ast.Module function, dealing with:
621 https://bugs.python.org/issue35894"""
622 if sys.version_info >= (3, 8): 622 ↛ 625line 622 didn't jump to line 625 because the condition on line 622 was always true
623 ast_Module = partial(ast.Module, type_ignores=[])
624 else:
625 ast_Module = ast.Module
626 return ast_Module
629def _check_reset_logging_tee(src_file: Path):
630 # Helper to deal with our tests not necessarily calling parse_and_execute
631 # but rather execute_code_block directly
632 if isinstance(sys.stdout, _LoggingTee): 632 ↛ 635line 632 didn't jump to line 635 because the condition on line 632 was always true
633 logging_tee = sys.stdout
634 else:
635 logging_tee = _LoggingTee(src_file)
636 logging_tee.set_std_and_reset_position()
637 return logging_tee
640def _exec_and_get_memory(compiler, ast_Module, code_ast, script: GalleryScript):
641 """Execute ast, capturing output if last line is expression and get max memory usage."""
643 src_file = script.src_py_file.as_posix()
645 # capture output if last line is expression
646 is_last_expr = False
648 if len(code_ast.body) and isinstance(code_ast.body[-1], ast.Expr):
649 is_last_expr = True
650 last_val = code_ast.body.pop().value
651 # exec body minus last expression
652 mem_body, _ = script.gallery_conf["call_memory"](
653 _exec_once(compiler(code_ast, src_file, "exec"), script.run_vars.fake_main)
654 )
655 # exec last expression, made into assignment
656 body = [ast.Assign(targets=[ast.Name(id="___", ctx=ast.Store())], value=last_val)]
657 last_val_ast = ast_Module(body=body)
658 ast.fix_missing_locations(last_val_ast)
659 mem_last, _ = script.gallery_conf["call_memory"](
660 _exec_once(compiler(last_val_ast, src_file, "exec"), script.run_vars.fake_main)
661 )
662 mem_max = max(mem_body, mem_last)
663 else:
664 mem_max, _ = script.gallery_conf["call_memory"](
665 _exec_once(compiler(code_ast, src_file, "exec"), script.run_vars.fake_main)
666 )
668 return is_last_expr, mem_max
671def _get_last_repr(gallery_conf, ___):
672 """Get a repr of the last expression, using first method in 'capture_repr'
673 available for the last expression."""
674 for meth in gallery_conf["capture_repr"]: 674 ↛ 688line 674 didn't jump to line 688 because the loop on line 674 didn't complete
675 try:
676 last_repr = getattr(___, meth)()
677 # for case when last statement is print()
678 if last_repr is None or last_repr == "None":
679 repr_meth = None
680 else:
681 repr_meth = meth
682 except Exception:
683 last_repr = None
684 repr_meth = None
685 else:
686 if isinstance(last_repr, str): 686 ↛ 674line 686 didn't jump to line 674 because the condition on line 686 was always true
687 break
688 return last_repr, repr_meth
691def _get_code_output(is_last_expr, script: GalleryScript, logging_tee, images_md):
692 """Obtain standard output and html output in md."""
694 example_globals = script.run_vars.example_globals
695 gallery_conf = script.gallery_conf
697 last_repr = None
698 repr_meth = None
699 if is_last_expr:
700 # capture the last repr variable
701 ___ = example_globals["___"]
702 ignore_repr = False
703 if gallery_conf["ignore_repr_types"]: 703 ↛ 704line 703 didn't jump to line 704 because the condition on line 703 was never true
704 ignore_repr = re.search(gallery_conf["ignore_repr_types"], str(type(___)))
705 if gallery_conf["capture_repr"] != () and not ignore_repr: 705 ↛ 708line 705 didn't jump to line 708 because the condition on line 705 was always true
706 last_repr, repr_meth = _get_last_repr(gallery_conf, ___)
708 captured_std = logging_tee.output.getvalue().expandtabs()
710 # normal string output
711 if repr_meth in ["__repr__", "__str__"] and last_repr:
712 captured_std = f"{captured_std}\n{last_repr}"
714 if captured_std and not captured_std.isspace():
715 captured_std = CODE_OUTPUT.format(captured_std)
716 else:
717 captured_std = ""
719 # give html output its own header
720 if repr_meth == "_repr_html_":
721 captured_html = HTML_HEADER.format(indent(last_repr, " " * 4))
722 else:
723 captured_html = ""
725 code_output = f"""
726{images_md}
728{captured_std}
730{captured_html}
732"""
733 return code_output
736def _reset_cwd_syspath(cwd, path_to_remove):
737 """Reset current working directory to `cwd` and remove `path_to_remove` from `sys.path`."""
738 if path_to_remove in sys.path:
739 sys.path.remove(path_to_remove)
740 os.chdir(cwd)
743def _parse_code(bcontent, src_file, *, compiler_flags):
744 code_ast = compile(bcontent, src_file, "exec", compiler_flags | ast.PyCF_ONLY_AST, dont_inherit=1)
745 if _needs_async_handling(bcontent, src_file, compiler_flags=compiler_flags):
746 code_ast = _apply_async_handling(code_ast, compiler_flags=compiler_flags)
747 return code_ast
750def _needs_async_handling(bcontent, src_file, *, compiler_flags) -> bool:
751 try:
752 compile(bcontent, src_file, "exec", compiler_flags, dont_inherit=1)
753 except SyntaxError as error:
754 # mkdocs-gallery supports top-level async code similar to jupyter notebooks.
755 # Without handling, this will raise a SyntaxError. In such a case, we apply a
756 # minimal async handling and try again. If the error persists, we bubble it up
757 # and let the caller handle it.
758 try:
759 compile(
760 f"async def __async_wrapper__():\n{indent(bcontent, ' ' * 4)}",
761 src_file,
762 "exec",
763 compiler_flags,
764 dont_inherit=1,
765 )
766 except SyntaxError:
767 # Raise the original error to avoid leaking the internal async handling to
768 # generated output.
769 raise error from None
770 else:
771 return True
772 else:
773 return False
776def _apply_async_handling(code_ast, *, compiler_flags):
777 async_handling = compile(
778 dedent(
779 """
780 async def __async_wrapper__():
781 # original AST goes here
782 return locals()
783 __async_wrapper_locals__ = __run_async__(__async_wrapper__())
784 __async_wrapper_result__ = __async_wrapper_locals__.pop("__async_wrapper_result__", None)
785 globals().update(__async_wrapper_locals__)
786 __async_wrapper_result__
787 """
788 ),
789 "<_apply_async_handling()>",
790 "exec",
791 compiler_flags | ast.PyCF_ONLY_AST,
792 dont_inherit=1,
793 )
795 *original_body, last_node = code_ast.body
796 if isinstance(last_node, ast.Expr):
797 last_node = ast.Assign(
798 targets=[ast.Name(id="__async_wrapper_result__", ctx=ast.Store())], value=last_node.value
799 )
800 original_body.append(last_node)
802 async_wrapper = async_handling.body[0]
803 async_wrapper.body = [*original_body, *async_wrapper.body]
805 return ast.fix_missing_locations(async_handling)
808def execute_code_block(compiler, block, script: GalleryScript):
809 """Execute the code block of the example file.
811 Parameters
812 ----------
813 compiler : codeop.Compile
814 Compiler to compile AST of code block.
816 block : List[Tuple[str, str, int]]
817 List of Tuples, each Tuple contains label ('text' or 'code'),
818 the corresponding content string of block and the leading line number.
820 script: GalleryScript
821 The gallery script
823 Returns
824 -------
825 code_output : str
826 Output of executing code in md.
827 """
828 # if script.run_vars.example_globals is None: # testing shortcut
829 # script.run_vars.example_globals = script.run_vars.fake_main.__dict__
831 blabel, bcontent, lineno = block
833 # If example is not suitable to run anymore, skip executing its blocks
834 if script.run_vars.stop_executing or blabel == "text":
835 return ""
837 cwd = os.getcwd()
838 # Redirect output to stdout
839 src_file = script.src_py_file
840 logging_tee = _check_reset_logging_tee(src_file)
841 assert isinstance(logging_tee, _LoggingTee) # noqa
843 # First cd in the original example dir, so that any file
844 # created by the example get created in this directory
845 os.chdir(src_file.parent)
847 # Add the example dir to the path temporarily (will be removed after execution)
848 new_path = os.getcwd()
849 sys.path.append(new_path)
851 # Save figures unless there is a `mkdocs_gallery_defer_figures` flag
852 match = re.search(r"^[\ \t]*#\s*mkdocs_gallery_defer_figures[\ \t]*\n?", bcontent, re.MULTILINE)
853 need_save_figures = match is None
855 try:
856 ast_Module = _ast_module()
857 code_ast = _parse_code(bcontent, src_file, compiler_flags=compiler.flags)
858 ast.increment_lineno(code_ast, lineno - 1)
860 is_last_expr, mem_max = _exec_and_get_memory(compiler, ast_Module, code_ast, script=script)
861 script.run_vars.memory_used_in_blocks.append(mem_max)
863 # This should be inside the try block, e.g., in case of a savefig error
864 logging_tee.restore_std()
865 if need_save_figures: 865 ↛ 869line 865 didn't jump to line 869 because the condition on line 865 was always true
866 need_save_figures = False
867 images_md = save_figures(block, script)
868 else:
869 images_md = ""
871 except Exception:
872 logging_tee.restore_std()
873 except_md, formatted_exception = handle_exception(sys.exc_info(), script)
875 # Breaks build on first example error
876 if script.gallery_conf["abort_on_example_error"]: 876 ↛ 877line 876 didn't jump to line 877 because the condition on line 876 was never true
877 raise
879 # Stores failing file
880 script.gallery_conf["failing_examples"][src_file] = formatted_exception
882 # Stop further execution on that script
883 script.run_vars.stop_executing = True
885 code_output = "\n{0}\n\n\n\n".format(except_md)
886 # still call this even though we won't use the images so that
887 # figures are closed
888 if need_save_figures: 888 ↛ 895line 888 didn't jump to line 895 because the condition on line 888 was always true
889 save_figures(block, script)
890 else:
891 _reset_cwd_syspath(cwd, new_path)
893 code_output = _get_code_output(is_last_expr, script, logging_tee, images_md)
894 finally:
895 _reset_cwd_syspath(cwd, new_path)
896 logging_tee.restore_std() 896 ↛ exitline 896 didn't except from function 'execute_code_block' because the raise on line 877 wasn't executed
898 # Sanitize ANSI escape characters from MD output
899 ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
900 code_output = ansi_escape.sub("", code_output)
902 return code_output
905def _check_input(prompt=None):
906 raise ExtensionError("Cannot use input() builtin function in mkdocs-gallery examples")
909def parse_and_execute(script: GalleryScript, script_blocks):
910 """Execute and capture output from python script already in block structure
912 Parameters
913 ----------
914 script : GalleryScript
915 The script
917 script_blocks : list
918 (label, content, line_number)
919 List where each element is a tuple with the label ('text' or 'code'),
920 the corresponding content string of block and the leading line number
922 Returns
923 -------
924 output_blocks : list
925 List of strings where each element is the restructured text
926 representation of the output of each block
928 time_elapsed : float
929 Time elapsed during execution
931 memory_used : float
932 Memory used during execution
933 """
934 # Examples may contain if __name__ == '__main__' guards for in example scikit-learn if the example uses
935 # multiprocessing. Here we create a new __main__ module, and temporarily change sys.modules when running our example
936 fake_main = importlib.util.module_from_spec(importlib.util.spec_from_loader("__main__", None))
937 script.run_vars.fake_main = fake_main
939 example_globals = fake_main.__dict__
940 example_globals.update(
941 {
942 # A lot of examples contains 'print(__doc__)' for example in
943 # scikit-learn so that running the example prints some useful
944 # information. Because the docstring has been separated from
945 # the code blocks in mkdocs-gallery, __doc__ is actually
946 # __builtin__.__doc__ in the execution context and we do not
947 # want to print it
948 "__doc__": "",
949 # Don't ever support __file__: Issues #166 #212
950 # Don't let them use input()
951 "input": _check_input,
952 "__run_async__": run_async,
953 }
954 )
955 script.run_vars.example_globals = example_globals
957 # Manipulate the sys.argv before running the example
958 # See https://github.com/sphinx-gallery/sphinx-gallery/pull/252
960 # Remember the original argv so that we can put them back after run
961 argv_orig = sys.argv[:]
963 # Remember the original sys.path so that we can reset it after run
964 sys_path_orig = deepcopy(sys.path)
966 # Python file is the original one (not the copy for download)
967 sys.argv[0] = script.src_py_file.as_posix()
969 # Allow users to provide additional args through the 'reset_argv' option
970 sys.argv[1:] = script.gallery_conf["reset_argv"](script)
972 # Perform a garbage collection before starting so that perf kpis are accurate (memory and time)
973 gc.collect()
975 # Initial memory used
976 memory_start, _ = script.gallery_conf["call_memory"](lambda: None)
977 script.run_vars.memory_used_in_blocks = [memory_start] # include at least one entry to avoid max() ever failing
979 t_start = time()
980 compiler = codeop.Compile()
982 # Execute block by block
983 output_blocks = list()
984 with _LoggingTee(script.src_py_file) as logging_tee:
985 for block in script_blocks:
986 logging_tee.set_std_and_reset_position()
987 output_blocks.append(execute_code_block(compiler, block, script))
989 # Compute the elapsed time
990 time_elapsed = time() - t_start
992 # Set back the sys argv
993 sys.argv = argv_orig
995 # Set back the sys path
996 sys.path = sys_path_orig
998 # Write md5 checksum if the example was meant to run (no-plot shall not cache md5sum) and has built correctly
999 script.write_final_md5_file()
1001 # Declare the example as "passing"
1002 script.gallery_conf["passing_examples"].append(script)
1004 script.run_vars.memory_delta = max(script.run_vars.memory_used_in_blocks) - memory_start
1005 memory_used = script.gallery_conf["memory_base"] + script.run_vars.memory_delta
1007 return output_blocks, time_elapsed, memory_used
1010def generate_file_md(script: GalleryScript, seen_backrefs=None) -> GalleryScriptResults:
1011 """Generate the md file for a given example.
1013 Parameters
1014 ----------
1015 script : GalleryScript
1016 The script to process
1018 seen_backrefs : set
1019 The seen backreferences.
1021 Returns
1022 -------
1023 result: FileResult
1024 The result of running this script
1025 """
1026 seen_backrefs = set() if seen_backrefs is None else seen_backrefs
1028 # Extract the contents of the script
1029 file_conf, script_blocks, node = split_code_and_text_blocks(script.src_py_file, return_node=True)
1031 # Extract the title and introduction from the module docstring and save the title in the object
1032 script.title, intro = extract_intro_and_title(docstring=script_blocks[0][1], script=script)
1034 # Copy source python script to target folder if it is not there/up to date, so that it can be served/downloaded
1035 # Note: surprisingly this uses a md5 too, but not the final .md5 persisted on disk.
1036 script.make_dwnld_py_file()
1038 # Can the script be entirely skipped (both doc generation and execution) ?
1039 if not script.has_changed_wrt_persisted_md5():
1040 # A priori we can...
1041 skip_and_return = True
1043 # ...however for executables (not shared modules) we might need to run anyway because of config
1044 if script.is_executable_example(): 1044 ↛ 1056line 1044 didn't jump to line 1056 because the condition on line 1044 was always true
1045 if script.gallery_conf["run_stale_examples"]: 1045 ↛ 1047line 1045 didn't jump to line 1047 because the condition on line 1045 was never true
1046 # Run anyway because config says so.
1047 skip_and_return = False
1048 else:
1049 # Add the example to the "stale examples" before returning
1050 script.gallery_conf["stale_examples"].append(script.dwnld_py_file)
1051 # If expected to fail, let's remove it from the 'expected_failing_examples' list,
1052 # assuming it did when previously executed
1053 if script.src_py_file in script.gallery_conf["expected_failing_examples"]:
1054 script.gallery_conf["expected_failing_examples"].remove(script.src_py_file)
1056 if skip_and_return: 1056 ↛ 1063line 1056 didn't jump to line 1063 because the condition on line 1056 was always true
1057 # Return with 0 exec time and mem usage, and the existing thumbnail
1058 thumb_source_path = script.get_thumbnail_source(file_conf)
1059 thumb_file = create_thumb_from_image(script, thumb_source_path)
1060 return GalleryScriptResults(script=script, intro=intro, exec_time=0.0, memory=0.0, thumb=thumb_file)
1062 # Reset matplotlib, seaborn, etc. if needed
1063 if script.is_executable_example():
1064 clean_modules(gallery_conf=script.gallery_conf, file=script.src_py_file)
1066 # Init the runtime vars. Create the images directory and init the image files template
1067 script.init_before_processing()
1069 if script.is_executable_example():
1070 # Note: this writes the md5 checksum if the example was meant to run
1071 output_blocks, time_elapsed, memory_used = parse_and_execute(script, script_blocks)
1072 logger.debug(f"{script.src_py_file} ran in : {time_elapsed:.2g} seconds\n")
1073 else:
1074 output_blocks = [""] * len(script_blocks)
1075 time_elapsed = memory_used = 0.0 # don't let the output change
1076 logger.debug(f"{script.src_py_file} parsed (not executed)\n")
1078 # Create as many dummy images as required if needed (default none) so that references to script images
1079 # Can still work, even if the script was not executed (in development mode typically, to go fast).
1080 # See https://sphinx-gallery.github.io/stable/configuration.html#generating-dummy-images
1081 nb_dummy_images_to_generate = file_conf.get("dummy_images", None)
1082 if nb_dummy_images_to_generate is not None: 1082 ↛ 1083line 1082 didn't jump to line 1083 because the condition on line 1082 was never true
1083 if type(nb_dummy_images_to_generate) is not int:
1084 raise ExtensionError("mkdocs_gallery: 'dummy_images' setting is not a number, got {dummy_image!r}")
1086 stock_img = os.path.join(glr_path_static(), "no_image.png")
1087 script.generate_n_dummy_images(img=stock_img, nb=nb_dummy_images_to_generate)
1089 # Remove the mkdocs-gallery configuration comments from the script if needed
1090 if script.gallery_conf["remove_config_comments"]: 1090 ↛ 1091line 1090 didn't jump to line 1091 because the condition on line 1090 was never true
1091 script_blocks = [
1092 (label, remove_config_comments(content), line_number) for label, content, line_number in script_blocks
1093 ]
1095 # Remove final empty block, which can occur after config comments are removed
1096 if script_blocks[-1][1].isspace(): 1096 ↛ 1097line 1096 didn't jump to line 1097 because the condition on line 1096 was never true
1097 script_blocks = script_blocks[:-1]
1098 output_blocks = output_blocks[:-1]
1100 # Generate the markdown string containing the script prose, code and output.
1101 example_md = generate_md_from_blocks(script_blocks, output_blocks, file_conf, script.gallery_conf)
1103 # Write the generated markdown file
1104 md_header, md_footer = get_example_md_wrapper(script, time_elapsed, memory_used)
1105 full_md = md_header + example_md + md_footer
1106 script.save_md_example(full_md)
1108 # Create the image thumbnail for the gallery summary
1109 if is_failing_example(script):
1110 # Failing example thumbnail
1111 thumb_source_path = Path(os.path.join(glr_path_static(), "broken_example.png"))
1112 else:
1113 # Get the thumbnail source image, possibly from config
1114 thumb_source_path = script.get_thumbnail_source(file_conf)
1116 thumb_file = create_thumb_from_image(script, thumb_source_path)
1118 # Generate the jupyter notebook
1119 example_nb = jupyter_notebook(script, script_blocks)
1120 ipy_file = _new_file(script.ipynb_file)
1121 save_notebook(example_nb, ipy_file)
1122 _replace_by_new_if_needed(ipy_file, md5_mode="t")
1124 # Write names
1125 if script.gallery_conf["inspect_global_variables"]: 1125 ↛ 1128line 1125 didn't jump to line 1128 because the condition on line 1125 was always true
1126 global_variables = script.run_vars.example_globals
1127 else:
1128 global_variables = None
1130 # TODO dig in just in case
1131 example_code_obj = identify_names(script_blocks, global_variables, node)
1132 if example_code_obj:
1133 # Write a pickle file (.pickle) containing `example_code_obj`
1134 codeobj_fname = _new_file(script.codeobj_file)
1135 with open(codeobj_fname, "wb") as fid:
1136 pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL)
1137 _replace_by_new_if_needed(codeobj_fname)
1139 backrefs = set(
1140 "{module_short}.{name}".format(**cobj)
1141 for cobjs in example_code_obj.values()
1142 for cobj in cobjs
1143 if cobj["module"].startswith(script.gallery_conf["doc_module"])
1144 )
1146 # Create results object
1147 res = GalleryScriptResults(
1148 script=script,
1149 intro=intro,
1150 exec_time=time_elapsed,
1151 memory=memory_used,
1152 thumb=thumb_file,
1153 )
1155 # Write backreferences if required
1156 if script.gallery_conf["backreferences_dir"] is not None: 1156 ↛ 1159line 1156 didn't jump to line 1159 because the condition on line 1156 was always true
1157 _write_backreferences(backrefs, seen_backrefs, script_results=res)
1159 return res
1162# TODO the note should only appear in html mode. (.. only:: html)
1163# TODO maybe remove as much as possible the css for now?
1164EXAMPLE_HEADER = """
1165<!--
1166 DO NOT EDIT.
1167 THIS FILE WAS AUTOMATICALLY GENERATED BY mkdocs-gallery.
1168 TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
1169 "{pyfile_to_edit}"
1170 LINE NUMBERS ARE GIVEN BELOW.
1171-->
1173!!! note
1175 Click [here](#download_links)
1176 to download the full example code{opt_binder_text}
1178""" # TODO there was a {{: .mkd-glr-example-title }} for the title but is it useful ?
1179MD_BLOCK_HEADER = """\
1180<!-- GENERATED FROM PYTHON SOURCE LINES {0}-{1} -->
1182"""
1185def generate_md_from_blocks(script_blocks, output_blocks, file_conf, gallery_conf) -> str:
1186 """Generate the md string containing the script prose, code and output.
1188 Parameters
1189 ----------
1190 script_blocks : list
1191 (label, content, line_number)
1192 List where each element is a tuple with the label ('text' or 'code'),
1193 the corresponding content string of block and the leading line number
1195 output_blocks : list
1196 List of strings where each element is the restructured text
1197 representation of the output of each block
1199 file_conf : dict
1200 File-specific settings given in source file comments as:
1201 ``# mkdocs_gallery_<name> = <value>``
1203 gallery_conf : dict
1204 Contains the configuration of mkdocs-gallery
1206 Returns
1207 -------
1208 out : str
1209 The resulting markdown page.
1210 """
1212 # A simple example has two blocks: one for the
1213 # example introduction/explanation and one for the code
1214 is_example_notebook_like = len(script_blocks) > 2
1215 example_md = ""
1216 for bi, ((blabel, bcontent, lineno), code_output) in enumerate(zip(script_blocks, output_blocks)):
1217 # do not add comment to the title block (bi=0), otherwise the linking does not work properly
1218 if bi > 0:
1219 example_md += MD_BLOCK_HEADER.format(lineno, lineno + bcontent.count("\n"))
1221 if blabel == "code":
1222 if not file_conf.get("line_numbers", gallery_conf.get("line_numbers", False)): 1222 ↛ 1225line 1222 didn't jump to line 1225 because the condition on line 1222 was always true
1223 lineno = None
1225 code_md = codestr2md(bcontent, lang=gallery_conf["lang"], lineno=lineno) + "\n"
1226 if is_example_notebook_like:
1227 example_md += code_md
1228 example_md += code_output
1229 else:
1230 example_md += code_output
1231 if "mkd-glr-script-out" in code_output:
1232 # Add some vertical space after output
1233 example_md += "\n\n<br />\n\n" # "|\n\n"
1234 example_md += code_md
1235 else:
1236 block_separator = "\n\n" if not bcontent.endswith("\n") else "\n"
1237 example_md += bcontent + block_separator
1239 return example_md
1242def get_example_md_wrapper(script: GalleryScript, time_elapsed: float, memory_used: float) -> Tuple[str, str]:
1243 """Creates the headers and footers for the example markdown. Returns a template
1245 Parameters
1246 ----------
1247 script : GalleryScript
1248 The script for which to generate the md.
1250 time_elapsed : float
1251 Time elapsed in seconds while executing file
1253 memory_used : float
1254 Additional memory used during the run.
1256 Returns
1257 -------
1258 md_before : str
1259 Part of the final markdown that goes before the notebook / python script.
1261 md_after : str
1262 Part of the final markdown that goes after the notebook / python script.
1263 """
1264 # Check binder configuration
1265 binder_conf = check_binder_conf(script.gallery_conf.get("binder"))
1266 use_binder = len(binder_conf) > 0
1268 # Write header
1269 src_relative = script.src_py_file_rel_project.as_posix()
1270 binder_text = " or to run this example in your browser via Binder" if use_binder else ""
1271 md_before = EXAMPLE_HEADER.format(pyfile_to_edit=src_relative, opt_binder_text=binder_text)
1273 # Footer
1274 md_after = ""
1275 # Report Time and Memory
1276 if time_elapsed >= script.gallery_conf["min_reported_time"]: 1276 ↛ 1280line 1276 didn't jump to line 1280 because the condition on line 1276 was always true
1277 time_m, time_s = divmod(time_elapsed, 60)
1278 md_after += TIMING_CONTENT.format(time_m, time_s)
1280 if script.gallery_conf["show_memory"]: 1280 ↛ 1281line 1280 didn't jump to line 1281 because the condition on line 1280 was never true
1281 md_after += f"**Estimated memory usage:** {memory_used:.0f} MB\n\n"
1283 # Download buttons
1284 # - Generate a binder URL if specified
1285 binder_badge_md = gen_binder_md(script, binder_conf) if use_binder else ""
1286 # - Rely on mkdocs-material for the icon
1287 icon = ":fontawesome-solid-download:"
1288 # - Generate the download buttons
1289 # TODO why aren't they centered actually ? does .center work ?
1290 md_after += f"""
1291<div id="download_links"></div>
1293{binder_badge_md}
1295[{icon} Download Python source code: {script.dwnld_py_file.name}](./{script.dwnld_py_file.name}){{ .md-button .center}}
1297[{icon} Download Jupyter notebook: {script.ipynb_file.name}](./{script.ipynb_file.name}){{ .md-button .center}}
1298"""
1300 # Add the "generated by mkdocs-gallery" footer
1301 md_after += MKD_GLR_SIG
1303 return md_before, md_after