Coverage for src/mkdocs_gallery/gen_single.py: 82%
522 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-15 17:10 +0000
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/mkdocs-gallery>
3#
4# Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5# License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6"""
7Generator for a single script example in a gallery.
8"""
10from __future__ import absolute_import, division, print_function
12import ast
13import codeop
14import contextlib
15import copy
16import gc
17import importlib
18import os
19import pickle
20import re
21import subprocess
22import sys
23import traceback
24import warnings
25from copy import deepcopy
26from functools import partial
27from io import StringIO
28from pathlib import Path
29from shutil import copyfile
30from textwrap import indent, dedent
31from time import time
32from typing import List, Set, Tuple
34from tqdm import tqdm
36from . import glr_path_static, mkdocs_compatibility
37from .backreferences import _thumbnail_div, _write_backreferences, identify_names
38from .binder import check_binder_conf, gen_binder_md
39from .errors import ExtensionError
40from .gen_data_model import GalleryBase, GalleryScript, GalleryScriptResults
41from .notebook import jupyter_notebook, save_notebook
42from .py_source_parser import remove_config_comments, split_code_and_text_blocks
43from .scrapers import ImageNotFoundError, _find_image_ext, clean_modules, save_figures
44from .utils import _new_file, _replace_by_new_if_needed, optipng, rescale_image
46logger = mkdocs_compatibility.getLogger("mkdocs-gallery")
49###############################################################################
52class _LoggingTee(object):
53 """A tee object to redirect streams to the logger."""
55 def __init__(self, src_filename: Path):
56 self.logger = logger
57 self.src_filename = src_filename
58 self.logger_buffer = ""
59 self.set_std_and_reset_position()
61 def set_std_and_reset_position(self):
62 if not isinstance(sys.stdout, _LoggingTee):
63 self.origs = (sys.stdout, sys.stderr)
64 sys.stdout = sys.stderr = self
65 self.first_write = True
66 self.output = StringIO()
67 return self
69 def restore_std(self):
70 sys.stdout.flush()
71 sys.stderr.flush()
72 sys.stdout, sys.stderr = self.origs
74 def write(self, data):
75 self.output.write(data)
77 if self.first_write:
78 self.logger.verbose("Output from %s", self.src_filename) # color='brown')
79 self.first_write = False
81 data = self.logger_buffer + data
82 lines = data.splitlines()
83 if data and data[-1] not in "\r\n":
84 # Wait to write last line if it's incomplete. It will write next
85 # time or when the LoggingTee is flushed.
86 self.logger_buffer = lines[-1]
87 lines = lines[:-1]
88 else:
89 self.logger_buffer = ""
91 for line in lines:
92 self.logger.verbose("%s", line)
94 def flush(self):
95 self.output.flush()
96 if self.logger_buffer: 96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true
97 self.logger.verbose("%s", self.logger_buffer)
98 self.logger_buffer = ""
100 # When called from a local terminal seaborn needs it in Python3
101 def isatty(self):
102 return self.output.isatty()
104 # When called in gen_single, conveniently use context managing
105 def __enter__(self):
106 return self
108 def __exit__(self, type_, value, tb):
109 self.restore_std()
112###############################################################################
113# The following strings are used when we have several pictures: we use
114# an html div tag that our CSS uses to turn the lists into horizontal
115# lists.
116HLIST_HEADER = """
117{: .mkd-glr-horizontal }
119"""
121HLIST_IMAGE_TEMPLATE = """
122 *
124 .. image:: /%s
125 {: .mkd-glr-multi-img }
126"""
128SINGLE_IMAGE = """
129.. image:: /%s
130 {: .mkd-glr-single-img }
131"""
133# Note: since this seems to be a one-liner, we use inline code. TODO check
134CODE_OUTPUT = """Out:
135{{: .mkd-glr-script-out }}
137```{{.shell .mkd-glr-script-out-disp }}
138{0}
139```
140\n"""
142TIMING_CONTENT = """
143**Total running time of the script:** ({0: .0f} minutes {1: .3f} seconds)
144""" # Strange enough: this CSS class does not actually exist in sphinx-gallery {{: .mkd-glr-timing }}
146# TODO only if html ? .. only:: html
147MKD_GLR_SIG = """\n
148[Gallery generated by mkdocs-gallery](https://mkdocs-gallery.github.io){: .mkd-glr-signature }
149"""
151# Header used to include raw html from data _repr_html_
152HTML_HEADER = """<div class="output_subarea output_html rendered_html output_result">
153{0}
154</div>
155"""
158def codestr2md(codestr, lang: str = "python", lineno=None, is_exc: bool = False):
159 """Return markdown code block from code string."""
161 # if lineno is not None:
162 # # Sphinx only starts numbering from the first non-empty line.
163 # blank_lines = codestr.count('\n', 0, -len(codestr.lstrip()))
164 # lineno = ' :lineno-start: {0}\n'.format(lineno + blank_lines)
165 # else:
166 # lineno = ''
167 # code_directive = ".. code-block:: {0}\n{1}\n".format(lang, lineno)
168 # indented_block = indent(codestr, ' ' * 4)
169 # return code_directive + indented_block
170 style = " .mkd-glr-script-err-disp" if is_exc else ""
171 if lineno is not None: 171 ↛ 174line 171 didn't jump to line 174, because the condition on line 171 was never true
172 # Sphinx only starts numbering from the first non-empty line. TODO do we need this too ?
173 # blank_lines = codestr.count('\n', 0, -len(codestr.lstrip()))
174 return f'```{{.{lang} {style} linenums="{lineno}"}}\n{codestr}```\n'
175 else:
176 return f"```{{.{lang} {style}}}\n{codestr}```\n"
179def _regroup(x):
180 x = x.groups()
181 return x[0] + x[1].split(".")[-1] + x[2]
184def _sanitize_md(string):
185 """Use regex to remove at least some sphinx directives.
187 TODO is this still needed ?
188 """
189 # :class:`a.b.c <thing here>`, :ref:`abc <thing here>` --> thing here
190 p, e = r"(\s|^):[^:\s]+:`", r"`(\W|$)"
191 string = re.sub(p + r"\S+\s*<([^>`]+)>" + e, r"\1\2\3", string)
192 # :class:`~a.b.c` --> c
193 string = re.sub(p + r"~([^`]+)" + e, _regroup, string)
194 # :class:`a.b.c` --> a.b.c
195 string = re.sub(p + r"([^`]+)" + e, r"\1\2\3", string)
197 # ``whatever thing`` --> whatever thing
198 p = r"(\s|^)`"
199 string = re.sub(p + r"`([^`]+)`" + e, r"\1\2\3", string)
200 # `whatever thing` --> whatever thing
201 string = re.sub(p + r"([^`]+)" + e, r"\1\2\3", string)
202 return string
205# Find RST/Markdown title chars,
206# i.e. lines that consist of (3 or more of the same) 7-bit non-ASCII chars.
207# This conditional is not perfect but should hopefully be good enough.
208RE_3_OR_MORE_NON_ASCII = r"([\W _])\1{3,}" # 3 or more identical chars
210RST_TITLE_MARKER = re.compile(rf"^[ ]*{RE_3_OR_MORE_NON_ASCII}[ ]*$")
211MD_TITLE_MARKER = re.compile(r"^[ ]*[#]+[ ]*(.*)[ ]*$") # One or more starting hash with optional whitespaces before.
212FIRST_NON_MARKER_WITHOUT_HASH = re.compile(rf"^[# ]*(?!{RE_3_OR_MORE_NON_ASCII})[# ]*(.+)", re.MULTILINE)
215def extract_readme_title(file: Path, contents: str) -> str:
216 """Same as `extract_intro_and_title` for the readme files in galleries, but does not return the introduction.
218 Parameters
219 ----------
220 file : Path
221 The readme file path (used for error messages only).
223 contents : str
224 The already parsed readme contents
226 Returns
227 -------
228 title : str
229 The readme title
230 """
231 # Remove html comments.
232 contents = re.sub("(<!--.*?-->)", "", contents, flags=re.DOTALL)
234 match = FIRST_NON_MARKER_WITHOUT_HASH.search(contents)
235 if match is None: 235 ↛ 236line 235 didn't jump to line 236, because the condition on line 235 was never true
236 raise ExtensionError(f"Could not find a title in readme file: {file}")
238 title = match.group(2).strip()
239 return title
242def extract_readme_last_subtitle(file: Path, contents: str) -> str:
243 """Same as `extract_intro_and_title` for the readme files in galleries, but does not return the introduction.
245 Parameters
246 ----------
247 file : Path
248 The readme file path (used for error messages only).
250 contents : str
251 The already parsed readme contents
253 Returns
254 -------
255 last_subtitle : str
256 The readme last title, or None.
257 """
258 paragraphs = extract_paragraphs(contents)
260 # iterate from last paragraph
261 last_subtitle = None
262 for p in reversed(paragraphs): 262 ↛ 283line 262 didn't jump to line 283, because the loop on line 262 didn't complete
263 current_is_good = False
264 for line in reversed(p.splitlines()):
265 if current_is_good: 265 ↛ 266line 265 didn't jump to line 266, because the condition on line 265 was never true
266 last_subtitle = line
267 break
268 # Does this line contain a title ?
269 # - md style
270 md_match = MD_TITLE_MARKER.search(line)
271 if md_match:
272 last_subtitle = md_match.group(1)
273 break
275 # - rst style
276 rst_match = RST_TITLE_MARKER.search(line)
277 if rst_match: 277 ↛ 278line 277 didn't jump to line 278, because the condition on line 277 was never true
278 current_is_good = True
280 if last_subtitle:
281 break
283 return last_subtitle
286def extract_paragraphs(doc: str) -> List[str]:
287 # lstrip is just in case docstring has a '\n\n' at the beginning
288 paragraphs = doc.lstrip().split("\n\n")
290 # remove comments and other syntax like `.. _link:`
291 paragraphs = [p for p in paragraphs if not p.startswith(".. ") and len(p) > 0]
293 return paragraphs
296def extract_intro_and_title(docstring: str, script: GalleryScript) -> Tuple[str, str]:
297 """Extract and clean the first paragraph of module-level docstring.
299 The title is not saved in the `script` object in this process, users have to do it explicitly.
301 Parameters
302 ----------
303 docstring : str
304 The docstring extracted from the top of the script.
306 script : GalleryScript
307 The script where the docstring was extracted from (used for error messages only).
309 Returns
310 -------
311 title : str
312 The title
314 introduction : str
315 The introduction
316 """
317 # Extract paragraphs from the text
318 paragraphs = extract_paragraphs(docstring)
319 if len(paragraphs) == 0: 319 ↛ 320line 319 didn't jump to line 320, because the condition on line 319 was never true
320 raise ExtensionError(
321 f"Example docstring should have a header for the example title. "
322 f"Please check the example file:\n {script.script_file}\n"
323 )
325 # Title is the first paragraph with any RST/Markdown title chars
326 # removed, i.e. lines that consist of (3 or more of the same) 7-bit
327 # non-ASCII chars.
328 # This conditional is not perfect but should hopefully be good enough.
329 title_paragraph = paragraphs[0]
330 match = FIRST_NON_MARKER_WITHOUT_HASH.search(title_paragraph)
331 if match is None: 331 ↛ 332line 331 didn't jump to line 332, because the condition on line 331 was never true
332 raise ExtensionError(f"Could not find a title in first paragraph:\n{title_paragraph}")
334 title = match.group(2).strip()
336 # Use the title if no other paragraphs are provided
337 intro_paragraph = title if len(paragraphs) < 2 else paragraphs[1]
339 # Concatenate all lines of the first paragraph
340 intro = re.sub("\n", " ", intro_paragraph)
341 intro = _sanitize_md(intro)
343 # Truncate at 95 chars
344 if len(intro) > 95:
345 intro = intro[:95] + "..."
347 return title, intro
350def create_thumb_from_image(script: GalleryScript, src_image_path: Path) -> Path:
351 """Create a thumbnail image from the `src_image_path`.
353 Parameters
354 ----------
355 script : GalleryScript
356 The gallery script.
358 src_image_path : Path
359 The source image path, with some flexibility about the extension.
360 TODO do we actually need this flexibility here ?
362 Returns
363 -------
364 actual_thumb_file : Path
365 The actual thumbnail file generated.
366 """
367 try:
368 # Find the image, with flexibility about the actual extenstion ('png', 'svg', 'jpg', 'gif' are supported)
369 src_image_path, ext = _find_image_ext(src_image_path)
370 except ImageNotFoundError:
371 # The source image does not exist !
372 try:
373 # Does a thumbnail already exist ? with extenstion ('png', 'svg', 'jpg', 'gif')
374 thumb_file, ext = _find_image_ext(script.get_thumbnail_file(".png"))
375 # Yes - let's assume this one will suit the needs
376 return thumb_file
377 except ImageNotFoundError:
378 # Create something to replace the thumbnail
379 default_thumb_path = script.gallery_conf.get("default_thumb_file")
380 if default_thumb_path is None: 380 ↛ 383line 380 didn't jump to line 383, because the condition on line 380 was never false
381 default_thumb_path = os.path.join(glr_path_static(), "no_image.png")
383 src_image_path, ext = _find_image_ext(Path(default_thumb_path))
385 # Now let's create the thumbnail.
386 # - First Make sure the thumb dir exists
387 script.gallery.make_thumb_dir()
389 # - Then create the thum file by copying the src image, possibly rescaling it.
390 thumb_file = script.get_thumbnail_file(ext)
391 if ext in (".svg", ".gif"):
392 # No need to rescale image
393 copyfile(src_image_path, thumb_file)
394 else:
395 # Need to rescale image
396 max_width, max_hegiht = script.gallery_conf["thumbnail_size"]
397 rescale_image(
398 in_file=src_image_path,
399 out_file=thumb_file,
400 max_width=max_width,
401 max_height=max_hegiht,
402 )
403 if "thumbnails" in script.gallery_conf["compress_images"]: 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true
404 optipng(thumb_file, script.gallery_conf["compress_images_args"])
406 return thumb_file
409def generate(gallery: GalleryBase, seen_backrefs: Set) -> Tuple[str, str, str, List[GalleryScriptResults]]:
410 """
411 Generate the gallery md for an example directory, including the index.
413 Parameters
414 ----------
415 gallery : GalleryBase
416 The gallery or subgallery to process
418 seen_backrefs : Set
419 Backrefs seen so far.
421 Returns
422 -------
423 title : str
424 The gallery title, that is, the title of the readme file.
426 root_subtitle : str
427 The gallery suptitle that will be used in case the gallery has subsections.
429 index_md : str
430 The markdown to include in the global gallery readme.
432 results : List[GalleryScriptResults]
433 A list of processing results for all scripts in this gallery.
434 """
435 # Read the gallery readme and add it to the index
436 readme_contents = gallery.readme_file.read_text(encoding="utf-8")
437 readme_title = extract_readme_title(gallery.readme_file, readme_contents)
438 if gallery.has_subsections():
439 # parse and try to also extract the last subtitle
440 last_readme_subtitle = extract_readme_last_subtitle(gallery.readme_file, readme_contents)
441 else:
442 # Dont look for the last subtitle
443 last_readme_subtitle = None
445 # Create the destination dir if needed
446 gallery.make_generated_dir()
448 all_thumbnail_entries = []
449 results = []
451 for script in tqdm(gallery.scripts, desc=f"generating gallery for {gallery.generated_dir}... "):
452 # Generate all files related to this example: download file, jupyter notebook, pickle, markdown...
453 script_results = generate_file_md(script=script, seen_backrefs=seen_backrefs)
454 results.append(script_results)
456 # Create the thumbnails-containing div <div class="mkd-glr-thumbcontainer" ...> to place in the readme
457 thumb_div = _thumbnail_div(script_results)
458 all_thumbnail_entries.append(thumb_div)
460 # Write the gallery summary index.md
461 index_md = f"""<!-- {str(gallery.generated_dir_rel_project).replace(os.path.sep, '_')} -->
463{readme_contents}
465{"".join(all_thumbnail_entries)}
466<div class="mkd-glr-clear"></div>
469"""
470 # Note: the "clear" is to disable floating elements again, now that the gallery section is over.
472 return readme_title, last_readme_subtitle, index_md, results
475def is_failing_example(script: GalleryScript):
476 return script.src_py_file in script.gallery_conf["failing_examples"]
479def handle_exception(exc_info, script: GalleryScript):
480 """Trim and format exception, maybe raise error, etc."""
481 from .gen_gallery import _expected_failing_examples
483 etype, exc, tb = exc_info
484 stack = traceback.extract_tb(tb)
485 # The full traceback will look something like:
486 #
487 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_single.py...
488 # mem_max, _ = gallery_conf['call_memory'](
489 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_galler...
490 # mem, out = memory_usage(func, max_usage=True, retval=True,
491 # File "/home/larsoner/.local/lib/python3.8/site-packages/memory_profi...
492 # returned = f(*args, **kw)
493 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_single.py...
494 # exec(self.code, self.fake_main.__dict__)
495 # File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/tests/tiny...
496 # raise RuntimeError('some error')
497 # RuntimeError: some error
498 #
499 # But we should trim these to just the relevant trace at the user level,
500 # so we inspect the traceback to find the start and stop points.
501 start = 0
502 stop = len(stack)
503 root = os.path.dirname(__file__) + os.sep
504 for ii, s in enumerate(stack, 1):
505 # Trim our internal stack
506 if s.filename.startswith(root + "gen_gallery.py") and s.name == "call_memory":
507 start = max(ii, start)
508 elif s.filename.startswith(root + "gen_single.py"):
509 # SyntaxError
510 if s.name == "execute_code_block" and ("compile(" in s.line or "save_figures" in s.line): 510 ↛ 511line 510 didn't jump to line 511, because the condition on line 510 was never true
511 start = max(ii, start)
512 # Any other error
513 elif s.name == "__call__":
514 start = max(ii, start)
515 # Our internal input() check
516 elif s.name == "_check_input" and ii == len(stack): 516 ↛ 517line 516 didn't jump to line 517, because the condition on line 516 was never true
517 stop = ii - 1
518 stack = stack[start:stop]
520 formatted_exception = "Traceback (most recent call last):\n" + "".join(
521 traceback.format_list(stack) + traceback.format_exception_only(etype, exc)
522 )
524 src_file = script.src_py_file
525 expected = src_file in _expected_failing_examples(
526 gallery_conf=script.gallery_conf,
527 mkdocs_conf=script.gallery.all_info.mkdocs_conf,
528 )
529 if expected: 529 ↛ 534line 529 didn't jump to line 534, because the condition on line 529 was never false
530 # func, color = logger.info, 'blue'
531 func = logger.info
532 else:
533 # func, color = logger.warning, 'red'
534 func = logger.warning
535 func(f"{src_file} failed to execute correctly: {formatted_exception}") # , color=color)
537 except_md = codestr2md(formatted_exception, lang="pytb", is_exc=True)
539 # Ensure it's marked as our style: this is now already done in codestr2md
540 # except_md = "{: .mkd-glr-script-out }\n\n" + except_md
541 return except_md, formatted_exception
544# Adapted from github.com/python/cpython/blob/3.7/Lib/warnings.py
545def _showwarning(message, category, filename, lineno, file=None, line=None):
546 if file is None:
547 file = sys.stderr
548 if file is None:
549 # sys.stderr is None when run with pythonw.exe:
550 # warnings get lost
551 return
552 text = warnings.formatwarning(message, category, filename, lineno, line)
553 try:
554 file.write(text)
555 except OSError:
556 # the file (probably stderr) is invalid - this warning gets lost.
557 pass
560@contextlib.contextmanager
561def patch_warnings():
562 """Patch warnings.showwarning to actually write out the warning."""
563 # Sphinx or logging or someone is patching warnings, but we want to
564 # capture them, so let's patch over their patch...
565 orig_showwarning = warnings.showwarning
566 try:
567 warnings.showwarning = _showwarning
568 yield
569 finally:
570 warnings.showwarning = orig_showwarning
573class _exec_once(object):
574 """Deal with memory_usage calling functions more than once (argh)."""
576 def __init__(self, code, fake_main):
577 self.code = code
578 self.fake_main = fake_main
579 self.run = False
581 def __call__(self):
582 if not self.run: 582 ↛ exitline 582 didn't return from function '__call__', because the condition on line 582 was never false
583 self.run = True
584 old_main = sys.modules.get("__main__", None)
585 with patch_warnings():
586 sys.modules["__main__"] = self.fake_main
587 try:
588 exec(self.code, self.fake_main.__dict__) # noqa # our purpose is to execute code :)
589 finally:
590 if old_main is not None: 590 ↛ exitline 590 didn't return from function '__call__', because the condition on line 590 was never false
591 sys.modules["__main__"] = old_main
594def _get_memory_base(gallery_conf):
595 """Get the base amount of memory used by running a Python process."""
596 if not gallery_conf["plot_gallery"]:
597 return 0.0
598 # There might be a cleaner way to do this at some point
599 from memory_profiler import memory_usage
601 if sys.platform in ("win32", "darwin"):
602 sleep, timeout = (1, 2)
603 else:
604 sleep, timeout = (0.5, 1)
605 proc = subprocess.Popen(
606 [sys.executable, "-c", "import time, sys; time.sleep(%s); sys.exit(0)" % sleep],
607 close_fds=True,
608 )
609 memories = memory_usage(proc, interval=1e-3, timeout=timeout)
610 kwargs = dict(timeout=timeout) if sys.version_info >= (3, 5) else {}
611 proc.communicate(**kwargs)
612 # On OSX sometimes the last entry can be None
613 memories = [mem for mem in memories if mem is not None] + [0.0]
614 memory_base = max(memories)
615 return memory_base
618def _ast_module():
619 """Get ast.Module function, dealing with:
620 https://bugs.python.org/issue35894"""
621 if sys.version_info >= (3, 8): 621 ↛ 624line 621 didn't jump to line 624, because the condition on line 621 was never false
622 ast_Module = partial(ast.Module, type_ignores=[])
623 else:
624 ast_Module = ast.Module
625 return ast_Module
628def _check_reset_logging_tee(src_file: Path):
629 # Helper to deal with our tests not necessarily calling parse_and_execute
630 # but rather execute_code_block directly
631 if isinstance(sys.stdout, _LoggingTee): 631 ↛ 634line 631 didn't jump to line 634, because the condition on line 631 was never false
632 logging_tee = sys.stdout
633 else:
634 logging_tee = _LoggingTee(src_file)
635 logging_tee.set_std_and_reset_position()
636 return logging_tee
639def _exec_and_get_memory(compiler, ast_Module, code_ast, script: GalleryScript):
640 """Execute ast, capturing output if last line is expression and get max memory usage."""
642 src_file = script.src_py_file.as_posix()
644 # capture output if last line is expression
645 is_last_expr = False
647 if len(code_ast.body) and isinstance(code_ast.body[-1], ast.Expr):
648 is_last_expr = True
649 last_val = code_ast.body.pop().value
650 # exec body minus last expression
651 mem_body, _ = script.gallery_conf["call_memory"](
652 _exec_once(compiler(code_ast, src_file, "exec"), script.run_vars.fake_main)
653 )
654 # exec last expression, made into assignment
655 body = [ast.Assign(targets=[ast.Name(id="___", ctx=ast.Store())], value=last_val)]
656 last_val_ast = ast_Module(body=body)
657 ast.fix_missing_locations(last_val_ast)
658 mem_last, _ = script.gallery_conf["call_memory"](
659 _exec_once(compiler(last_val_ast, src_file, "exec"), script.run_vars.fake_main)
660 )
661 mem_max = max(mem_body, mem_last)
662 else:
663 mem_max, _ = script.gallery_conf["call_memory"](
664 _exec_once(compiler(code_ast, src_file, "exec"), script.run_vars.fake_main)
665 )
667 return is_last_expr, mem_max
670def _get_last_repr(gallery_conf, ___):
671 """Get a repr of the last expression, using first method in 'capture_repr'
672 available for the last expression."""
673 for meth in gallery_conf["capture_repr"]: 673 ↛ 687line 673 didn't jump to line 687, because the loop on line 673 didn't complete
674 try:
675 last_repr = getattr(___, meth)()
676 # for case when last statement is print()
677 if last_repr is None or last_repr == "None":
678 repr_meth = None
679 else:
680 repr_meth = meth
681 except Exception:
682 last_repr = None
683 repr_meth = None
684 else:
685 if isinstance(last_repr, str): 685 ↛ 673line 685 didn't jump to line 673, because the condition on line 685 was never false
686 break
687 return last_repr, repr_meth
690def _get_code_output(is_last_expr, script: GalleryScript, logging_tee, images_md):
691 """Obtain standard output and html output in md."""
693 example_globals = script.run_vars.example_globals
694 gallery_conf = script.gallery_conf
696 last_repr = None
697 repr_meth = None
698 if is_last_expr:
699 # capture the last repr variable
700 ___ = example_globals["___"]
701 ignore_repr = False
702 if gallery_conf["ignore_repr_types"]: 702 ↛ 703line 702 didn't jump to line 703, because the condition on line 702 was never true
703 ignore_repr = re.search(gallery_conf["ignore_repr_types"], str(type(___)))
704 if gallery_conf["capture_repr"] != () and not ignore_repr: 704 ↛ 707line 704 didn't jump to line 707, because the condition on line 704 was never false
705 last_repr, repr_meth = _get_last_repr(gallery_conf, ___)
707 captured_std = logging_tee.output.getvalue().expandtabs()
709 # normal string output
710 if repr_meth in ["__repr__", "__str__"] and last_repr:
711 captured_std = f"{captured_std}\n{last_repr}"
713 if captured_std and not captured_std.isspace():
714 captured_std = CODE_OUTPUT.format(captured_std)
715 else:
716 captured_std = ""
718 # give html output its own header
719 if repr_meth == "_repr_html_":
720 captured_html = HTML_HEADER.format(indent(last_repr, " " * 4))
721 else:
722 captured_html = ""
724 code_output = f"""
725{images_md}
727{captured_std}
729{captured_html}
731"""
732 return code_output
735def _reset_cwd_syspath(cwd, path_to_remove):
736 """Reset current working directory to `cwd` and remove `path_to_remove` from `sys.path`."""
737 if path_to_remove in sys.path:
738 sys.path.remove(path_to_remove)
739 os.chdir(cwd)
742def _parse_code(bcontent, src_file, *, compiler_flags):
743 code_ast = compile(bcontent, src_file, "exec", compiler_flags | ast.PyCF_ONLY_AST, dont_inherit=1)
744 if _needs_async_handling(bcontent, src_file, compiler_flags=compiler_flags): 744 ↛ 745line 744 didn't jump to line 745, because the condition on line 744 was never true
745 code_ast = _apply_async_handling(code_ast, compiler_flags=compiler_flags)
746 return code_ast
749def _needs_async_handling(bcontent, src_file, *, compiler_flags) -> bool:
750 try:
751 compile(bcontent, src_file, "exec", compiler_flags, dont_inherit=1)
752 except SyntaxError as error:
753 # mkdocs-gallery supports top-level async code similar to jupyter notebooks.
754 # Without handling, this will raise a SyntaxError. In such a case, we apply a
755 # minimal async handling and try again. If the error persists, we bubble it up
756 # and let the caller handle it.
757 try:
758 compile(
759 f"async def __async_wrapper__():\n{indent(bcontent, ' ' * 4)}",
760 src_file,
761 "exec",
762 compiler_flags,
763 dont_inherit=1,
764 )
765 except SyntaxError:
766 # Raise the original error to avoid leaking the internal async handling to
767 # generated output.
768 raise error from None
769 else:
770 return True
771 else:
772 return False
775def _apply_async_handling(code_ast, *, compiler_flags):
776 async_handling = compile(
777 dedent(
778 """
779 async def __async_wrapper__():
780 # original AST goes here
781 return locals()
782 import asyncio as __asyncio__
783 __async_wrapper_locals__ = __asyncio__.run(__async_wrapper__())
784 __async_wrapper_result__ = __async_wrapper_locals__.pop("__async_wrapper_result__", None)
785 globals().update(__async_wrapper_locals__)
786 __async_wrapper_result__
787 """
788 ),
789 "<_apply_async_handling()>",
790 "exec",
791 compiler_flags | ast.PyCF_ONLY_AST,
792 dont_inherit=1,
793 )
795 *original_body, last_node = code_ast.body
796 if isinstance(last_node, ast.Expr):
797 last_node = ast.Assign(
798 targets=[ast.Name(id="__async_wrapper_result__", ctx=ast.Store())], value=last_node.value
799 )
800 original_body.append(last_node)
802 async_wrapper = async_handling.body[0]
803 async_wrapper.body = [*original_body, *async_wrapper.body]
805 return ast.fix_missing_locations(async_handling)
808def execute_code_block(compiler, block, script: GalleryScript):
809 """Execute the code block of the example file.
811 Parameters
812 ----------
813 compiler : codeop.Compile
814 Compiler to compile AST of code block.
816 block : List[Tuple[str, str, int]]
817 List of Tuples, each Tuple contains label ('text' or 'code'),
818 the corresponding content string of block and the leading line number.
820 script: GalleryScript
821 The gallery script
823 Returns
824 -------
825 code_output : str
826 Output of executing code in md.
827 """
828 # if script.run_vars.example_globals is None: # testing shortcut
829 # script.run_vars.example_globals = script.run_vars.fake_main.__dict__
831 blabel, bcontent, lineno = block
833 # If example is not suitable to run anymore, skip executing its blocks
834 if script.run_vars.stop_executing or blabel == "text":
835 return ""
837 cwd = os.getcwd()
838 # Redirect output to stdout
839 src_file = script.src_py_file
840 logging_tee = _check_reset_logging_tee(src_file)
841 assert isinstance(logging_tee, _LoggingTee) # noqa
843 # First cd in the original example dir, so that any file
844 # created by the example get created in this directory
845 os.chdir(src_file.parent)
847 # Add the example dir to the path temporarily (will be removed after execution)
848 new_path = os.getcwd()
849 sys.path.append(new_path)
851 # Save figures unless there is a `mkdocs_gallery_defer_figures` flag
852 match = re.search(r"^[\ \t]*#\s*mkdocs_gallery_defer_figures[\ \t]*\n?", bcontent, re.MULTILINE)
853 need_save_figures = match is None
855 try:
856 ast_Module = _ast_module()
857 code_ast = _parse_code(bcontent, src_file, compiler_flags=compiler.flags)
858 ast.increment_lineno(code_ast, lineno - 1)
860 is_last_expr, mem_max = _exec_and_get_memory(compiler, ast_Module, code_ast, script=script)
861 script.run_vars.memory_used_in_blocks.append(mem_max)
863 # This should be inside the try block, e.g., in case of a savefig error
864 logging_tee.restore_std()
865 if need_save_figures: 865 ↛ 869line 865 didn't jump to line 869, because the condition on line 865 was never false
866 need_save_figures = False
867 images_md = save_figures(block, script)
868 else:
869 images_md = ""
871 except Exception:
872 logging_tee.restore_std()
873 except_md, formatted_exception = handle_exception(sys.exc_info(), script)
875 # Breaks build on first example error
876 if script.gallery_conf["abort_on_example_error"]: 876 ↛ 877line 876 didn't jump to line 877, because the condition on line 876 was never true
877 raise
879 # Stores failing file
880 script.gallery_conf["failing_examples"][src_file] = formatted_exception
882 # Stop further execution on that script
883 script.run_vars.stop_executing = True
885 code_output = "\n{0}\n\n\n\n".format(except_md)
886 # still call this even though we won't use the images so that
887 # figures are closed
888 if need_save_figures: 888 ↛ 895line 888 didn't jump to line 895, because the condition on line 888 was never false
889 save_figures(block, script)
890 else:
891 _reset_cwd_syspath(cwd, new_path)
893 code_output = _get_code_output(is_last_expr, script, logging_tee, images_md)
894 finally:
895 _reset_cwd_syspath(cwd, new_path)
896 logging_tee.restore_std() 896 ↛ exitline 896 didn't except from function 'execute_code_block', because the raise on line 877 wasn't executed
898 # Sanitize ANSI escape characters from MD output
899 ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
900 code_output = ansi_escape.sub("", code_output)
902 return code_output
905def _check_input(prompt=None):
906 raise ExtensionError("Cannot use input() builtin function in mkdocs-gallery examples")
909def parse_and_execute(script: GalleryScript, script_blocks):
910 """Execute and capture output from python script already in block structure
912 Parameters
913 ----------
914 script : GalleryScript
915 The script
917 script_blocks : list
918 (label, content, line_number)
919 List where each element is a tuple with the label ('text' or 'code'),
920 the corresponding content string of block and the leading line number
922 Returns
923 -------
924 output_blocks : list
925 List of strings where each element is the restructured text
926 representation of the output of each block
928 time_elapsed : float
929 Time elapsed during execution
931 memory_used : float
932 Memory used during execution
933 """
934 # Examples may contain if __name__ == '__main__' guards for in example scikit-learn if the example uses
935 # multiprocessing. Here we create a new __main__ module, and temporarily change sys.modules when running our example
936 fake_main = importlib.util.module_from_spec(importlib.util.spec_from_loader("__main__", None))
937 script.run_vars.fake_main = fake_main
939 example_globals = fake_main.__dict__
940 example_globals.update(
941 {
942 # A lot of examples contains 'print(__doc__)' for example in
943 # scikit-learn so that running the example prints some useful
944 # information. Because the docstring has been separated from
945 # the code blocks in mkdocs-gallery, __doc__ is actually
946 # __builtin__.__doc__ in the execution context and we do not
947 # want to print it
948 "__doc__": "",
949 # Don't ever support __file__: Issues #166 #212
950 # Don't let them use input()
951 "input": _check_input,
952 }
953 )
954 script.run_vars.example_globals = example_globals
956 # Manipulate the sys.argv before running the example
957 # See https://github.com/sphinx-gallery/sphinx-gallery/pull/252
959 # Remember the original argv so that we can put them back after run
960 argv_orig = sys.argv[:]
962 # Remember the original sys.path so that we can reset it after run
963 sys_path_orig = deepcopy(sys.path)
965 # Python file is the original one (not the copy for download)
966 sys.argv[0] = script.src_py_file.as_posix()
968 # Allow users to provide additional args through the 'reset_argv' option
969 sys.argv[1:] = script.gallery_conf["reset_argv"](script)
971 # Perform a garbage collection before starting so that perf kpis are accurate (memory and time)
972 gc.collect()
974 # Initial memory used
975 memory_start, _ = script.gallery_conf["call_memory"](lambda: None)
976 script.run_vars.memory_used_in_blocks = [memory_start] # include at least one entry to avoid max() ever failing
978 t_start = time()
979 compiler = codeop.Compile()
981 # Execute block by block
982 output_blocks = list()
983 with _LoggingTee(script.src_py_file) as logging_tee:
984 for block in script_blocks:
985 logging_tee.set_std_and_reset_position()
986 output_blocks.append(execute_code_block(compiler, block, script))
988 # Compute the elapsed time
989 time_elapsed = time() - t_start
991 # Set back the sys argv
992 sys.argv = argv_orig
994 # Set back the sys path
995 sys.path = sys_path_orig
997 # Write md5 checksum if the example was meant to run (no-plot shall not cache md5sum) and has built correctly
998 script.write_final_md5_file()
1000 # Declare the example as "passing"
1001 script.gallery_conf["passing_examples"].append(script)
1003 script.run_vars.memory_delta = max(script.run_vars.memory_used_in_blocks) - memory_start
1004 memory_used = script.gallery_conf["memory_base"] + script.run_vars.memory_delta
1006 return output_blocks, time_elapsed, memory_used
1009def generate_file_md(script: GalleryScript, seen_backrefs=None) -> GalleryScriptResults:
1010 """Generate the md file for a given example.
1012 Parameters
1013 ----------
1014 script : GalleryScript
1015 The script to process
1017 seen_backrefs : set
1018 The seen backreferences.
1020 Returns
1021 -------
1022 result: FileResult
1023 The result of running this script
1024 """
1025 seen_backrefs = set() if seen_backrefs is None else seen_backrefs
1027 # Extract the contents of the script
1028 file_conf, script_blocks, node = split_code_and_text_blocks(script.src_py_file, return_node=True)
1030 # Extract the title and introduction from the module docstring and save the title in the object
1031 script.title, intro = extract_intro_and_title(docstring=script_blocks[0][1], script=script)
1033 # Copy source python script to target folder if it is not there/up to date, so that it can be served/downloaded
1034 # Note: surprisingly this uses a md5 too, but not the final .md5 persisted on disk.
1035 script.make_dwnld_py_file()
1037 # Can the script be entirely skipped (both doc generation and execution) ?
1038 if not script.has_changed_wrt_persisted_md5():
1039 # A priori we can...
1040 skip_and_return = True
1042 # ...however for executables (not shared modules) we might need to run anyway because of config
1043 if script.is_executable_example(): 1043 ↛ 1055line 1043 didn't jump to line 1055, because the condition on line 1043 was never false
1044 if script.gallery_conf["run_stale_examples"]: 1044 ↛ 1046line 1044 didn't jump to line 1046, because the condition on line 1044 was never true
1045 # Run anyway because config says so.
1046 skip_and_return = False
1047 else:
1048 # Add the example to the "stale examples" before returning
1049 script.gallery_conf["stale_examples"].append(script.dwnld_py_file)
1050 # If expected to fail, let's remove it from the 'expected_failing_examples' list,
1051 # assuming it did when previously executed
1052 if script.src_py_file in script.gallery_conf["expected_failing_examples"]:
1053 script.gallery_conf["expected_failing_examples"].remove(script.src_py_file)
1055 if skip_and_return: 1055 ↛ 1062line 1055 didn't jump to line 1062, because the condition on line 1055 was never false
1056 # Return with 0 exec time and mem usage, and the existing thumbnail
1057 thumb_source_path = script.get_thumbnail_source(file_conf)
1058 thumb_file = create_thumb_from_image(script, thumb_source_path)
1059 return GalleryScriptResults(script=script, intro=intro, exec_time=0.0, memory=0.0, thumb=thumb_file)
1061 # Reset matplotlib, seaborn, etc. if needed
1062 if script.is_executable_example():
1063 clean_modules(gallery_conf=script.gallery_conf, file=script.src_py_file)
1065 # Init the runtime vars. Create the images directory and init the image files template
1066 script.init_before_processing()
1068 if script.is_executable_example():
1069 # Note: this writes the md5 checksum if the example was meant to run
1070 output_blocks, time_elapsed, memory_used = parse_and_execute(script, script_blocks)
1071 logger.debug(f"{script.src_py_file} ran in : {time_elapsed:.2g} seconds\n")
1072 else:
1073 output_blocks = [""] * len(script_blocks)
1074 time_elapsed = memory_used = 0.0 # don't let the output change
1075 logger.debug(f"{script.src_py_file} parsed (not executed)\n")
1077 # Create as many dummy images as required if needed (default none) so that references to script images
1078 # Can still work, even if the script was not executed (in development mode typically, to go fast).
1079 # See https://sphinx-gallery.github.io/stable/configuration.html#generating-dummy-images
1080 nb_dummy_images_to_generate = file_conf.get("dummy_images", None)
1081 if nb_dummy_images_to_generate is not None: 1081 ↛ 1082line 1081 didn't jump to line 1082, because the condition on line 1081 was never true
1082 if type(nb_dummy_images_to_generate) is not int:
1083 raise ExtensionError("mkdocs_gallery: 'dummy_images' setting is not a number, got {dummy_image!r}")
1085 stock_img = os.path.join(glr_path_static(), "no_image.png")
1086 script.generate_n_dummy_images(img=stock_img, nb=nb_dummy_images_to_generate)
1088 # Remove the mkdocs-gallery configuration comments from the script if needed
1089 if script.gallery_conf["remove_config_comments"]: 1089 ↛ 1090line 1089 didn't jump to line 1090, because the condition on line 1089 was never true
1090 script_blocks = [
1091 (label, remove_config_comments(content), line_number) for label, content, line_number in script_blocks
1092 ]
1094 # Remove final empty block, which can occur after config comments are removed
1095 if script_blocks[-1][1].isspace(): 1095 ↛ 1096line 1095 didn't jump to line 1096, because the condition on line 1095 was never true
1096 script_blocks = script_blocks[:-1]
1097 output_blocks = output_blocks[:-1]
1099 # Generate the markdown string containing the script prose, code and output.
1100 example_md = generate_md_from_blocks(script_blocks, output_blocks, file_conf, script.gallery_conf)
1102 # Write the generated markdown file
1103 md_header, md_footer = get_example_md_wrapper(script, time_elapsed, memory_used)
1104 full_md = md_header + example_md + md_footer
1105 script.save_md_example(full_md)
1107 # Create the image thumbnail for the gallery summary
1108 if is_failing_example(script):
1109 # Failing example thumbnail
1110 thumb_source_path = Path(os.path.join(glr_path_static(), "broken_example.png"))
1111 else:
1112 # Get the thumbnail source image, possibly from config
1113 thumb_source_path = script.get_thumbnail_source(file_conf)
1115 thumb_file = create_thumb_from_image(script, thumb_source_path)
1117 # Generate the jupyter notebook
1118 example_nb = jupyter_notebook(script, script_blocks)
1119 ipy_file = _new_file(script.ipynb_file)
1120 save_notebook(example_nb, ipy_file)
1121 _replace_by_new_if_needed(ipy_file, md5_mode="t")
1123 # Write names
1124 if script.gallery_conf["inspect_global_variables"]: 1124 ↛ 1127line 1124 didn't jump to line 1127, because the condition on line 1124 was never false
1125 global_variables = script.run_vars.example_globals
1126 else:
1127 global_variables = None
1129 # TODO dig in just in case
1130 example_code_obj = identify_names(script_blocks, global_variables, node)
1131 if example_code_obj:
1132 # Write a pickle file (.pickle) containing `example_code_obj`
1133 codeobj_fname = _new_file(script.codeobj_file)
1134 with open(codeobj_fname, "wb") as fid:
1135 pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL)
1136 _replace_by_new_if_needed(codeobj_fname)
1138 backrefs = set(
1139 "{module_short}.{name}".format(**cobj)
1140 for cobjs in example_code_obj.values()
1141 for cobj in cobjs
1142 if cobj["module"].startswith(script.gallery_conf["doc_module"])
1143 )
1145 # Create results object
1146 res = GalleryScriptResults(
1147 script=script,
1148 intro=intro,
1149 exec_time=time_elapsed,
1150 memory=memory_used,
1151 thumb=thumb_file,
1152 )
1154 # Write backreferences if required
1155 if script.gallery_conf["backreferences_dir"] is not None: 1155 ↛ 1158line 1155 didn't jump to line 1158, because the condition on line 1155 was never false
1156 _write_backreferences(backrefs, seen_backrefs, script_results=res)
1158 return res
1161# TODO the note should only appear in html mode. (.. only:: html)
1162# TODO maybe remove as much as possible the css for now?
1163EXAMPLE_HEADER = """
1164<!--
1165 DO NOT EDIT.
1166 THIS FILE WAS AUTOMATICALLY GENERATED BY mkdocs-gallery.
1167 TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
1168 "{pyfile_to_edit}"
1169 LINE NUMBERS ARE GIVEN BELOW.
1170-->
1172!!! note
1174 Click [here](#download_links)
1175 to download the full example code{opt_binder_text}
1177""" # TODO there was a {{: .mkd-glr-example-title }} for the title but is it useful ?
1178MD_BLOCK_HEADER = """\
1179<!-- GENERATED FROM PYTHON SOURCE LINES {0}-{1} -->
1181"""
1184def generate_md_from_blocks(script_blocks, output_blocks, file_conf, gallery_conf) -> str:
1185 """Generate the md string containing the script prose, code and output.
1187 Parameters
1188 ----------
1189 script_blocks : list
1190 (label, content, line_number)
1191 List where each element is a tuple with the label ('text' or 'code'),
1192 the corresponding content string of block and the leading line number
1194 output_blocks : list
1195 List of strings where each element is the restructured text
1196 representation of the output of each block
1198 file_conf : dict
1199 File-specific settings given in source file comments as:
1200 ``# mkdocs_gallery_<name> = <value>``
1202 gallery_conf : dict
1203 Contains the configuration of mkdocs-gallery
1205 Returns
1206 -------
1207 out : str
1208 The resulting markdown page.
1209 """
1211 # A simple example has two blocks: one for the
1212 # example introduction/explanation and one for the code
1213 is_example_notebook_like = len(script_blocks) > 2
1214 example_md = ""
1215 for bi, ((blabel, bcontent, lineno), code_output) in enumerate(zip(script_blocks, output_blocks)):
1216 # do not add comment to the title block (bi=0), otherwise the linking does not work properly
1217 if bi > 0:
1218 example_md += MD_BLOCK_HEADER.format(lineno, lineno + bcontent.count("\n"))
1220 if blabel == "code":
1221 if not file_conf.get("line_numbers", gallery_conf.get("line_numbers", False)): 1221 ↛ 1224line 1221 didn't jump to line 1224, because the condition on line 1221 was never false
1222 lineno = None
1224 code_md = codestr2md(bcontent, lang=gallery_conf["lang"], lineno=lineno) + "\n"
1225 if is_example_notebook_like:
1226 example_md += code_md
1227 example_md += code_output
1228 else:
1229 example_md += code_output
1230 if "mkd-glr-script-out" in code_output:
1231 # Add some vertical space after output
1232 example_md += "\n\n<br />\n\n" # "|\n\n"
1233 example_md += code_md
1234 else:
1235 block_separator = "\n\n" if not bcontent.endswith("\n") else "\n"
1236 example_md += bcontent + block_separator
1238 return example_md
1241def get_example_md_wrapper(script: GalleryScript, time_elapsed: float, memory_used: float) -> Tuple[str, str]:
1242 """Creates the headers and footers for the example markdown. Returns a template
1244 Parameters
1245 ----------
1246 script : GalleryScript
1247 The script for which to generate the md.
1249 time_elapsed : float
1250 Time elapsed in seconds while executing file
1252 memory_used : float
1253 Additional memory used during the run.
1255 Returns
1256 -------
1257 md_before : str
1258 Part of the final markdown that goes before the notebook / python script.
1260 md_after : str
1261 Part of the final markdown that goes after the notebook / python script.
1262 """
1263 # Check binder configuration
1264 binder_conf = check_binder_conf(script.gallery_conf.get("binder"))
1265 use_binder = len(binder_conf) > 0
1267 # Write header
1268 src_relative = script.src_py_file_rel_project.as_posix()
1269 binder_text = " or to run this example in your browser via Binder" if use_binder else ""
1270 md_before = EXAMPLE_HEADER.format(pyfile_to_edit=src_relative, opt_binder_text=binder_text)
1272 # Footer
1273 md_after = ""
1274 # Report Time and Memory
1275 if time_elapsed >= script.gallery_conf["min_reported_time"]: 1275 ↛ 1279line 1275 didn't jump to line 1279, because the condition on line 1275 was never false
1276 time_m, time_s = divmod(time_elapsed, 60)
1277 md_after += TIMING_CONTENT.format(time_m, time_s)
1279 if script.gallery_conf["show_memory"]: 1279 ↛ 1280line 1279 didn't jump to line 1280, because the condition on line 1279 was never true
1280 md_after += f"**Estimated memory usage:** {memory_used:.0f} MB\n\n"
1282 # Download buttons
1283 # - Generate a binder URL if specified
1284 binder_badge_md = gen_binder_md(script, binder_conf) if use_binder else ""
1285 # - Rely on mkdocs-material for the icon
1286 icon = ":fontawesome-solid-download:"
1287 # - Generate the download buttons
1288 # TODO why aren't they centered actually ? does .center work ?
1289 md_after += f"""
1290<div id="download_links"></div>
1292{binder_badge_md}
1294[{icon} Download Python source code: {script.dwnld_py_file.name}](./{script.dwnld_py_file.name}){{ .md-button .center}}
1296[{icon} Download Jupyter notebook: {script.ipynb_file.name}](./{script.ipynb_file.name}){{ .md-button .center}}
1297"""
1299 # Add the "generated by mkdocs-gallery" footer
1300 md_after += MKD_GLR_SIG
1302 return md_before, md_after