Coverage for src/mkdocs_gallery/notebook.py: 43%
104 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-15 17:10 +0000
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/mkdocs-gallery>
3#
4# Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5# License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6"""
7Parser for Jupyter notebooks
8"""
10from __future__ import absolute_import, division, print_function
12import argparse
13import base64
14import copy
15import json
16import mimetypes
17import os
18import re
19import sys
20from functools import partial
21from pathlib import Path
22from typing import Dict, List
24from . import mkdocs_compatibility
25from .errors import ExtensionError
26from .gen_data_model import GalleryScript
27from .py_source_parser import split_code_and_text_blocks
29logger = mkdocs_compatibility.getLogger("mkdocs-gallery")
32def jupyter_notebook_skeleton():
33 """Returns a dictionary with the elements of a Jupyter notebook"""
34 py_version = sys.version_info
35 notebook_skeleton = {
36 "cells": [],
37 "metadata": {
38 "kernelspec": {
39 "display_name": "Python " + str(py_version[0]),
40 "language": "python",
41 "name": "python" + str(py_version[0]),
42 },
43 "language_info": {
44 "codemirror_mode": {"name": "ipython", "version": py_version[0]},
45 "file_extension": ".py",
46 "mimetype": "text/x-python",
47 "name": "python",
48 "nbconvert_exporter": "python",
49 "pygments_lexer": "ipython" + str(py_version[0]),
50 "version": "{0}.{1}.{2}".format(*sys.version_info[:3]),
51 },
52 },
53 "nbformat": 4,
54 "nbformat_minor": 0,
55 }
56 return notebook_skeleton
59def directive_fun(match, directive):
60 """Helper to fill in directives"""
61 directive_to_alert = dict(note="info", warning="danger")
62 return '<div class="alert alert-{0}"><h4>{1}</h4><p>{2}</p></div>'.format(
63 directive_to_alert[directive], directive.capitalize(), match.group(1).strip()
64 )
67def rst2md(text, gallery_conf, target_dir, heading_levels):
68 """Converts the RST text from the examples docstrings and comments
69 into markdown text for the Jupyter notebooks
71 Parameters
72 ----------
73 text: str
74 RST input to be converted to MD
75 gallery_conf : dict
76 The mkdocs-gallery configuration dictionary.
77 target_dir : str
78 Path that notebook is intended for. Used where relative paths
79 may be required.
80 heading_levels: dict
81 Mapping of heading style ``(over_char, under_char)`` to heading level.
82 Note that ``over_char`` is `None` when only underline is present.
83 """
85 # Characters recommended for use with headings
86 # https://docutils.readthedocs.io/en/sphinx-docs/user/rst/quickstart.html#sections
87 adornment_characters = "=`:.'\"~^_*+#<>-"
88 headings = re.compile(
89 # Start of string or blank line
90 r"(?P<pre>\A|^[ \t]*\n)"
91 # Optional over characters, allowing leading space on heading text
92 r"(?:(?P<over>[{0}])(?P=over)*\n[ \t]*)?"
93 # The heading itself, with at least one non-white space character
94 r"(?P<heading>\S[^\n]*)\n"
95 # Under character, setting to same character if over present.
96 r"(?P<under>(?(over)(?P=over)|[{0}]))(?P=under)*$" r"".format(adornment_characters),
97 flags=re.M,
98 )
100 text = re.sub(
101 headings,
102 lambda match: "{1}{0} {2}".format(
103 "#" * heading_levels[match.group("over", "under")], *match.group("pre", "heading")
104 ),
105 text,
106 )
108 math_eq = re.compile(r"^\.\. math::((?:.+)?(?:\n+^ .+)*)", flags=re.M)
109 text = re.sub(
110 math_eq,
111 lambda match: r"\begin{{align}}{0}\end{{align}}".format(match.group(1).strip()),
112 text,
113 )
114 inline_math = re.compile(r":math:`(.+?)`", re.DOTALL)
115 text = re.sub(inline_math, r"$\1$", text)
117 directives = ("warning", "note")
118 for directive in directives:
119 directive_re = re.compile(r"^\.\. %s::((?:.+)?(?:\n+^ .+)*)" % directive, flags=re.M)
120 text = re.sub(directive_re, partial(directive_fun, directive=directive), text)
122 links = re.compile(r"^ *\.\. _.*:.*$\n", flags=re.M)
123 text = re.sub(links, "", text)
125 refs = re.compile(r":ref:`")
126 text = re.sub(refs, "`", text)
128 contents = re.compile(r"^\s*\.\. contents::.*$(\n +:\S+: *$)*\n", flags=re.M)
129 text = re.sub(contents, "", text)
131 images = re.compile(r"^\.\. image::(.*$)((?:\n +:\S+:.*$)*)\n", flags=re.M)
132 image_opts = re.compile(r"\n +:(\S+): +(.*)$", flags=re.M)
133 text = re.sub(
134 images,
135 lambda match: '<img src="{}"{}>\n'.format(
136 generate_image_src(match.group(1).strip(), gallery_conf, target_dir),
137 re.sub(image_opts, r' \1="\2"', match.group(2) or ""),
138 ),
139 text,
140 )
142 return text
145def generate_image_src(image_path, gallery_conf, target_dir):
146 if re.match(r"https?://", image_path):
147 return image_path
149 if not gallery_conf["notebook_images"]:
150 return "file://" + image_path.lstrip("/")
152 # If absolute path from source directory given
153 if image_path.startswith("/"):
154 # Path should now be relative to source dir, not target dir
155 target_dir = mkdocs_conf["docs_dir"]
156 image_path = image_path.lstrip("/")
157 full_path = os.path.join(target_dir, image_path.replace("/", os.sep))
159 if isinstance(gallery_conf["notebook_images"], str):
160 # Use as prefix e.g. URL
161 prefix = gallery_conf["notebook_images"]
162 rel_path = os.path.relpath(full_path, mkdocs_conf["docs_dir"])
163 return prefix + rel_path.replace(os.sep, "/")
164 else:
165 # True, but not string. Embed as data URI.
166 try:
167 with open(full_path, "rb") as image_file:
168 data = base64.b64encode(image_file.read())
169 except OSError:
170 raise ExtensionError("Unable to open {} to generate notebook data URI" "".format(full_path))
171 mime_type = mimetypes.guess_type(full_path)
172 return "data:{};base64,{}".format(mime_type[0], data.decode("ascii"))
175def jupyter_notebook(script: GalleryScript, script_blocks: List):
176 """Generate a Jupyter notebook file cell-by-cell
178 Parameters
179 ----------
180 script : GalleryScript
181 Script
183 script_blocks : list
184 Script execution cells.
185 """
186 # Grab the possibly custom first and last cells
187 first_cell = script.gallery_conf["first_notebook_cell"]
188 last_cell = script.gallery_conf["last_notebook_cell"]
190 # Initialize with a notebook skeleton
191 work_notebook = jupyter_notebook_skeleton()
193 # Custom first cell
194 if first_cell is not None: 194 ↛ 198line 194 didn't jump to line 198, because the condition on line 194 was never false
195 add_code_cell(work_notebook, first_cell)
197 # Fill the notebook per se
198 fill_notebook(work_notebook, script_blocks)
200 # Custom last cell
201 if last_cell is not None: 201 ↛ 202line 201 didn't jump to line 202, because the condition on line 201 was never true
202 add_code_cell(work_notebook, last_cell)
204 return work_notebook
207def add_code_cell(work_notebook, code):
208 """Add a code cell to the notebook
210 Parameters
211 ----------
212 code : str
213 Cell content
214 """
216 code_cell = {
217 "cell_type": "code",
218 "execution_count": None,
219 "metadata": {"collapsed": False},
220 "outputs": [],
221 "source": [code.strip()],
222 }
223 work_notebook["cells"].append(code_cell)
226def add_markdown_cell(work_notebook, markdown):
227 """Add a markdown cell to the notebook
229 Parameters
230 ----------
231 markdown : str
232 Markdown cell content.
233 """
234 markdown_cell = {"cell_type": "markdown", "metadata": {}, "source": [markdown]}
235 work_notebook["cells"].append(markdown_cell)
238def fill_notebook(work_notebook, script_blocks):
239 """Writes the Jupyter notebook cells
241 If available, uses pypandoc to convert rst to markdown >> not anymore.
243 Parameters
244 ----------
245 script_blocks : list
246 Each list element should be a tuple of (label, content, lineno).
247 """
248 # heading_level_counter = count(start=1)
249 # heading_levels = defaultdict(lambda: next(heading_level_counter))
250 for blabel, bcontent, _lineno in script_blocks:
251 if blabel == "code":
252 add_code_cell(work_notebook, bcontent)
253 else:
254 # if gallery_conf["pypandoc"] is False:
255 # markdown = rst2md(
256 # bcontent + '\n', gallery_conf, target_dir, heading_levels)
257 # else:
258 # import pypandoc
259 # # pandoc automatically addds \n to the end
260 # markdown = pypandoc.convert_text(
261 # bcontent, to='md', format='rst', **gallery_conf["pypandoc"]
262 # )
263 markdown = bcontent + "\n"
264 add_markdown_cell(work_notebook, markdown)
267def save_notebook(work_notebook: Dict, write_file: Path):
268 """Saves the Jupyter work_notebook to write_file"""
269 with open(str(write_file), "w") as out_nb:
270 json.dump(work_notebook, out_nb, indent=2)
273###############################################################################
274# Notebook shell utility
277def python_to_jupyter_cli(args=None, namespace=None):
278 """Exposes the jupyter notebook renderer to the command line
280 Takes the same arguments as ArgumentParser.parse_args
281 """
282 from . import gen_gallery # To avoid circular import
284 parser = argparse.ArgumentParser(description="mkdocs-gallery Notebook converter")
285 parser.add_argument(
286 "python_src_file",
287 nargs="+",
288 help="Input Python file script to convert. " "Supports multiple files and shell wildcards" " (e.g. *.py)",
289 )
290 args = parser.parse_args(args, namespace)
292 for src_file in args.python_src_file:
293 file_conf, blocks = split_code_and_text_blocks(src_file)
294 print("Converting {0}".format(src_file)) # noqa # this is a cli
295 gallery_conf = copy.deepcopy(gen_gallery.DEFAULT_GALLERY_CONF)
296 target_dir = os.path.dirname(src_file)
297 example_nb = jupyter_notebook(blocks, gallery_conf, target_dir)
298 save_notebook(example_nb, get_ipynb_for_py_script(src_file))