mkdocs_gallery/notebook.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/mkdocs-gallery>
3 #
4 # Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5 # License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6 """
7 Parser for Jupyter notebooks
8 """
9
10 from __future__ import absolute_import, division, print_function
11
12 import argparse
13 import base64
14 import copy
15 import json
16 import mimetypes
17 import os
18 import re
19 import sys
20 from functools import partial
21 from pathlib import Path
22 from typing import Dict, List
23
24 from . import mkdocs_compatibility
25 from .errors import ExtensionError
26 from .gen_data_model import GalleryScript
27 from .py_source_parser import split_code_and_text_blocks
28
29 logger = mkdocs_compatibility.getLogger("mkdocs-gallery")
30
31
32 def jupyter_notebook_skeleton():
33 """Returns a dictionary with the elements of a Jupyter notebook"""
34 py_version = sys.version_info
35 notebook_skeleton = {
36 "cells": [],
37 "metadata": {
38 "kernelspec": {
39 "display_name": "Python " + str(py_version[0]),
40 "language": "python",
41 "name": "python" + str(py_version[0]),
42 },
43 "language_info": {
44 "codemirror_mode": {"name": "ipython", "version": py_version[0]},
45 "file_extension": ".py",
46 "mimetype": "text/x-python",
47 "name": "python",
48 "nbconvert_exporter": "python",
49 "pygments_lexer": "ipython" + str(py_version[0]),
50 "version": "{0}.{1}.{2}".format(*sys.version_info[:3]),
51 },
52 },
53 "nbformat": 4,
54 "nbformat_minor": 0,
55 }
56 return notebook_skeleton
57
58
59 def directive_fun(match, directive):
60 """Helper to fill in directives"""
61 directive_to_alert = dict(note="info", warning="danger")
62 return '<div class="alert alert-{0}"><h4>{1}</h4><p>{2}</p></div>'.format(
63 directive_to_alert[directive], directive.capitalize(), match.group(1).strip()
64 )
65
66
67 def rst2md(text, gallery_conf, target_dir, heading_levels):
68 """Converts the RST text from the examples docstrings and comments
69 into markdown text for the Jupyter notebooks
70
71 Parameters
72 ----------
73 text: str
74 RST input to be converted to MD
75 gallery_conf : dict
76 The mkdocs-gallery configuration dictionary.
77 target_dir : str
78 Path that notebook is intended for. Used where relative paths
79 may be required.
80 heading_levels: dict
81 Mapping of heading style ``(over_char, under_char)`` to heading level.
82 Note that ``over_char`` is `None` when only underline is present.
83 """
84
85 # Characters recommended for use with headings
86 # https://docutils.readthedocs.io/en/sphinx-docs/user/rst/quickstart.html#sections
87 adornment_characters = "=`:.'\"~^_*+#<>-"
88 headings = re.compile(
89 # Start of string or blank line
90 r"(?P<pre>\A|^[ \t]*\n)"
91 # Optional over characters, allowing leading space on heading text
92 r"(?:(?P<over>[{0}])(?P=over)*\n[ \t]*)?"
93 # The heading itself, with at least one non-white space character
94 r"(?P<heading>\S[^\n]*)\n"
95 # Under character, setting to same character if over present.
96 r"(?P<under>(?(over)(?P=over)|[{0}]))(?P=under)*$" r"".format(adornment_characters),
97 flags=re.M,
98 )
99
100 text = re.sub(
101 headings,
102 lambda match: "{1}{0} {2}".format(
103 "#" * heading_levels[match.group("over", "under")], *match.group("pre", "heading")
104 ),
105 text,
106 )
107
108 math_eq = re.compile(r"^\.\. math::((?:.+)?(?:\n+^ .+)*)", flags=re.M)
109 text = re.sub(
110 math_eq,
111 lambda match: r"\begin{{align}}{0}\end{{align}}".format(match.group(1).strip()),
112 text,
113 )
114 inline_math = re.compile(r":math:`(.+?)`", re.DOTALL)
115 text = re.sub(inline_math, r"$\1$", text)
116
117 directives = ("warning", "note")
118 for directive in directives:
119 directive_re = re.compile(r"^\.\. %s::((?:.+)?(?:\n+^ .+)*)" % directive, flags=re.M)
120 text = re.sub(directive_re, partial(directive_fun, directive=directive), text)
121
122 links = re.compile(r"^ *\.\. _.*:.*$\n", flags=re.M)
123 text = re.sub(links, "", text)
124
125 refs = re.compile(r":ref:`")
126 text = re.sub(refs, "`", text)
127
128 contents = re.compile(r"^\s*\.\. contents::.*$(\n +:\S+: *$)*\n", flags=re.M)
129 text = re.sub(contents, "", text)
130
131 images = re.compile(r"^\.\. image::(.*$)((?:\n +:\S+:.*$)*)\n", flags=re.M)
132 image_opts = re.compile(r"\n +:(\S+): +(.*)$", flags=re.M)
133 text = re.sub(
134 images,
135 lambda match: '<img src="{}"{}>\n'.format(
136 generate_image_src(match.group(1).strip(), gallery_conf, target_dir),
137 re.sub(image_opts, r' \1="\2"', match.group(2) or ""),
138 ),
139 text,
140 )
141
142 return text
143
144
145 def generate_image_src(image_path, gallery_conf, target_dir):
146 if re.match(r"https?://", image_path):
147 return image_path
148
149 if not gallery_conf["notebook_images"]:
150 return "file://" + image_path.lstrip("/")
151
152 # If absolute path from source directory given
153 if image_path.startswith("/"):
154 # Path should now be relative to source dir, not target dir
- F821 Undefined name 'mkdocs_conf'
156 image_path = image_path.lstrip("/")
157 full_path = os.path.join(target_dir, image_path.replace("/", os.sep))
158
159 if isinstance(gallery_conf["notebook_images"], str):
160 # Use as prefix e.g. URL
161 prefix = gallery_conf["notebook_images"]
- F821 Undefined name 'mkdocs_conf'
163 return prefix + rel_path.replace(os.sep, "/")
164 else:
165 # True, but not string. Embed as data URI.
166 try:
167 with open(full_path, "rb") as image_file:
168 data = base64.b64encode(image_file.read())
169 except OSError:
170 raise ExtensionError("Unable to open {} to generate notebook data URI" "".format(full_path))
171 mime_type = mimetypes.guess_type(full_path)
172 return "data:{};base64,{}".format(mime_type[0], data.decode("ascii"))
173
174
175 def jupyter_notebook(script: GalleryScript, script_blocks: List):
176 """Generate a Jupyter notebook file cell-by-cell
177
178 Parameters
179 ----------
180 script : GalleryScript
181 Script
182
183 script_blocks : list
184 Script execution cells.
185 """
186 # Grab the possibly custom first and last cells
187 first_cell = script.gallery_conf["first_notebook_cell"]
188 last_cell = script.gallery_conf["last_notebook_cell"]
189
190 # Initialize with a notebook skeleton
191 work_notebook = jupyter_notebook_skeleton()
192
193 # Custom first cell
194 if first_cell is not None:
195 add_code_cell(work_notebook, first_cell)
196
197 # Fill the notebook per se
198 fill_notebook(work_notebook, script_blocks)
199
200 # Custom last cell
201 if last_cell is not None:
202 add_code_cell(work_notebook, last_cell)
203
204 return work_notebook
205
206
207 def add_code_cell(work_notebook, code):
208 """Add a code cell to the notebook
209
210 Parameters
211 ----------
212 code : str
213 Cell content
214 """
215
216 code_cell = {
217 "cell_type": "code",
218 "execution_count": None,
219 "metadata": {"collapsed": False},
220 "outputs": [],
221 "source": [code.strip()],
222 }
223 work_notebook["cells"].append(code_cell)
224
225
226 def add_markdown_cell(work_notebook, markdown):
227 """Add a markdown cell to the notebook
228
229 Parameters
230 ----------
231 markdown : str
232 Markdown cell content.
233 """
234 markdown_cell = {"cell_type": "markdown", "metadata": {}, "source": [markdown]}
235 work_notebook["cells"].append(markdown_cell)
236
237
238 def fill_notebook(work_notebook, script_blocks):
239 """Writes the Jupyter notebook cells
240
241 If available, uses pypandoc to convert rst to markdown >> not anymore.
242
243 Parameters
244 ----------
245 script_blocks : list
246 Each list element should be a tuple of (label, content, lineno).
247 """
248 # heading_level_counter = count(start=1)
249 # heading_levels = defaultdict(lambda: next(heading_level_counter))
250 for blabel, bcontent, _lineno in script_blocks:
251 if blabel == "code":
252 add_code_cell(work_notebook, bcontent)
253 else:
254 # if gallery_conf["pypandoc"] is False:
255 # markdown = rst2md(
256 # bcontent + '\n', gallery_conf, target_dir, heading_levels)
257 # else:
258 # import pypandoc
259 # # pandoc automatically addds \n to the end
260 # markdown = pypandoc.convert_text(
261 # bcontent, to='md', format='rst', **gallery_conf["pypandoc"]
262 # )
263 markdown = bcontent + "\n"
264 add_markdown_cell(work_notebook, markdown)
265
266
267 def save_notebook(work_notebook: Dict, write_file: Path):
268 """Saves the Jupyter work_notebook to write_file"""
269 with open(str(write_file), "w") as out_nb:
270 json.dump(work_notebook, out_nb, indent=2)
271
272
273 ###############################################################################
274 # Notebook shell utility
275
276
277 def python_to_jupyter_cli(args=None, namespace=None):
278 """Exposes the jupyter notebook renderer to the command line
279
280 Takes the same arguments as ArgumentParser.parse_args
281 """
282 from . import gen_gallery # To avoid circular import
283
284 parser = argparse.ArgumentParser(description="mkdocs-gallery Notebook converter")
285 parser.add_argument(
286 "python_src_file",
287 nargs="+",
288 help="Input Python file script to convert. " "Supports multiple files and shell wildcards" " (e.g. *.py)",
289 )
290 args = parser.parse_args(args, namespace)
291
292 for src_file in args.python_src_file:
293 file_conf, blocks = split_code_and_text_blocks(src_file)
294 print("Converting {0}".format(src_file)) # noqa # this is a cli
295 gallery_conf = copy.deepcopy(gen_gallery.DEFAULT_GALLERY_CONF)
296 target_dir = os.path.dirname(src_file)
297 example_nb = jupyter_notebook(blocks, gallery_conf, target_dir)
- F821 Undefined name 'get_ipynb_for_py_script'