mkdocs_gallery/plugin.py source
1 # Authors: Sylvain MARIE <sylvain.marie@se.com>
2 # + All contributors to <https://github.com/smarie/mkdocs-gallery>
3 #
4 # Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5 # License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6 """
7 The mkdocs plugin entry point
8 """
9 import os
10 import platform
11 import re
12 from os.path import relpath
13 from pathlib import Path
14 from typing import Any, Dict, List, Union
15
16 from mkdocs import __version__ as mkdocs_version_str
17 from mkdocs.config import config_options as co
18 from mkdocs.config.base import Config, ValidationError
19 from mkdocs.exceptions import ConfigurationError
20 from mkdocs.plugins import BasePlugin
21 from mkdocs.structure.files import Files
22 from mkdocs.structure.pages import Page
23 from packaging import version
24 from packaging.version import parse as parse_version
25
26 from . import glr_path_static
27 from .binder import copy_binder_files
28
29 # from .docs_resolv import embed_code_links
30 from .gen_gallery import fill_mkdocs_nav, generate_gallery_md, parse_config, summarize_failing_examples
31 from .utils import is_relative_to
32
33 IS_PY37 = parse_version("3.7") <= parse_version(platform.python_version()) < parse_version("3.8")
34
35 mkdocs_version = version.parse(mkdocs_version_str)
36 is_mkdocs_14_or_greater = mkdocs_version >= version.parse("1.4")
37
38
39 class ConfigList(co.OptionallyRequired):
40 """A list or single element of configuration matching a specific ConfigOption"""
41
42 def __init__(
43 self,
44 item_config: co.BaseConfigOption,
45 single_elt_allowed: bool = True,
46 **kwargs,
47 ):
48 super().__init__(**kwargs)
49 self.single_elt_allowed = single_elt_allowed
50 self.item_config = item_config
51
52 def run_validation(self, value):
53 if not isinstance(value, (list, tuple)):
54 if self.single_elt_allowed:
55 value = (value,)
56 else:
57 msg = f"Expected a list but received a single element: {value}."
58 raise ValidationError(msg)
59
60 # Validate all elements in the list
61 result = []
62 for i, v in enumerate(value):
63 try:
64 result.append(self.item_config.validate(v))
65 except ValidationError as e:
66 raise ValidationError(f"Error validating config item #{i+1}: {e}")
67 return result
68
69
70 class Dir(co.Dir):
71 """mkdocs.config.config_options.Dir replacement: returns a pathlib object instead of a string"""
72
73 def run_validation(self, value):
74 return Path(co.Dir.run_validation(self, value))
75
76
77 class File(co.File):
78 """mkdocs.config.config_options.File replacement: returns a pathlib object instead of a string"""
79
80 def run_validation(self, value):
81 return Path(co.File.run_validation(self, value))
82
83
84 # Binder configuration is a "sub config". This has changed in mkdocs 1.4, handling both here.
85 if is_mkdocs_14_or_greater:
86 # Construct a class where class attributes are the config options
87 class _BinderOptions(co.Config):
88 # Required keys
89 org = co.Type(str)
90 repo = co.Type(str)
91 dependencies = ConfigList(File(exists=True))
92 # Optional keys
93 branch = co.Type(str, default="gh-pages")
94 binderhub_url = co.URL(default="https://mybinder.org")
95 filepath_prefix = co.Optional(co.Type(str)) # default is None
96 notebooks_dir = co.Optional(co.Type(str)) # default is "notebooks"
97 use_jupyter_lab = co.Optional(co.Type(bool)) # default is False
98
99 def create_binder_config():
100 opt = co.SubConfig(_BinderOptions)
101 # Set the default value to None so that it is not (invalid) empty dict anymore. GH#45
102 opt.default = None
103 return opt
104
105 else:
106 # Use MySubConfig
107 class MySubConfig(co.SubConfig):
108 """Same as SubConfig except that it will be an empty dict when nothing is provided by user,
109 instead of a dict with all options containing their default values."""
110
111 def validate(self, value):
112 if value is None or len(value) == 0:
113 return None
114 else:
115 return super(MySubConfig, self).validate(value)
116
117 def run_validation(self, value):
118 """Fix SubConfig: errors and warnings were not caught
119
120 See https://github.com/mkdocs/mkdocs/pull/2710
121 """
122 failed, self.warnings = Config.validate(self)
123 if len(failed) > 0:
124 # get the first failing one
125 key, err = failed[0]
126 raise ConfigurationError(f"Sub-option {key!r} configuration error: {err}")
127
128 return self
129
130 def create_binder_config():
131 return MySubConfig(
132 # Required keys
133 ("org", co.Type(str, required=True)),
134 ("repo", co.Type(str, required=True)),
135 ("dependencies", ConfigList(File(exists=True), required=True)),
136 # Optional keys
137 ("branch", co.Type(str, required=True, default="gh-pages")),
138 ("binderhub_url", co.URL(required=True, default="https://mybinder.org")),
139 ("filepath_prefix", co.Type(str)), # default is None
140 ("notebooks_dir", co.Type(str)), # default is "notebooks"
141 ("use_jupyter_lab", co.Type(bool)), # default is False
142 )
143
144
145 class GalleryPlugin(BasePlugin):
146 # # Mandatory to display plotly graph within the site
147 # import plotly.io as pio
148 # pio.renderers.default = "sphinx_gallery"
149
150 config_scheme = (
151 ("conf_script", File(exists=True)),
152 ("filename_pattern", co.Type(str)),
153 ("ignore_pattern", co.Type(str)),
154 ("examples_dirs", ConfigList(Dir(exists=True))),
155 # 'reset_argv': DefaultResetArgv(),
156 ("subsection_order", co.Choice(choices=(None, "ExplicitOrder"))),
157 (
158 "within_subsection_order",
159 co.Choice(choices=("FileNameSortKey", "NumberOfCodeLinesSortKey")),
160 ),
161 ("gallery_dirs", ConfigList(Dir(exists=False))),
162 ("backreferences_dir", Dir(exists=False)),
163 ("doc_module", ConfigList(co.Type(str))),
164 # 'reference_url': {}, TODO how to link code to external functions?
165 ("capture_repr", ConfigList(co.Type(str))),
166 ("ignore_repr_types", co.Type(str)),
167 # Build options
168 ("plot_gallery", co.Type(bool)),
169 ("download_all_examples", co.Type(bool)),
170 ("abort_on_example_error", co.Type(bool)),
171 ("only_warn_on_example_error", co.Type(bool)),
172 # 'failing_examples': {}, # type: Set[str]
173 # 'passing_examples': [],
174 # 'stale_examples': [],
175 ("run_stale_examples", co.Type(bool)),
176 ("expected_failing_examples", ConfigList(File(exists=True))),
177 ("thumbnail_size", ConfigList(co.Type(int), single_elt_allowed=False)),
178 ("min_reported_time", co.Type(int)),
179 ("binder", co.Optional(create_binder_config())),
180 ("image_scrapers", ConfigList(co.Type(str))),
181 ("compress_images", ConfigList(co.Type(str))),
182 ("reset_modules", ConfigList(co.Type(str))),
183 ("first_notebook_cell", co.Type(str)),
184 ("last_notebook_cell", co.Type(str)),
185 ("notebook_images", co.Type(bool)),
186 # # 'pypandoc': False,
187 ("remove_config_comments", co.Type(bool)),
188 ("show_memory", co.Type(bool)),
189 ("show_signature", co.Type(bool)),
190 # 'junit': '',
191 # 'log_level': {'backreference_missing': 'warning'},
192 ("inspect_global_variables", co.Type(bool)),
193 # 'css': _KNOWN_CSS,
194 ("matplotlib_animations", co.Type(bool)),
195 ("image_srcset", ConfigList(co.Type(str))),
196 ("default_thumb_file", File(exists=True)),
197 ("line_numbers", co.Type(bool)),
198 )
199
200 def on_config(self, config, **kwargs):
201 """
202 TODO Add plugin templates and scripts to config.
203 """
204
205 from mkdocs.utils import yaml_load
206
207 # Enable navigation indexes in "material" theme,
208 # see https://squidfunk.github.io/mkdocs-material/setup/setting-up-navigation/#section-index-pages
209 if config["theme"].name == "material":
210 if "navigation.indexes" not in config["theme"]["features"]:
211 if "toc.integrate" not in config["theme"]["features"]:
212 config["theme"]["features"].append("navigation.indexes")
213
214 # Set Python version dependent emoji logic for backward compatibility
215 mx_name = "materialx" if IS_PY37 else "material.extensions"
216
217 extra_config_yml = f"""
218 markdown_extensions:
219 # to declare attributes such as css classes on markdown elements. For example to change the color
220 - attr_list
221
222 # to add notes such as http://squidfunk.github.io/mkdocs-material/extensions/admonition/
223 - admonition
224
225 # to display the code blocks https://squidfunk.github.io/mkdocs-material/reference/code-blocks/
226 - pymdownx.highlight
227 - pymdownx.inlinehilite
228 - pymdownx.details
229 - pymdownx.superfences
230 - pymdownx.snippets
231
232 # to have the download icons in the buttons
233 - pymdownx.emoji:
234 emoji_index: !!python/name:{mx_name}.emoji.twemoji
235 emoji_generator: !!python/name:{mx_name}.emoji.to_svg
236
237
238 """
239 extra_config = yaml_load(extra_config_yml)
240 merge_extra_config(extra_config, config)
241
242 # Append static resources
243 static_resources_dir = glr_path_static()
244 config["theme"].dirs.append(static_resources_dir)
245 for css_file in os.listdir(static_resources_dir):
246 if css_file.endswith(".css"):
247 config["extra_css"].append(css_file)
248 # config['theme'].static_templates.add('search.html')
249 # config['extra_javascript'].append('search/main.js')
250
251 # Handle our custom class - convert to a dict
252 if self.config["binder"]:
253 self.config["binder"] = dict(self.config["binder"])
254
255 # Remember the conf script location for later (for excluding files in `on_files` below)
256 self.conf_script = self.config["conf_script"]
257
258 # Use almost the original sphinx-gallery config validator
259 self.config = parse_config(self.config, mkdocs_conf=config)
260
261 # TODO do we need to register those CSS files and how ? (they are already registered ads
262 # for css in self.config['css']:
263 # if css not in _KNOWN_CSS:
264 # raise ConfigError('Unknown css %r, must be one of %r'
265 # % (css, _KNOWN_CSS))
266 # if gallery_conf['app'] is not None: # can be None in testing
267 # gallery_conf['app'].add_css_file(css + '.css')
268
269 return config
270
271 def on_pre_build(self, config, **kwargs):
272 """Create one md file for each python example in the gallery, and update the navigation."""
273
274 # TODO ?
275 # if 'sphinx.ext.autodoc' in app.extensions:
276 # app.connect('autodoc-process-docstring', touch_empty_backreferences)
277 # app.add_directive('minigallery', MiniGallery)
278 # app.add_directive("image-sg", ImageSg)
279 # imagesg_addnode(app)
280
281 galleries_tocs, self.md_to_src = generate_gallery_md(self.config, config)
282
283 # Update the nav for all galleries if needed
284 new_nav = fill_mkdocs_nav(config, galleries_tocs)
285 config["nav"] = new_nav
286
287 # Store the docs dir relative to project dir
288 # (note: same as in AllInformation class but we do not store the whole object)
289 project_root_dir = Path(os.path.abspath(config["config_file_path"])).parent
290 self.docs_dir_rel_proj = Path(config["docs_dir"]).relative_to(project_root_dir).as_posix()
291
292 def on_files(self, files, config):
293 """Remove the gallery examples *source* md files (in "examples_dirs") from the built website"""
294
295 # Get the list of gallery source files, possibly containing the readme.md that we wish to exclude
296 examples_dirs = self._get_dirs_relative_to(self.config["examples_dirs"], rel_to_dir=config["docs_dir"])
297
298 # Add the binder config files if needed
299 binder_cfg = self.config["binder"]
300 if binder_cfg:
301 binder_files = [
302 Path(path).relative_to(config["docs_dir"]).as_posix() for path in binder_cfg["dependencies"]
303 ]
304 else:
305 binder_files = []
306
307 # Add the gallery config script if needed
308 if self.conf_script:
309 conf_script = Path(self.conf_script).relative_to(config["docs_dir"])
310 conf_script_parent = conf_script.parent.as_posix()
311 if conf_script_parent == ".":
312 conf_script_parent = ""
313 else:
314 conf_script_parent += r"\/"
315 conf_script_match = re.compile(rf"^{conf_script_parent}__\w*cache\w*__\/{conf_script.stem}[\w\-\.]*$")
316 conf_script = conf_script.as_posix()
317
318 def exclude(i):
319 # Get a posix version of the relative path so as to be sure to match ok
320 posix_src_path = Path(i.src_path).as_posix()
321
322 # Is it the conf script or a derived work of the conf script ?
323 if self.conf_script:
324 if posix_src_path == conf_script or conf_script_match.match(posix_src_path):
325 return True
326
327 # Is it located in a gallery source directory ?
328 for d in examples_dirs:
329 if posix_src_path.startswith(d):
330 return True
331
332 # Is it a binder dependency file ?
333 if posix_src_path in binder_files:
334 return True
335
336 return False
337
338 out = []
339 for i in files:
340 if not exclude(i):
341 out.append(i)
342
343 return Files(out)
344
345 def _get_dirs_relative_to(self, dir_or_list_of_dirs: Union[str, List[str]], rel_to_dir: str) -> List[str]:
346 """Return dirs relative to another dir. If dirs is a single element, converts to a list first"""
347
348 # Make sure the list is a list (handle single elements)
349 if not isinstance(dir_or_list_of_dirs, list):
350 dir_or_list_of_dirs = [dir_or_list_of_dirs]
351
352 # Get them relative to the mkdocs source dir
353 return [Path(relpath(Path(e), start=Path(rel_to_dir))).as_posix() for e in dir_or_list_of_dirs]
354
355 # def on_nav(self, nav, config, files):
356 # # Nav is already modded in on_pre_build, do not change it
357 # return nav
358
359 def on_page_content(self, html, page: Page, config: Config, files: Files):
360 """Edit the 'edit this page' link so that it points to gallery example source files."""
361 page_path = Path(page.file.src_path).as_posix()
362 try:
363 # Do we have a gallery example source file path for this page ?
364 src = self.md_to_src[page_path]
365 except KeyError:
366 pass
367 else:
368 # Note: page.edit_url is the concatenation of repo_url and edit_uri
369 # (see https://www.mkdocs.org/user-guide/configuration/)
370
371 if page.edit_url is not None:
372 # Remove the dest gallery md file path relative to docs_dir
- S101 Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
374 edit_url = page.edit_url[: -len(page_path) - 1]
375
376 # Remove the docs_dir relative path with respect to project root
- S101 Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
378 edit_url = edit_url[: -len(self.docs_dir_rel_proj) - 1]
379
380 # Finally add the example source relative to project root
381 page.edit_url = f"{edit_url}/{src.as_posix()}"
382
383 return html
384
385 def on_serve(self, server, config, builder):
386 """Exclude gallery target dirs ("gallery_dirs") from monitored files to avoid neverending build loops."""
387
388 # self.observer.schedule(handler, path, recursive=recursive)
389 excluded_dirs = self.config["gallery_dirs"]
390 if isinstance(excluded_dirs, str):
391 excluded_dirs = [excluded_dirs] # a single dir
392 backrefs_dir = self.config["backreferences_dir"]
393 if backrefs_dir:
394 excluded_dirs.append(backrefs_dir)
395
396 def wrap_callback(original_callback):
397 def _callback(event):
398 for g in excluded_dirs:
399 if is_relative_to(g, Path(event.src_path)):
400 # ignore this event: the file is in the gallery target dir.
401 # log.info(f"Ignoring event: {event}")
402 return
403 return original_callback(event)
404
405 return _callback
406
407 # TODO this is an ugly hack...
408 # Find the objects in charge of monitoring the dirs and modify their callbacks
409 for _watch, handlers in server.observer._handlers.items():
410 for h in handlers:
411 h.on_any_event = wrap_callback(h.on_any_event)
412
413 return server
414
415 def on_post_build(self, config, **kwargs):
416 """Create one md file for each python example in the gallery."""
417
418 copy_binder_files(gallery_conf=self.config, mkdocs_conf=config)
419 summarize_failing_examples(gallery_conf=self.config, mkdocs_conf=config)
420 # TODO embed_code_links()
421
422
423 def merge_extra_config(extra_config: Dict[str, Any], config):
424 """Extend the configuration 'markdown_extensions' list with extension_name if needed."""
425
426 for extension_cfg in extra_config["markdown_extensions"]:
427 if isinstance(extension_cfg, str):
428 extension_name = extension_cfg
429 if extension_name not in config["markdown_extensions"]:
430 config["markdown_extensions"].append(extension_name)
431 elif isinstance(extension_cfg, dict):
432 assert len(extension_cfg) == 1 # noqa
433 extension_name, extension_options = extension_cfg.popitem()
434 if extension_name not in config["markdown_extensions"]:
435 config["markdown_extensions"].append(extension_name)
436 if extension_name not in config["mdx_configs"]:
437 config["mdx_configs"][extension_name] = extension_options
438 else:
439 # Only add options that are not already set
440 # TODO should we warn ?
441 for cfg_key, cfg_val in extension_options.items():
442 if cfg_key not in config["mdx_configs"][extension_name]:
443 config["mdx_configs"][extension_name][cfg_key] = cfg_val
444 else:
445 raise TypeError(extension_cfg)
446
447
448 # class SearchPlugin(BasePlugin):
449 # """ Add a search feature to MkDocs. """
450 #
451 # config_scheme = (
452 # ('lang', LangOption()),
453 # ('separator', co.Type(str, default=r'[\s\-]+')),
454 # ('min_search_length', co.Type(int, default=3)),
455 # ('prebuild_index', co.Choice((False, True, 'node', 'python'), default=False)),
456 # ('indexing', co.Choice(('full', 'sections', 'titles'), default='full'))
457 # )
458 #
459 # def on_config(self, config, **kwargs):
460 # "Add plugin templates and scripts to config."
461 # if 'include_search_page' in config['theme'] and config['theme']['include_search_page']:
462 # config['theme'].static_templates.add('search.html')
463 # if not ('search_index_only' in config['theme'] and config['theme']['search_index_only']):
464 # path = os.path.join(base_path, 'templates')
465 # config['theme'].dirs.append(path)
466 # if 'search/main.js' not in config['extra_javascript']:
467 # config['extra_javascript'].append('search/main.js')
468 # if self.config['lang'] is None:
469 # # lang setting undefined. Set default based on theme locale
470 # validate = self.config_scheme[0][1].run_validation
471 # self.config['lang'] = validate(config['theme']['locale'].language)
472 # # The `python` method of `prebuild_index` is pending deprecation as of version 1.2.
473 # # TODO: Raise a deprecation warning in a future release (1.3?).
474 # if self.config['prebuild_index'] == 'python':
475 # log.info(
476 # "The 'python' method of the search plugin's 'prebuild_index' config option "
477 # "is pending deprecation and will not be supported in a future release."
478 # )
479 # return config
480 #
481 # def on_page_context(self, context, **kwargs):
482 # "Add page to search index."
483 # self.search_index.add_entry_from_context(context['page'])
484 #
485 # def on_post_build(self, config, **kwargs):
486 # "Build search index."
487 # output_base_path = os.path.join(config['site_dir'], 'search')
488 # search_index = self.search_index.generate_search_index()
489 # json_output_path = os.path.join(output_base_path, 'search_index.json')
490 # utils.write_file(search_index.encode('utf-8'), json_output_path)
491 #
492 # if not ('search_index_only' in config['theme'] and config['theme']['search_index_only']):
493 # # Include language support files in output. Copy them directly
494 # # so that only the needed files are included.
495 # files = []
496 # if len(self.config['lang']) > 1 or 'en' not in self.config['lang']:
497 # files.append('lunr.stemmer.support.js')
498 # if len(self.config['lang']) > 1:
499 # files.append('lunr.multi.js')
500 # if ('ja' in self.config['lang'] or 'jp' in self.config['lang']):
501 # files.append('tinyseg.js')
502 # for lang in self.config['lang']:
503 # if (lang != 'en'):
504 # files.append(f'lunr.{lang}.js')
505 #
506 # for filename in files:
507 # from_path = os.path.join(base_path, 'lunr-language', filename)
508 # to_path = os.path.join(output_base_path, filename)
509 # utils.copy_file(from_path, to_path)