Coverage for src/mkdocs_gallery/plugin.py: 71%
183 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-15 17:10 +0000
1# Authors: Sylvain MARIE <sylvain.marie@se.com>
2# + All contributors to <https://github.com/smarie/mkdocs-gallery>
3#
4# Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
5# License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
6"""
7The mkdocs plugin entry point
8"""
9import os
10from os.path import relpath
11import re
12from pathlib import Path
13from typing import Any, Dict, List, Union
15from mkdocs import __version__ as mkdocs_version_str
16from mkdocs.config import config_options as co
17from mkdocs.config.base import Config, ValidationError
18from mkdocs.exceptions import ConfigurationError
19from mkdocs.plugins import BasePlugin
20from mkdocs.structure.files import Files
21from mkdocs.structure.pages import Page
22from packaging import version
24from . import glr_path_static
25from .binder import copy_binder_files
27# from .docs_resolv import embed_code_links
28from .gen_gallery import fill_mkdocs_nav, generate_gallery_md, parse_config, summarize_failing_examples
29from .utils import is_relative_to
31mkdocs_version = version.parse(mkdocs_version_str)
32is_mkdocs_14_or_greater = mkdocs_version >= version.parse("1.4")
35class ConfigList(co.OptionallyRequired):
36 """A list or single element of configuration matching a specific ConfigOption"""
38 def __init__(
39 self,
40 item_config: co.BaseConfigOption,
41 single_elt_allowed: bool = True,
42 **kwargs,
43 ):
44 super().__init__(**kwargs)
45 self.single_elt_allowed = single_elt_allowed
46 self.item_config = item_config
48 def run_validation(self, value):
49 if not isinstance(value, (list, tuple)):
50 if self.single_elt_allowed: 50 ↛ 53line 50 didn't jump to line 53, because the condition on line 50 was never false
51 value = (value,)
52 else:
53 msg = f"Expected a list but received a single element: {value}."
54 raise ValidationError(msg)
56 # Validate all elements in the list
57 result = []
58 for i, v in enumerate(value):
59 try:
60 result.append(self.item_config.validate(v))
61 except ValidationError as e:
62 raise ValidationError(f"Error validating config item #{i+1}: {e}")
63 return result
66class Dir(co.Dir):
67 """mkdocs.config.config_options.Dir replacement: returns a pathlib object instead of a string"""
69 def run_validation(self, value):
70 return Path(co.Dir.run_validation(self, value))
73class File(co.File):
74 """mkdocs.config.config_options.File replacement: returns a pathlib object instead of a string"""
76 def run_validation(self, value):
77 return Path(co.File.run_validation(self, value))
80# Binder configuration is a "sub config". This has changed in mkdocs 1.4, handling both here.
81if is_mkdocs_14_or_greater: 81 ↛ 103line 81 didn't jump to line 103, because the condition on line 81 was never false
82 # Construct a class where class attributes are the config options
83 class _BinderOptions(co.Config):
84 # Required keys
85 org = co.Type(str)
86 repo = co.Type(str)
87 dependencies = ConfigList(File(exists=True))
88 # Optional keys
89 branch = co.Type(str, default="gh-pages")
90 binderhub_url = co.URL(default="https://mybinder.org")
91 filepath_prefix = co.Optional(co.Type(str)) # default is None
92 notebooks_dir = co.Optional(co.Type(str)) # default is "notebooks"
93 use_jupyter_lab = co.Optional(co.Type(bool)) # default is False
95 def create_binder_config():
96 opt = co.SubConfig(_BinderOptions)
97 # Set the default value to None so that it is not (invalid) empty dict anymore. GH#45
98 opt.default = None
99 return opt
101else:
102 # Use MySubConfig
103 class MySubConfig(co.SubConfig):
104 """Same as SubConfig except that it will be an empty dict when nothing is provided by user,
105 instead of a dict with all options containing their default values."""
107 def validate(self, value):
108 if value is None or len(value) == 0:
109 return None
110 else:
111 return super(MySubConfig, self).validate(value)
113 def run_validation(self, value):
114 """Fix SubConfig: errors and warnings were not caught
116 See https://github.com/mkdocs/mkdocs/pull/2710
117 """
118 failed, self.warnings = Config.validate(self)
119 if len(failed) > 0:
120 # get the first failing one
121 key, err = failed[0]
122 raise ConfigurationError(f"Sub-option {key!r} configuration error: {err}")
124 return self
126 def create_binder_config():
127 return MySubConfig(
128 # Required keys
129 ("org", co.Type(str, required=True)),
130 ("repo", co.Type(str, required=True)),
131 ("dependencies", ConfigList(File(exists=True), required=True)),
132 # Optional keys
133 ("branch", co.Type(str, required=True, default="gh-pages")),
134 ("binderhub_url", co.URL(required=True, default="https://mybinder.org")),
135 ("filepath_prefix", co.Type(str)), # default is None
136 ("notebooks_dir", co.Type(str)), # default is "notebooks"
137 ("use_jupyter_lab", co.Type(bool)), # default is False
138 )
141class GalleryPlugin(BasePlugin):
142 # # Mandatory to display plotly graph within the site
143 # import plotly.io as pio
144 # pio.renderers.default = "sphinx_gallery"
146 config_scheme = (
147 ("conf_script", File(exists=True)),
148 ("filename_pattern", co.Type(str)),
149 ("ignore_pattern", co.Type(str)),
150 ("examples_dirs", ConfigList(Dir(exists=True))),
151 # 'reset_argv': DefaultResetArgv(),
152 ("subsection_order", co.Choice(choices=(None, "ExplicitOrder"))),
153 (
154 "within_subsection_order",
155 co.Choice(choices=("FileNameSortKey", "NumberOfCodeLinesSortKey")),
156 ),
157 ("gallery_dirs", ConfigList(Dir(exists=False))),
158 ("backreferences_dir", Dir(exists=False)),
159 ("doc_module", ConfigList(co.Type(str))),
160 # 'reference_url': {}, TODO how to link code to external functions?
161 ("capture_repr", ConfigList(co.Type(str))),
162 ("ignore_repr_types", co.Type(str)),
163 # Build options
164 ("plot_gallery", co.Type(bool)),
165 ("download_all_examples", co.Type(bool)),
166 ("abort_on_example_error", co.Type(bool)),
167 ("only_warn_on_example_error", co.Type(bool)),
168 # 'failing_examples': {}, # type: Set[str]
169 # 'passing_examples': [],
170 # 'stale_examples': [],
171 ("run_stale_examples", co.Type(bool)),
172 ("expected_failing_examples", ConfigList(File(exists=True))),
173 ("thumbnail_size", ConfigList(co.Type(int), single_elt_allowed=False)),
174 ("min_reported_time", co.Type(int)),
175 ("binder", co.Optional(create_binder_config())),
176 ("image_scrapers", ConfigList(co.Type(str))),
177 ("compress_images", ConfigList(co.Type(str))),
178 ("reset_modules", ConfigList(co.Type(str))),
179 ("first_notebook_cell", co.Type(str)),
180 ("last_notebook_cell", co.Type(str)),
181 ("notebook_images", co.Type(bool)),
182 # # 'pypandoc': False,
183 ("remove_config_comments", co.Type(bool)),
184 ("show_memory", co.Type(bool)),
185 ("show_signature", co.Type(bool)),
186 # 'junit': '',
187 # 'log_level': {'backreference_missing': 'warning'},
188 ("inspect_global_variables", co.Type(bool)),
189 # 'css': _KNOWN_CSS,
190 ("matplotlib_animations", co.Type(bool)),
191 ("image_srcset", ConfigList(co.Type(str))),
192 ("default_thumb_file", File(exists=True)),
193 ("line_numbers", co.Type(bool)),
194 )
196 def on_config(self, config, **kwargs):
197 """
198 TODO Add plugin templates and scripts to config.
199 """
201 from mkdocs.utils import yaml_load
203 # Enable navigation indexes in "material" theme,
204 # see https://squidfunk.github.io/mkdocs-material/setup/setting-up-navigation/#section-index-pages
205 if config["theme"].name == "material":
206 if "navigation.indexes" not in config["theme"]["features"]: 206 ↛ 210line 206 didn't jump to line 210, because the condition on line 206 was never false
207 if "toc.integrate" not in config["theme"]["features"]: 207 ↛ 210line 207 didn't jump to line 210, because the condition on line 207 was never false
208 config["theme"]["features"].append("navigation.indexes")
210 extra_config_yml = """
211markdown_extensions:
212 # to declare attributes such as css classes on markdown elements. For example to change the color
213 - attr_list
215 # to add notes such as http://squidfunk.github.io/mkdocs-material/extensions/admonition/
216 - admonition
218 # to display the code blocks https://squidfunk.github.io/mkdocs-material/reference/code-blocks/
219 - pymdownx.highlight
220 - pymdownx.inlinehilite
221 - pymdownx.details
222 - pymdownx.superfences
223 - pymdownx.snippets
225 # to have the download icons in the buttons
226 - pymdownx.emoji:
227 emoji_index: !!python/name:materialx.emoji.twemoji
228 emoji_generator: !!python/name:materialx.emoji.to_svg
231"""
232 extra_config = yaml_load(extra_config_yml)
233 merge_extra_config(extra_config, config)
235 # Append static resources
236 static_resources_dir = glr_path_static()
237 config["theme"].dirs.append(static_resources_dir)
238 for css_file in os.listdir(static_resources_dir):
239 if css_file.endswith(".css"):
240 config["extra_css"].append(css_file)
241 # config['theme'].static_templates.add('search.html')
242 # config['extra_javascript'].append('search/main.js')
244 # Handle our custom class - convert to a dict
245 if self.config["binder"]:
246 self.config["binder"] = dict(self.config["binder"])
248 # Remember the conf script location for later (for excluding files in `on_files` below)
249 self.conf_script = self.config["conf_script"]
251 # Use almost the original sphinx-gallery config validator
252 self.config = parse_config(self.config, mkdocs_conf=config)
254 # TODO do we need to register those CSS files and how ? (they are already registered ads
255 # for css in self.config['css']:
256 # if css not in _KNOWN_CSS:
257 # raise ConfigError('Unknown css %r, must be one of %r'
258 # % (css, _KNOWN_CSS))
259 # if gallery_conf['app'] is not None: # can be None in testing
260 # gallery_conf['app'].add_css_file(css + '.css')
262 return config
264 def on_pre_build(self, config, **kwargs):
265 """Create one md file for each python example in the gallery, and update the navigation."""
267 # TODO ?
268 # if 'sphinx.ext.autodoc' in app.extensions:
269 # app.connect('autodoc-process-docstring', touch_empty_backreferences)
270 # app.add_directive('minigallery', MiniGallery)
271 # app.add_directive("image-sg", ImageSg)
272 # imagesg_addnode(app)
274 galleries_tocs, self.md_to_src = generate_gallery_md(self.config, config)
276 # Update the nav for all galleries if needed
277 new_nav = fill_mkdocs_nav(config, galleries_tocs)
278 config["nav"] = new_nav
280 # Store the docs dir relative to project dir
281 # (note: same as in AllInformation class but we do not store the whole object)
282 project_root_dir = Path(os.path.abspath(config["config_file_path"])).parent
283 self.docs_dir_rel_proj = Path(config["docs_dir"]).relative_to(project_root_dir).as_posix()
285 def on_files(self, files, config):
286 """Remove the gallery examples *source* md files (in "examples_dirs") from the built website"""
288 # Get the list of gallery source files, possibly containing the readme.md that we wish to exclude
289 examples_dirs = self._get_dirs_relative_to(self.config["examples_dirs"], rel_to_dir=config["docs_dir"])
291 # Add the binder config files if needed
292 binder_cfg = self.config["binder"]
293 if binder_cfg: 293 ↛ 298line 293 didn't jump to line 298, because the condition on line 293 was never false
294 binder_files = [
295 Path(path).relative_to(config["docs_dir"]).as_posix() for path in binder_cfg["dependencies"]
296 ]
297 else:
298 binder_files = []
300 # Add the gallery config script if needed
301 if self.conf_script: 301 ↛ 311line 301 didn't jump to line 311, because the condition on line 301 was never false
302 conf_script = Path(self.conf_script).relative_to(config["docs_dir"])
303 conf_script_parent = conf_script.parent.as_posix()
304 if conf_script_parent == ".": 304 ↛ 307line 304 didn't jump to line 307, because the condition on line 304 was never false
305 conf_script_parent = ""
306 else:
307 conf_script_parent += r"\/"
308 conf_script_match = re.compile(rf"^{conf_script_parent}__\w*cache\w*__\/{conf_script.stem}[\w\-\.]*$")
309 conf_script = conf_script.as_posix()
311 def exclude(i):
312 # Get a posix version of the relative path so as to be sure to match ok
313 posix_src_path = Path(i.src_path).as_posix()
315 # Is it the conf script or a derived work of the conf script ?
316 if self.conf_script: 316 ↛ 321line 316 didn't jump to line 321, because the condition on line 316 was never false
317 if posix_src_path == conf_script or conf_script_match.match(posix_src_path):
318 return True
320 # Is it located in a gallery source directory ?
321 for d in examples_dirs:
322 if posix_src_path.startswith(d):
323 return True
325 # Is it a binder dependency file ?
326 if posix_src_path in binder_files:
327 return True
329 return False
331 out = []
332 for i in files:
333 if not exclude(i):
334 out.append(i)
336 return Files(out)
338 def _get_dirs_relative_to(self, dir_or_list_of_dirs: Union[str, List[str]], rel_to_dir: str) -> List[str]:
339 """Return dirs relative to another dir. If dirs is a single element, converts to a list first"""
341 # Make sure the list is a list (handle single elements)
342 if not isinstance(dir_or_list_of_dirs, list): 342 ↛ 343line 342 didn't jump to line 343, because the condition on line 342 was never true
343 dir_or_list_of_dirs = [dir_or_list_of_dirs]
345 # Get them relative to the mkdocs source dir
346 return [Path(relpath(Path(e), start=Path(rel_to_dir))).as_posix() for e in dir_or_list_of_dirs]
348 # def on_nav(self, nav, config, files):
349 # # Nav is already modded in on_pre_build, do not change it
350 # return nav
352 def on_page_content(self, html, page: Page, config: Config, files: Files):
353 """Edit the 'edit this page' link so that it points to gallery example source files."""
354 page_path = Path(page.file.src_path).as_posix()
355 try:
356 # Do we have a gallery example source file path for this page ?
357 src = self.md_to_src[page_path]
358 except KeyError:
359 pass
360 else:
361 # Note: page.edit_url is the concatenation of repo_url and edit_uri
362 # (see https://www.mkdocs.org/user-guide/configuration/)
364 if page.edit_url is not None: 364 ↛ 376line 364 didn't jump to line 376, because the condition on line 364 was never false
365 # Remove the dest gallery md file path relative to docs_dir
366 assert page.edit_url.endswith("/" + page_path)
367 edit_url = page.edit_url[: -len(page_path) - 1]
369 # Remove the docs_dir relative path with respect to project root
370 assert edit_url.endswith("/" + self.docs_dir_rel_proj)
371 edit_url = edit_url[: -len(self.docs_dir_rel_proj) - 1]
373 # Finally add the example source relative to project root
374 page.edit_url = f"{edit_url}/{src.as_posix()}"
376 return html
378 def on_serve(self, server, config, builder):
379 """Exclude gallery target dirs ("gallery_dirs") from monitored files to avoid neverending build loops."""
381 # self.observer.schedule(handler, path, recursive=recursive)
382 excluded_dirs = self.config["gallery_dirs"]
383 if isinstance(excluded_dirs, str):
384 excluded_dirs = [excluded_dirs] # a single dir
385 backrefs_dir = self.config["backreferences_dir"]
386 if backrefs_dir:
387 excluded_dirs.append(backrefs_dir)
389 def wrap_callback(original_callback):
390 def _callback(event):
391 for g in excluded_dirs:
392 if is_relative_to(g, Path(event.src_path)):
393 # ignore this event: the file is in the gallery target dir.
394 # log.info(f"Ignoring event: {event}")
395 return
396 return original_callback(event)
398 return _callback
400 # TODO this is an ugly hack...
401 # Find the objects in charge of monitoring the dirs and modify their callbacks
402 for _watch, handlers in server.observer._handlers.items():
403 for h in handlers:
404 h.on_any_event = wrap_callback(h.on_any_event)
406 return server
408 def on_post_build(self, config, **kwargs):
409 """Create one md file for each python example in the gallery."""
411 copy_binder_files(gallery_conf=self.config, mkdocs_conf=config)
412 summarize_failing_examples(gallery_conf=self.config, mkdocs_conf=config)
413 # TODO embed_code_links()
416def merge_extra_config(extra_config: Dict[str, Any], config):
417 """Extend the configuration 'markdown_extensions' list with extension_name if needed."""
419 for extension_cfg in extra_config["markdown_extensions"]:
420 if isinstance(extension_cfg, str):
421 extension_name = extension_cfg
422 if extension_name not in config["markdown_extensions"]: 422 ↛ 419line 422 didn't jump to line 419, because the condition on line 422 was never false
423 config["markdown_extensions"].append(extension_name)
424 elif isinstance(extension_cfg, dict): 424 ↛ 438line 424 didn't jump to line 438, because the condition on line 424 was never false
425 assert len(extension_cfg) == 1 # noqa
426 extension_name, extension_options = extension_cfg.popitem()
427 if extension_name not in config["markdown_extensions"]: 427 ↛ 429line 427 didn't jump to line 429, because the condition on line 427 was never false
428 config["markdown_extensions"].append(extension_name)
429 if extension_name not in config["mdx_configs"]: 429 ↛ 434line 429 didn't jump to line 434, because the condition on line 429 was never false
430 config["mdx_configs"][extension_name] = extension_options
431 else:
432 # Only add options that are not already set
433 # TODO should we warn ?
434 for cfg_key, cfg_val in extension_options.items():
435 if cfg_key not in config["mdx_configs"][extension_name]:
436 config["mdx_configs"][extension_name][cfg_key] = cfg_val
437 else:
438 raise TypeError(extension_cfg)
441# class SearchPlugin(BasePlugin):
442# """ Add a search feature to MkDocs. """
443#
444# config_scheme = (
445# ('lang', LangOption()),
446# ('separator', co.Type(str, default=r'[\s\-]+')),
447# ('min_search_length', co.Type(int, default=3)),
448# ('prebuild_index', co.Choice((False, True, 'node', 'python'), default=False)),
449# ('indexing', co.Choice(('full', 'sections', 'titles'), default='full'))
450# )
451#
452# def on_config(self, config, **kwargs):
453# "Add plugin templates and scripts to config."
454# if 'include_search_page' in config['theme'] and config['theme']['include_search_page']:
455# config['theme'].static_templates.add('search.html')
456# if not ('search_index_only' in config['theme'] and config['theme']['search_index_only']):
457# path = os.path.join(base_path, 'templates')
458# config['theme'].dirs.append(path)
459# if 'search/main.js' not in config['extra_javascript']:
460# config['extra_javascript'].append('search/main.js')
461# if self.config['lang'] is None:
462# # lang setting undefined. Set default based on theme locale
463# validate = self.config_scheme[0][1].run_validation
464# self.config['lang'] = validate(config['theme']['locale'].language)
465# # The `python` method of `prebuild_index` is pending deprecation as of version 1.2.
466# # TODO: Raise a deprecation warning in a future release (1.3?).
467# if self.config['prebuild_index'] == 'python':
468# log.info(
469# "The 'python' method of the search plugin's 'prebuild_index' config option "
470# "is pending deprecation and will not be supported in a future release."
471# )
472# return config
473#
474# def on_page_context(self, context, **kwargs):
475# "Add page to search index."
476# self.search_index.add_entry_from_context(context['page'])
477#
478# def on_post_build(self, config, **kwargs):
479# "Build search index."
480# output_base_path = os.path.join(config['site_dir'], 'search')
481# search_index = self.search_index.generate_search_index()
482# json_output_path = os.path.join(output_base_path, 'search_index.json')
483# utils.write_file(search_index.encode('utf-8'), json_output_path)
484#
485# if not ('search_index_only' in config['theme'] and config['theme']['search_index_only']):
486# # Include language support files in output. Copy them directly
487# # so that only the needed files are included.
488# files = []
489# if len(self.config['lang']) > 1 or 'en' not in self.config['lang']:
490# files.append('lunr.stemmer.support.js')
491# if len(self.config['lang']) > 1:
492# files.append('lunr.multi.js')
493# if ('ja' in self.config['lang'] or 'jp' in self.config['lang']):
494# files.append('tinyseg.js')
495# for lang in self.config['lang']:
496# if (lang != 'en'):
497# files.append(f'lunr.{lang}.js')
498#
499# for filename in files:
500# from_path = os.path.join(base_path, 'lunr-language', filename)
501# to_path = os.path.join(output_base_path, filename)
502# utils.copy_file(from_path, to_path)