Coverage for src/mkdocs_gallery/gen_gallery.py: 69%

1# Authors: Sylvain MARIE <sylvain.marie@se.com> 

2# + All contributors to <https://github.com/smarie/mkdocs-gallery> 

3# 

4# Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io> 

5# License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE> 

6""" 

7Generator for a whole gallery. 

8""" 

9 

10from __future__ import absolute_import, division, print_function 

11 

12import codecs 

13import copy 

14import os 

15import re 

16from ast import literal_eval 

17from datetime import datetime, timedelta 

18from difflib import get_close_matches 

19from importlib import import_module 

20from importlib.util import module_from_spec, spec_from_file_location 

21from pathlib import Path 

22from typing import Dict, Iterable, List, Set, Tuple 

23from xml.sax.saxutils import escape, quoteattr # noqa # indeed this is just quoting and escaping 

24 

25from . import mkdocs_compatibility 

26from .backreferences import _finalize_backreferences 

27from .binder import check_binder_conf 

28from .downloads import generate_zipfiles 

29from .errors import ConfigError, ExtensionError 

30from .gen_data_model import AllInformation, GalleryBase, GalleryScript, GalleryScriptResults 

31from .gen_single import MKD_GLR_SIG, _get_memory_base, generate 

32from .mkdocs_compatibility import red 

33from .scrapers import _import_matplotlib, _reset_dict, _scraper_dict 

34from .sorting import NumberOfCodeLinesSortKey, str_to_sorting_method 

35from .utils import _has_optipng, _new_file, _replace_by_new_if_needed, matches_filepath_pattern 

36 

37_KNOWN_CSS = ( 

38 "sg_gallery", 

39 "sg_gallery-binder", 

40 "sg_gallery-dataframe", 

41 "sg_gallery-rendered-html", 

42) 

43 

44 

45class DefaultResetArgv: 

46 def __repr__(self): 

47 return "DefaultResetArgv" 

48 

49 def __call__(self, script: GalleryScript): 

50 return [] 

51 

52 

53DEFAULT_GALLERY_CONF = { 

54 "filename_pattern": re.escape(os.sep) + "plot", 

55 "ignore_pattern": r"__init__\.py", 

56 "examples_dirs": os.path.join("..", "examples"), 

57 "reset_argv": DefaultResetArgv(), 

58 "subsection_order": None, 

59 "within_subsection_order": NumberOfCodeLinesSortKey, 

60 "gallery_dirs": "auto_examples", 

61 "backreferences_dir": None, 

62 "doc_module": (), 

63 "reference_url": {}, 

64 "capture_repr": ("_repr_html_", "__repr__"), 

65 "ignore_repr_types": r"", 

66 # Build options 

67 # ------------- 

68 # 'plot_gallery' also accepts strings that evaluate to a bool, e.g. "True", 

69 # "False", "1", "0" so that they can be easily set via command line 

70 # switches of sphinx-build 

71 "plot_gallery": True, 

72 "download_all_examples": True, 

73 "abort_on_example_error": False, 

74 "only_warn_on_example_error": False, 

75 "failing_examples": {}, # type: Set[str] 

76 "passing_examples": [], 

77 "stale_examples": [], # type: List[str] # ones that did not need to be run due to md5sum 

78 "run_stale_examples": False, 

79 "expected_failing_examples": set(), # type: Set[str] 

80 "thumbnail_size": (400, 280), # Default CSS does 0.4 scaling (160, 112) 

81 "min_reported_time": 0, 

82 "binder": {}, 

83 "image_scrapers": ("matplotlib",), 

84 "compress_images": (), 

85 "reset_modules": ("matplotlib", "seaborn"), 

86 "first_notebook_cell": "%matplotlib inline", 

87 "last_notebook_cell": None, 

88 "notebook_images": False, 

89 # 'pypandoc': False, 

90 "remove_config_comments": False, 

91 "show_memory": False, 

92 "show_signature": True, 

93 "junit": "", 

94 "log_level": {"backreference_missing": "warning"}, 

95 "inspect_global_variables": True, 

96 "css": _KNOWN_CSS, 

97 "matplotlib_animations": False, 

98 "image_srcset": [], 

99 "default_thumb_file": None, 

100 "line_numbers": False, 

101} 

102 

103logger = mkdocs_compatibility.getLogger("mkdocs-gallery") 

104 

105 

106def _bool_eval(x): 

107 if isinstance(x, str): 107 ↛ 108line 107 didn't jump to line 108 because the condition on line 107 was never true

108 try: 

109 x = literal_eval(x) 

110 except TypeError: 

111 pass 

112 return bool(x) 

113 

114 

115def parse_config(mkdocs_gallery_conf, mkdocs_conf, check_keys=True): 

116 """Process the Sphinx Gallery configuration.""" 

117 

118 # Import the base configuration script 

119 gallery_conf = load_base_conf(mkdocs_gallery_conf.pop("conf_script", None)) 

120 # Transform all strings to paths: not needed 

121 

122 # Merge configs 

123 for opt_name, opt_value in mkdocs_gallery_conf.items(): 

124 # Did the user override the option in mkdocs.yml ? (for SubConfigswe do not receive None but {}) 

125 if opt_value is None or (opt_name in ("binder",) and len(opt_value) == 0): 125 ↛ 126line 125 didn't jump to line 126 because the condition on line 125 was never true

126 continue # Not user-specified, skip 

127 

128 # User has overridden it. Use it 

129 gallery_conf[opt_name] = opt_value 

130 

131 if isinstance(gallery_conf.get("doc_module", None), list): 

132 gallery_conf["doc_module"] = tuple(gallery_conf["doc_module"]) 

133 

134 gallery_conf = _complete_gallery_conf(gallery_conf, mkdocs_conf=mkdocs_conf, check_keys=check_keys) 

135 

136 return gallery_conf 

137 

138 

139def load_base_conf(script: Path = None) -> Dict: 

140 if script is None: 

141 return dict() 

142 

143 try: 

144 spec = spec_from_file_location("__mkdocs_gallery_conf", script) 

145 foo = module_from_spec(spec) 

146 spec.loader.exec_module(foo) 

147 except ImportError as err_msg: 

148 raise ExtensionError(f"Error importing base configuration from `base_conf_py` {script}\n{err_msg}") 

149 

150 try: 

151 return foo.conf 

152 except AttributeError as err_msg: 

153 raise ExtensionError( 

154 f"Error loading base configuration from `base_conf_py` {script}, module does not contain " 

155 f"a `conf` variable.\n{err_msg}" 

156 ) 

157 

158 

159def _complete_gallery_conf( 

160 mkdocs_gallery_conf, 

161 mkdocs_conf, 

162 lang="python", 

163 builder_name="html", 

164 app=None, 

165 check_keys=True, 

166): 

167 gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF) 

168 options = sorted(gallery_conf) 

169 extra_keys = sorted(set(mkdocs_gallery_conf) - set(options)) 

170 if extra_keys and check_keys: 170 ↛ 171line 170 didn't jump to line 171 because the condition on line 170 was never true

171 msg = "Unknown key(s) in mkdocs_gallery_conf:\n" 

172 for key in extra_keys: 

173 options = get_close_matches(key, options, cutoff=0.66) 

174 msg += repr(key) 

175 if len(options) == 1: 

176 msg += ", did you mean %r?" % (options[0],) 

177 elif len(options) > 1: 

178 msg += ", did you mean one of %r?" % (options,) 

179 msg += "\n" 

180 raise ConfigError(msg.strip()) 

181 gallery_conf.update(mkdocs_gallery_conf) 

182 if mkdocs_gallery_conf.get("find_mayavi_figures", False): 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true

183 logger.warning( 

184 "Deprecated image scraping variable `find_mayavi_figures`\n" 

185 "detected, use `image_scrapers` instead as:\n\n" 

186 " image_scrapers=('matplotlib', 'mayavi')", 

187 type=DeprecationWarning, 

188 ) 

189 gallery_conf["image_scrapers"] += ("mayavi",) 

190 

191 # Text to Class for sorting methods 

192 _order = gallery_conf["subsection_order"] 

193 if isinstance(_order, str): 193 ↛ 195line 193 didn't jump to line 195 because the condition on line 193 was never true

194 # the option was passed from the mkdocs.yml file 

195 gallery_conf["subsection_order"] = str_to_sorting_method(_order) 

196 

197 _order = gallery_conf["within_subsection_order"] 

198 if isinstance(_order, str): 

199 # the option was passed from the mkdocs.yml file 

200 gallery_conf["within_subsection_order"] = str_to_sorting_method(_order) 

201 

202 # XXX anything that can only be a bool (rather than str) should probably be 

203 # evaluated this way as it allows setting via -D on the command line 

204 for key in ("run_stale_examples",): 

205 gallery_conf[key] = _bool_eval(gallery_conf[key]) 

206 # gallery_conf['src_dir'] = mkdocs_conf['docs_dir'] 

207 # gallery_conf['app'] = app 

208 

209 # Check capture_repr 

210 capture_repr = gallery_conf["capture_repr"] 

211 supported_reprs = ["__repr__", "__str__", "_repr_html_"] 

212 if isinstance(capture_repr, list): 

213 # Convert to tuple. 

214 gallery_conf["capture_repr"] = capture_repr = tuple(capture_repr) 

215 if isinstance(capture_repr, tuple): 215 ↛ 222line 215 didn't jump to line 222 because the condition on line 215 was always true

216 for rep in capture_repr: 

217 if rep not in supported_reprs: 217 ↛ 218line 217 didn't jump to line 218 because the condition on line 217 was never true

218 raise ConfigError( 

219 "All entries in 'capture_repr' must be one " "of %s, got: %s" % (supported_reprs, rep) 

220 ) 

221 else: 

222 raise ConfigError("'capture_repr' must be a tuple, got: %s" % (type(capture_repr),)) 

223 # Check ignore_repr_types 

224 if not isinstance(gallery_conf["ignore_repr_types"], str): 224 ↛ 225line 224 didn't jump to line 225 because the condition on line 224 was never true

225 raise ConfigError("'ignore_repr_types' must be a string, got: %s" % (type(gallery_conf["ignore_repr_types"]),)) 

226 

227 # deal with show_memory 

228 gallery_conf["memory_base"] = 0.0 

229 if gallery_conf["show_memory"]: 

230 if not callable(gallery_conf["show_memory"]): # True-like 230 ↛ 249line 230 didn't jump to line 249 because the condition on line 230 was always true

231 try: 

232 from memory_profiler import memory_usage # noqa 

233 except ImportError: 

234 logger.warning("Please install 'memory_profiler' to enable " "peak memory measurements.") 

235 gallery_conf["show_memory"] = False 

236 else: 

237 

238 def call_memory(func): 

239 mem, out = memory_usage(func, max_usage=True, retval=True, multiprocess=True) 

240 try: 

241 mem = mem[0] # old MP always returned a list 

242 except TypeError: # 'float' object is not subscriptable 

243 pass 

244 return mem, out 

245 

246 gallery_conf["call_memory"] = call_memory 

247 gallery_conf["memory_base"] = _get_memory_base(gallery_conf) 

248 else: 

249 gallery_conf["call_memory"] = gallery_conf["show_memory"] 

250 if not gallery_conf["show_memory"]: # can be set to False above 250 ↛ 256line 250 didn't jump to line 256 because the condition on line 250 was always true

251 

252 def call_memory(func): 

253 return 0.0, func() 

254 

255 gallery_conf["call_memory"] = call_memory 

256 assert callable(gallery_conf["call_memory"]) # noqa 

257 

258 # deal with scrapers 

259 scrapers = gallery_conf["image_scrapers"] 

260 if not isinstance(scrapers, (tuple, list)): 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true

261 scrapers = [scrapers] 

262 scrapers = list(scrapers) 

263 for si, scraper in enumerate(scrapers): 

264 if isinstance(scraper, str): 264 ↛ 276line 264 didn't jump to line 276 because the condition on line 264 was always true

265 if scraper in _scraper_dict: 265 ↛ 268line 265 didn't jump to line 268 because the condition on line 265 was always true

266 scraper = _scraper_dict[scraper] 

267 else: 

268 orig_scraper = scraper 

269 try: 

270 scraper = import_module(scraper) 

271 scraper = scraper._get_sg_image_scraper 

272 scraper = scraper() 

273 except Exception as exp: 

274 raise ConfigError("Unknown image scraper %r, got:\n%s" % (orig_scraper, exp)) 

275 scrapers[si] = scraper 

276 if not callable(scraper): 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true

277 raise ConfigError("Scraper %r was not callable" % (scraper,)) 

278 gallery_conf["image_scrapers"] = tuple(scrapers) 

279 del scrapers 

280 # Here we try to set up matplotlib but don't raise an error, 

281 # we will raise an error later when we actually try to use it 

282 # (if we do so) in scrapers.py. 

283 # In principle we could look to see if there is a matplotlib scraper 

284 # in our scrapers list, but this would be backward incompatible with 

285 # anyone using or relying on our Agg-setting behavior (e.g., for some 

286 # custom matplotlib SVG scraper as in our docs). 

287 # Eventually we can make this a config var like matplotlib_agg or something 

288 # if people need us not to set it to Agg. 

289 try: 

290 _import_matplotlib() 

291 except (ImportError, ValueError): 

292 pass 

293 

294 # compress_images 

295 compress_images = gallery_conf["compress_images"] 

296 if isinstance(compress_images, str): 296 ↛ 297line 296 didn't jump to line 297 because the condition on line 296 was never true

297 compress_images = [compress_images] 

298 elif not isinstance(compress_images, (tuple, list)): 298 ↛ 299line 298 didn't jump to line 299 because the condition on line 298 was never true

299 raise ConfigError("compress_images must be a tuple, list, or str, " "got %s" % (type(compress_images),)) 

300 compress_images = list(compress_images) 

301 allowed_values = ("images", "thumbnails") 

302 pops = list() 

303 for ki, kind in enumerate(compress_images): 

304 if kind not in allowed_values: 304 ↛ 305line 304 didn't jump to line 305 because the condition on line 304 was never true

305 if kind.startswith("-"): 

306 pops.append(ki) 

307 continue 

308 raise ConfigError( 

309 "All entries in compress_images must be one of " 

310 '%s or a command-line switch starting with "-", ' 

311 "got %r" % (allowed_values, kind) 

312 ) 

313 compress_images_args = [compress_images.pop(p) for p in pops[::-1]] 

314 if len(compress_images) and not _has_optipng(): 

315 logger.warning("optipng binaries not found, PNG %s will not be optimized" % (" and ".join(compress_images),)) 

316 compress_images = () 

317 gallery_conf["compress_images"] = compress_images 

318 gallery_conf["compress_images_args"] = compress_images_args 

319 

320 # deal with resetters 

321 resetters = gallery_conf["reset_modules"] 

322 if not isinstance(resetters, (tuple, list)): 322 ↛ 323line 322 didn't jump to line 323 because the condition on line 322 was never true

323 resetters = [resetters] 

324 resetters = list(resetters) 

325 for ri, resetter in enumerate(resetters): 

326 if isinstance(resetter, str): 326 ↛ 330line 326 didn't jump to line 330 because the condition on line 326 was always true

327 if resetter not in _reset_dict: 327 ↛ 328line 327 didn't jump to line 328 because the condition on line 327 was never true

328 raise ConfigError("Unknown module resetter named %r" % (resetter,)) 

329 resetters[ri] = _reset_dict[resetter] 

330 elif not callable(resetter): 

331 raise ConfigError("Module resetter %r was not callable" % (resetter,)) 

332 gallery_conf["reset_modules"] = tuple(resetters) 

333 

334 lang = lang if lang in ("python", "python3", "default") else "python" 

335 gallery_conf["lang"] = lang 

336 del resetters 

337 

338 # Ensure the first cell text is a string if we have it 

339 first_cell = gallery_conf.get("first_notebook_cell") 

340 if (not isinstance(first_cell, str)) and (first_cell is not None): 340 ↛ 341line 340 didn't jump to line 341 because the condition on line 340 was never true

341 raise ConfigError( 

342 "The 'first_notebook_cell' parameter must be type " "str or None, found type %s" % type(first_cell) 

343 ) 

344 # Ensure the last cell text is a string if we have it 

345 last_cell = gallery_conf.get("last_notebook_cell") 

346 if (not isinstance(last_cell, str)) and (last_cell is not None): 346 ↛ 347line 346 didn't jump to line 347 because the condition on line 346 was never true

347 raise ConfigError( 

348 "The 'last_notebook_cell' parameter must be type str" " or None, found type %s" % type(last_cell) 

349 ) 

350 # Check pypandoc 

351 # pypandoc = gallery_conf['pypandoc'] 

352 # if not isinstance(pypandoc, (dict, bool)): 

353 # raise ConfigError("'pypandoc' parameter must be of type bool or dict," 

354 # "got: %s." % type(pypandoc)) 

355 # gallery_conf['pypandoc'] = dict() if pypandoc is True else pypandoc 

356 # has_pypandoc, version = _has_pypandoc() 

357 # if isinstance(gallery_conf['pypandoc'], dict) and has_pypandoc is None: 

358 # logger.warning("'pypandoc' not available. Using mkdocs-gallery to " 

359 # "convert md text blocks to markdown for .ipynb files.") 

360 # gallery_conf['pypandoc'] = False 

361 # elif isinstance(gallery_conf['pypandoc'], dict): 

362 # logger.info("Using pandoc version: %s to convert rst text blocks to " 

363 # "markdown for .ipynb files" % (version,)) 

364 # else: 

365 # logger.info("Using mkdocs-gallery to convert rst text blocks to " 

366 # "markdown for .ipynb files.") 

367 # if isinstance(pypandoc, dict): 

368 # accepted_keys = ('extra_args', 'filters') 

369 # for key in pypandoc: 

370 # if key not in accepted_keys: 

371 # raise ConfigError("'pypandoc' only accepts the following key " 

372 # "values: %s, got: %s." 

373 # % (accepted_keys, key)) 

374 

375 # Make it easy to know which builder we're in 

376 # gallery_conf['builder_name'] = builder_name 

377 # gallery_conf['titles'] = {} 

378 

379 # Ensure 'backreferences_dir' is str, Path or None 

380 backref = gallery_conf["backreferences_dir"] 

381 if (not isinstance(backref, (str, Path))) and (backref is not None): 381 ↛ 382line 381 didn't jump to line 382 because the condition on line 381 was never true

382 raise ConfigError( 

383 "The 'backreferences_dir' parameter must be of type " "str, Path or None, " "found type %s" % type(backref) 

384 ) 

385 # if 'backreferences_dir' is str, make Path 

386 # NO: we need it to remain a str so that plugin.py works (it uses it to exclude the dir in serve mode) 

387 # if isinstance(backref, str): 

388 # gallery_conf['backreferences_dir'] = Path(backref) 

389 

390 # binder 

391 gallery_conf["binder"] = check_binder_conf(gallery_conf["binder"]) 

392 

393 if not isinstance(gallery_conf["css"], (list, tuple)): 393 ↛ 394line 393 didn't jump to line 394 because the condition on line 393 was never true

394 raise ConfigError('gallery_conf["css"] must be list or tuple, got %r' % (gallery_conf["css"],)) 

395 # for css in gallery_conf['css']: 

396 # if css not in _KNOWN_CSS: 

397 # raise ConfigError('Unknown css %r, must be one of %r' 

398 # % (css, _KNOWN_CSS)) 

399 # if gallery_conf['app'] is not None: # can be None in testing 

400 # gallery_conf['app'].add_css_file(css + '.css') 

401 

402 return gallery_conf 

403 

404 

405def generate_gallery_md(gallery_conf, mkdocs_conf) -> Dict[Path, Tuple[str, Dict[str, str]]]: 

406 """Generate the Main examples gallery reStructuredText 

407 

408 Start the mkdocs-gallery configuration and recursively scan the examples 

409 directories in order to populate the examples gallery 

410 

411 Returns 

412 ------- 

413 md_files_toc : Dict[str, Tuple[str, Dict[str, str]]] 

414 A map of galleries src folders to title and galleries toc (map of title to path) 

415 

416 md_to_src_file : Dict[str, Path] 

417 A map of posix absolute file path to generated markdown example -> Path of the src file relative to project root 

418 """ 

419 logger.info("generating gallery...") # , color='white') 

420 # gallery_conf = parse_config(app) already done 

421 

422 seen_backrefs = set() 

423 md_files_toc = dict() 

424 md_to_src_file = dict() 

425 

426 # a list of pairs "gallery source" > "gallery dest" dirs 

427 all_info = AllInformation.from_cfg(gallery_conf, mkdocs_conf) 

428 

429 # Gather all files except ignored ones, and sort them according to the configuration. 

430 all_info.collect_script_files() 

431 

432 # Check for duplicate filenames to make sure linking works as expected 

433 files = all_info.get_all_script_files() 

434 check_duplicate_filenames(files) 

435 check_spaces_in_filenames(files) 

436 

437 # For each gallery, 

438 all_results = [] 

439 for gallery in all_info.galleries: 

440 # Process the root level 

441 title, root_nested_title, index_md, results = generate(gallery=gallery, seen_backrefs=seen_backrefs) 

442 write_computation_times(gallery, results) 

443 

444 # Remember the results so that we can write the final summary 

445 all_results.extend(results) 

446 

447 # Fill the md-to-srcfile dict 

448 md_to_src_file[gallery.index_md_rel_site_root.as_posix()] = gallery.readme_file_rel_project 

449 for res in results: 

450 md_to_src_file[res.script.md_file_rel_site_root.as_posix()] = res.script.src_py_file_rel_project 

451 

452 # Create the toc entries 

453 root_md_files = {res.script.title: res.script.md_file_rel_site_root.as_posix() for res in results} 

454 root_md_files = dict_to_list_of_dicts(root_md_files) 

455 if len(gallery.subsections) == 0: 

456 # No subsections: do not nest the gallery examples further 

457 md_files_toc[gallery.generated_dir] = (title, root_md_files) 

458 else: 

459 # There are subsections. Find the root gallery title if possible and nest the root contents 

460 subsection_tocs = [{(root_nested_title or title): root_md_files}] 

461 md_files_toc[gallery.generated_dir] = (title, subsection_tocs) 

462 

463 # Create an index.md with all examples 

464 index_md_new = _new_file(gallery.index_md) 

465 with codecs.open(str(index_md_new), "w", encoding="utf-8") as fhindex: 

466 # Write the README and thumbnails for the root-level examples 

467 fhindex.write(index_md) 

468 

469 # If there are any subsections, handle them 

470 for subg in gallery.subsections: 

471 # Process the root level 

472 sub_title, _, sub_index_md, sub_results = generate(gallery=subg, seen_backrefs=seen_backrefs) 

473 write_computation_times(subg, sub_results) 

474 

475 # Remember the results so that we can write the final summary 

476 all_results.extend(sub_results) 

477 

478 # Fill the md-to-srcfile dict 

479 for res in sub_results: 

480 md_to_src_file[res.script.md_file_rel_site_root.as_posix()] = res.script.src_py_file_rel_project 

481 

482 # Create the toc entries 

483 sub_md_files = {res.script.title: res.script.md_file_rel_site_root.as_posix() for res in sub_results} 

484 sub_md_files = dict_to_list_of_dicts(sub_md_files) 

485 # Both append the subsection contents to the parent gallery toc 

486 subsection_tocs.append({sub_title: sub_md_files}) 

487 # ... and also have an independent reference in case the subsection is directly referenced in the nav. 

488 md_files_toc[subg.generated_dir] = (sub_title, sub_md_files) 

489 

490 # Write the README and thumbnails for the subgallery examples 

491 fhindex.write(sub_index_md) 

492 

493 # Finally generate the download buttons 

494 if gallery_conf["download_all_examples"]: 494 ↛ 499line 494 didn't jump to line 499 because the condition on line 494 was always true

495 download_fhindex = generate_zipfiles(gallery) 

496 fhindex.write(download_fhindex) 

497 

498 # And the "generated by..." signature 

499 if gallery_conf["show_signature"]: 499 ↛ 503line 499 didn't jump to line 503 because the condition on line 499 was always true

500 fhindex.write(MKD_GLR_SIG) 

501 

502 # Remove the .new suffix and update the md5 

503 index_md = _replace_by_new_if_needed(index_md_new, md5_mode="t") 

504 

505 _finalize_backreferences(seen_backrefs, all_info) 

506 

507 if gallery_conf["plot_gallery"]: 507 ↛ 525line 507 didn't jump to line 525 because the condition on line 507 was always true

508 logger.info("computation time summary:") # , color='white') 

509 lines, lens = _format_for_writing(all_results, kind="console") 

510 for name, t, m in lines: 

511 text = (" - %s: " % (name,)).ljust(lens[0] + 10) 

512 if t is None: 512 ↛ 513line 512 didn't jump to line 513 because the condition on line 512 was never true

513 text += "(not run)" 

514 logger.info(text) 

515 else: 

516 t_float = float(t.split()[0]) 

517 if t_float >= gallery_conf["min_reported_time"]: 517 ↛ 510line 517 didn't jump to line 510 because the condition on line 517 was always true

518 text += t.rjust(lens[1]) + " " + m.rjust(lens[2]) 

519 logger.info(text) 

520 

521 # Also create a junit.xml file if needed for rep 

522 if gallery_conf["junit"] and gallery_conf["plot_gallery"]: 522 ↛ 523line 522 didn't jump to line 523 because the condition on line 522 was never true

523 write_junit_xml(all_info, all_results) 

524 

525 return md_files_toc, md_to_src_file 

526 

527 

528def dict_to_list_of_dicts(dct: Dict) -> List[Dict]: 

529 """Transform a dict containing several entries into a list of dicts containing one entry each (nav requirement)""" 

530 return [{k: v} for k, v in dct.items()] 

531 

532 

533def fill_mkdocs_nav(mkdocs_config: Dict, galleries_tocs: Dict[Path, Tuple[str, Dict[str, str]]]): 

534 """Creates a new nav by replacing all entries in the nav containing a reference to gallery_index 

535 

536 Parameters 

537 ---------- 

538 mkdocs_config 

539 

540 galleries_tocs : Dict[Path, Tuple[str, Dict[str, str]]] 

541 A reference dict containing for each gallery, its path (the key) and its title and contents. The 

542 contents is a dictionary containing title and path to md, for each element in the gallery. 

543 """ 

544 mkdocs_docs_dir = Path(mkdocs_config["docs_dir"]) 

545 

546 # galleries_tocs_rel = {os.path.relpath(k, mkdocs_config["docs_dir"]): v for k, v in galleries_tocs.items()} 

547 galleries_tocs_unique = {Path(k).absolute().as_posix(): v for k, v in galleries_tocs.items()} 

548 

549 def get_gallery_toc(gallery_target_dir_or_index): 

550 """ 

551 Return (title, gallery_toc) if gallery_target_dir_or_index matches a known gallery, 

552 or (None, None) otherwise. 

553 """ 

554 # Do not handle absolute paths 

555 if os.path.isabs(gallery_target_dir_or_index): 555 ↛ 556line 555 didn't jump to line 556 because the condition on line 555 was never true

556 return None, None, None 

557 

558 # Auto-remove the "/index.md" if needed 

559 if gallery_target_dir_or_index.endswith("/index.md"): 559 ↛ 560line 559 didn't jump to line 560 because the condition on line 559 was never true

560 main_toc_entry = gallery_target_dir_or_index 

561 gallery_target_dir_or_index = gallery_target_dir_or_index[:-9] 

562 else: 

563 if gallery_target_dir_or_index.endswith("/"): 563 ↛ 564line 563 didn't jump to line 564 because the condition on line 563 was never true

564 main_toc_entry = gallery_target_dir_or_index + "index.md" 

565 else: 

566 main_toc_entry = gallery_target_dir_or_index + "/index.md" 

567 

568 # Find the actual absolute path for comparison 

569 gallery_target_dir_or_index = (mkdocs_docs_dir / gallery_target_dir_or_index).absolute().as_posix() 

570 

571 try: 

572 title, contents = galleries_tocs_unique[gallery_target_dir_or_index] 

573 except KeyError: 

574 # Not a gallery toc 

575 return None, None, None 

576 else: 

577 # A Gallery Toc: fill contents 

578 return title, main_toc_entry, contents 

579 

580 def _get_replacement_for(toc_elt, custom_title=None): 

581 glr_title, main_toc_entry, gallery_toc_entries = get_gallery_toc(toc_elt) 

582 if custom_title is None: 

583 custom_title = glr_title 

584 if gallery_toc_entries is not None: 

585 # Put the new contents in place 

586 return {custom_title: [{custom_title: main_toc_entry}] + gallery_toc_entries} 

587 else: 

588 # Leave the usual item 

589 return toc_elt 

590 

591 def _replace_element(toc_elt): 

592 if isinstance(toc_elt, str): 

593 # A single file name directly, e.g. index.md or gallery 

594 return _get_replacement_for(toc_elt) 

595 

596 elif isinstance(toc_elt, list): 

597 # A list of items, either single file_names or one-entry dicts 

598 return [_replace_element(elt) for elt in toc_elt] 

599 

600 elif isinstance(toc_elt, dict): 600 ↛ 619line 600 didn't jump to line 619 because the condition on line 600 was always true

601 # A dictionary containing a single element: {title: file_name} of title : (list) 

602 assert len(toc_elt) == 1 # noqa 

603 toc_name, toc_elt = tuple(toc_elt.items())[0] 

604 

605 # Have a look at the element 

606 if isinstance(toc_elt, str): 606 ↛ 616line 606 didn't jump to line 616 because the condition on line 606 was always true

607 # Special case: this is a gallery with a custom name. 

608 new_toc_elt = _get_replacement_for(toc_elt, custom_title=toc_name) 

609 if new_toc_elt is not toc_elt: 

610 return new_toc_elt 

611 else: 

612 # not a gallery, return the original contents 

613 return {toc_name: toc_elt} 

614 else: 

615 # A list: recurse 

616 return {toc_name: _replace_element(toc_elt)} 

617 

618 else: 

619 raise TypeError( 

620 f"Unsupported nav item type: f{type(toc_elt)}. Please report this issue to " f"mkdocs-gallery." 

621 ) 

622 

623 modded_nav = _replace_element(mkdocs_config["nav"]) 

624 return modded_nav 

625 

626 

627def _sec_to_readable(t): 

628 """Convert a number of seconds to a more readable representation.""" 

629 # This will only work for < 1 day execution time 

630 # And we reserve 2 digits for minutes because presumably 

631 # there aren't many > 99 minute scripts, but occasionally some 

632 # > 9 minute ones 

633 t = datetime(1, 1, 1) + timedelta(seconds=t) 

634 t = "{0:02d}:{1:02d}.{2:03d}".format(t.hour * 60 + t.minute, t.second, int(round(t.microsecond / 1000.0))) 

635 return t 

636 

637 

638def cost_name_key(result: GalleryScriptResults): 

639 # sort by descending computation time, descending memory, alphabetical name 

640 return (-result.exec_time, -result.memory, result.script.src_py_file_rel_project) 

641 

642 

643def _format_for_writing(results: GalleryScriptResults, kind="md"): 

644 """Format (name, time, memory) for a single row in the mg_execution_times.md table.""" 

645 lines = list() 

646 for result in sorted(results, key=cost_name_key): 

647 if kind == "md": # like in mg_execution_times 

648 text = ( 

649 f"[{result.script.script_stem}](./{result.script.md_file.name}) " 

650 f"({result.script.src_py_file_rel_project.as_posix()})" 

651 ) 

652 t = _sec_to_readable(result.exec_time) 

653 else: # like in generate_gallery 

654 assert kind == "console" # noqa 

655 text = result.script.src_py_file_rel_project.as_posix() 

656 t = f"{result.exec_time:0.2f} sec" 

657 

658 # Memory usage 

659 m = f"{result.memory:.1f} MB" 

660 

661 # The 3 values in the table : name, time, memory 

662 lines.append([text, t, m]) 

663 

664 lens = [max(x) for x in zip(*[[len(item) for item in cost] for cost in lines])] 

665 return lines, lens 

666 

667 

668def write_computation_times(gallery: GalleryBase, results: List[GalleryScriptResults]): 

669 """Write the mg_execution_times.md file containing all execution times.""" 

670 

671 total_time = sum(result.exec_time for result in results) 

672 if total_time == 0: 

673 return 

674 

675 target_dir = gallery.generated_dir_rel_site_root 

676 target_dir_clean = target_dir.as_posix().replace("/", "_") 

677 # new_ref = 'mkd_glr_%s_mg_execution_times' % target_dir_clean 

678 with codecs.open(str(gallery.exec_times_md_file), "w", encoding="utf-8") as fid: 

679 # Write the header 

680 fid.write( 

681 f""" 

682 

683# Computation times 

684 

685**{_sec_to_readable(total_time)}** total execution time for **{target_dir_clean}** files: 

686 

687""" 

688 ) 

689 

690 # Write the table of execution times in markdown 

691 lines, lens = _format_for_writing(results) 

692 

693 # Create the markdown table. 

694 # First line of the table +--------------+ 

695 hline = "".join(("+" + "-" * (length + 2)) for length in lens) + "+\n" 

696 fid.write(hline) 

697 

698 # Table rows 

699 format_str = "".join("| {%s} " % (ii,) for ii in range(len(lines[0]))) + "|\n" 

700 for line in lines: 

701 line = [ll.ljust(len_) for ll, len_ in zip(line, lens)] 

702 text = format_str.format(*line) 

703 assert len(text) == len(hline) # noqa 

704 fid.write(text) 

705 fid.write(hline) 

706 

707 

708def write_junit_xml(all_info: AllInformation, all_results: List[GalleryScriptResults]): 

709 """ 

710 

711 Parameters 

712 ---------- 

713 all_info 

714 all_results 

715 

716 Returns 

717 ------- 

718 

719 """ 

720 gallery_conf = all_info.gallery_conf 

721 failing_as_expected, failing_unexpectedly, passing_unexpectedly = _parse_failures(gallery_conf) 

722 

723 n_tests = 0 

724 n_failures = 0 

725 n_skips = 0 

726 elapsed = 0.0 

727 src_dir = all_info.mkdocs_docs_dir 

728 target_dir = all_info.mkdocs_site_dir 

729 output = "" 

730 for result in all_results: 

731 t = result.exec_time 

732 fname = result.script.src_py_file_rel_project 

733 if not any( 

734 fname in x 

735 for x in ( 

736 gallery_conf["passing_examples"], 

737 failing_unexpectedly, 

738 failing_as_expected, 

739 passing_unexpectedly, 

740 ) 

741 ): 

742 continue # not subselected by our regex 

743 title = gallery_conf["titles"][fname] # use gallery.title 

744 

745 _cls_name = quoteattr(os.path.splitext(os.path.basename(fname))[0]) 

746 _file = quoteattr(os.path.relpath(fname, src_dir)) 

747 _name = quoteattr(title) 

748 

749 output += f'<testcase classname={_cls_name!s} file={_file!s} line="1" name={_name!s} time="{t!r}">' 

750 if fname in failing_as_expected: 

751 output += '<skipped message="expected example failure"></skipped>' 

752 n_skips += 1 

753 elif fname in failing_unexpectedly or fname in passing_unexpectedly: 

754 if fname in failing_unexpectedly: 

755 traceback = gallery_conf["failing_examples"][fname] 

756 else: # fname in passing_unexpectedly 

757 traceback = "Passed even though it was marked to fail" 

758 n_failures += 1 

759 _msg = quoteattr(traceback.splitlines()[-1].strip()) 

760 _tb = escape(traceback) 

761 output += f"<failure message={_msg!s}>{_tb!s}</failure>" 

762 output += "</testcase>" 

763 n_tests += 1 

764 elapsed += t 

765 

766 # Add the header and footer 

767 output = f"""<?xml version="1.0" encoding="utf-8"?> 

768<testsuite errors="0" failures="{n_failures}" name="mkdocs-gallery" skipped="{n_skips}" tests="{n_tests}" time="{elapsed}"> 

769{output} 

770</testsuite> 

771""" # noqa 

772 

773 # Actually write it at desired file location 

774 fname = os.path.normpath(os.path.join(target_dir, gallery_conf["junit"])) 

775 junit_dir = os.path.dirname(fname) 

776 # Make the dirs if needed 

777 if not os.path.isdir(junit_dir): 

778 os.makedirs(junit_dir) 

779 

780 with codecs.open(fname, "w", encoding="utf-8") as fid: 

781 fid.write(output) 

782 

783 

784def touch_empty_backreferences(mkdocs_conf, what, name, obj, options, lines): 

785 """Generate empty back-reference example files. 

786 

787 This avoids inclusion errors/warnings if there are no gallery 

788 examples for a class / module that is being parsed by autodoc""" 

789 

790 # TODO uncomment below 

791 return "TODO" 

792 # if not bool(app.config.mkdocs_gallery_conf['backreferences_dir']): 

793 # return 

794 # 

795 # examples_path = os.path.join(app.srcdir, 

796 # app.config.mkdocs_gallery_conf[ 

797 # "backreferences_dir"], 

798 # "%s.examples" % name) 

799 # 

800 # if not os.path.exists(examples_path): 

801 # # touch file 

802 # open(examples_path, 'w').close() 

803 

804 

805def _expected_failing_examples(gallery_conf: Dict, mkdocs_conf: Dict) -> Set[Path]: 

806 """The set of expected failing examples""" 

807 return set((Path(mkdocs_conf["docs_dir"]) / path) for path in gallery_conf["expected_failing_examples"]) 

808 

809 

810def _parse_failures(gallery_conf: Dict, mkdocs_conf: Dict): 

811 """Split the failures.""" 

812 failing_examples = set(gallery_conf["failing_examples"].keys()) 

813 expected_failing_examples = _expected_failing_examples(gallery_conf=gallery_conf, mkdocs_conf=mkdocs_conf) 

814 

815 failing_as_expected = failing_examples.intersection(expected_failing_examples) 

816 failing_unexpectedly = failing_examples.difference(expected_failing_examples) 

817 passing_unexpectedly = expected_failing_examples.difference(failing_examples) 

818 

819 # filter from examples actually run 

820 passing_unexpectedly = [ 

821 src_file 

822 for src_file in passing_unexpectedly 

823 if matches_filepath_pattern(src_file, gallery_conf.get("filename_pattern")) 

824 ] 

825 

826 return failing_as_expected, failing_unexpectedly, passing_unexpectedly 

827 

828 

829def summarize_failing_examples(gallery_conf: Dict, mkdocs_conf: Dict): 

830 """Collects the list of falling examples and prints them with a traceback. 

831 

832 Raises ValueError if there where failing examples. 

833 """ 

834 # if exception is not None: 

835 # return 

836 

837 # Under no-plot Examples are not run so nothing to summarize