Coverage for genbadge/tests/test_readme_common.py: 94%
137 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-11-10 20:37 +0000
1import platform
2import sys
3from shutil import copy
5import click
6from distutils.version import LooseVersion
8import pytest
10try:
11 from pathlib import Path
12except ImportError: # pragma: no cover
13 from pathlib2 import Path # python 2
15from click.testing import CliRunner
16from genbadge.main import genbadge as genbadge_cmd
19TESTS_FOLDER = Path(__file__).parent.absolute()
22class CmdReference:
23 """
24 Container class used to store our test reference for all commands
25 """
26 def __init__(self, name, default_infile, default_outfile, example_input_file, example_output_msg,
27 example_output_msg_long, help_msg):
28 self.name = name
29 self.default_infile = default_infile
30 self.default_outfile = default_outfile
31 self.example_input_file = example_input_file
32 self.example_output_msg = example_output_msg
33 self.example_output_msg_long = example_output_msg_long
34 self.help_msg = help_msg
36 def __str__(self):
37 return self.name
40TEST_CMD = CmdReference(
41 name="tests",
42 default_infile="reports/junit/junit.xml",
43 default_outfile = "tests-badge.svg",
44 example_input_file=(TESTS_FOLDER / "reports" / "junit" / "junit.xml").as_posix(),
45 example_output_msg="SUCCESS - Tests badge created: %r\n",
46 example_output_msg_long="""
47Test statistics parsed successfully from %r
48 - Nb tests: Total (6) = Success (2) + Skipped (1) + Failed (2) + Errors (1)
49 - Success percentage: 40.00%% (2 / 5) (Skipped tests are excluded)
51SUCCESS - Tests badge created: %r
52""",
53 help_msg="""Usage: genbadge tests [OPTIONS]
55 This command generates a badge for the test results, from an XML file in the
56 junit format. Such a file can be for example generated from python pytest
57 using the --junitxml flag, or from java junit.
59 By default the input file is the relative `./reports/junit/junit.xml` and the
60 output file is `./tests-badge.svg`. You can change these settings with the
61 `-i/--input_file` and `-o/--output-file` options.
63 By default the badge will have the name "tests" as the left-hand side text.
64 You can change these settings with the `-n/--name` option. The left-hand side
65 text can be left blank with `-n ""` or have the left-hand side of the badge
66 completely removed by passing `--noname`.
68 You can use the verbose flag `-v/--verbose` to display information on the
69 input file contents, for verification.
71 The resulting badge will by default look like this: [tests | 6/12] where 6 is
72 the number of tests that have run successfully, and 12 is the total number of
73 tests minus the number of skipped tests. When all tests pass with success, the
74 badge simply shows the number of tests [tests | 12]. You can change the
75 appearance of the badge with the --format option (not implemented, todo).
77 The success percentage is defined as 6/12 = 50.0%. You can use the
78 `-t/--threshold` flag to setup a minimum success percentage required. If the
79 success percentage is below the threshold, an error will be raised and the
80 badge will not be generated.
82Options:
83 -i, --input-file FILENAME An alternate test results XML file to read.
84 '-' is supported and means <stdin>.
85 -o, --output-file FILENAME An alternate SVG badge file to write to. '-'
86 is supported and means <stdout>. Note that in
87 this case no other message will be printed to
88 <stdout>. In particular the verbose flag will
89 have no effect.
90 -n, --name TEXT An alternate SVG badge text name to display on
91 the left-hand side of the badge.
92 -t, --threshold FLOAT An optional success percentage threshold to
93 use. The command will fail with exit code 1 if
94 theactual success percentage is strictly less
95 than the provided value.
96 --withname / --noname Indicates if a badge should be generated with
97 or without the left-hand side of the badge.
98 -w, --webshields / -l, --local Indicates if badges should be generated using
99 the shields.io HTTP API (default) or the local
100 SVG file template included.
101 -v, --verbose Use this flag to print details to stdout
102 during the badge generation process. Note that
103 this flag has no effect when '-' is used as
104 output, since the badge is written to
105 <stdout>. It also has no effect when the
106 silent flag `-s` is used.
107 -s, --silent When this flag is active nothing will be
108 written to stdout. Note that this flag has no
109 effect when '-' is used as the output file.
110 --help Show this message and exit.
111"""
112)
113COV_CMD = CmdReference(
114 name="coverage",
115 default_infile="reports/coverage/coverage.xml",
116 default_outfile = "coverage-badge.svg",
117 example_input_file=(TESTS_FOLDER / "reports" / "coverage" / "coverage.xml").as_posix(),
118 example_output_msg="SUCCESS - Coverage badge created: %r\n",
119 example_output_msg_long="""
120Coverage results parsed successfully from %r
121 - Branch coverage: 5.56%% (1/18)
122 - Line coverage: 17.81%% (13/73)
123 - Total coverage: 15.38%% ((1+13)/(18+73))
125SUCCESS - Coverage badge created: %r
126""",
127 help_msg="""Usage: genbadge coverage [OPTIONS]
129 This command generates a badge for the coverage results, from an XML file in
130 the 'coverage' format. Such a file can be for example generated using the
131 python `coverage` tool, or java `cobertura`.
133 By default the input file is the relative `./reports/coverage/coverage.xml`
134 and the output file is `./coverage-badge.svg`. You can change these settings
135 with the `-i/--input_file` and `-o/--output-file` options.
137 By default the badge will have the name "coverage" as the left-hand side text.
138 You can change these settings with the `-n/--name` option. The left-hand side
139 text can be left blank with `-n ""` or have the left-hand side of the badge
140 completely removed by passing `--noname`.
142 You can use the verbose flag `-v/--verbose` to display information on the
143 input file contents, for verification.
145 The resulting badge will by default look like this: [coverage | 98.1%] where
146 98.1 is the total coverage, obtained from the branch and line coverages using
147 the formula
149 (nb_lines_covered + nb_branches_covered) / (nb_lines / nb_branches)
151 and multiplying this by 100.
153Options:
154 -i, --input-file FILENAME An alternate coverage results XML file to
155 read. '-' is supported and means <stdin>.
156 -o, --output-file FILENAME An alternate SVG badge file to write to. '-'
157 is supported and means <stdout>. Note that in
158 this case no other message will be printed to
159 <stdout>. In particular the verbose flag will
160 have no effect.
161 -n, --name TEXT An alternate SVG badge text name to display on
162 the left-hand side of the badge.
163 --withname / --noname Indicates if a badge should be generated with
164 or without the left-hand side of the badge.
165 -w, --webshields / -l, --local Indicates if badges should be generated using
166 the shields.io HTTP API (default) or the local
167 SVG file template included.
168 -v, --verbose Use this flag to print details to stdout
169 during the badge generation process. Note that
170 this flag has no effect when '-' is used as
171 output, since the badge is written to
172 <stdout>. It also has no effect when the
173 silent flag `-s` is used.
174 -s, --silent When this flag is active nothing will be
175 written to stdout. Note that this flag has no
176 effect when '-' is used as the output file.
177 --help Show this message and exit.
178"""
179)
180FLAKE8_CMD = CmdReference(
181 name="flake8",
182 default_infile="reports/flake8/flake8stats.txt",
183 default_outfile = "flake8-badge.svg",
184 example_input_file=(TESTS_FOLDER / "reports" / "flake8" / "flake8stats.txt").as_posix(),
185 example_output_msg="SUCCESS - Flake8 badge created: %r\n",
186 example_output_msg_long="""
187Flake8 statistics parsed successfully from %r
188 - Total (20) = Critical (6) + Warning (9) + Info (5)
190SUCCESS - Flake8 badge created: %r
191""",
192 help_msg="""Usage: genbadge flake8 [OPTIONS]
194 This command generates a badge for the flake8 results, from a flake8stats.txt
195 file. Such a file can be generated from python `flake8` using the --statistics
196 flag.
198 By default the input file is the relative `./reports/flake8/flake8stats.txt`
199 and the output file is `./flake8-badge.svg`. You can change these settings
200 with the `-i/--input_file` and `-o/--output-file` options.
202 By default the badge will have the name "flake8" as the left-hand side text.
203 You can change these settings with the `-n/--name` option. The left-hand side
204 text can be left blank with `-n ""` or have the left-hand side of the badge
205 completely removed by passing `--noname`.
207 You can use the verbose flag `-v/--verbose` to display information on the
208 input file contents, for verification.
210 The resulting badge will by default look like this: [flake8 | 6 C, 0 W, 5 I]
211 where 6, 0, 5 denote the number of critical issues, warnings, and information
212 messages respectively. These severity levels are determined by the flake8-html
213 plugin so as to match the colors in the HTML report. You can change the
214 appearance of the badge with the --format option (not implemented, todo).
216Options:
217 -i, --input-file FILENAME An alternate flake8 results TXT file to read.
218 '-' is supported and means <stdin>.
219 -o, --output-file FILENAME An alternate SVG badge file to write to. '-'
220 is supported and means <stdout>. Note that in
221 this case no other message will be printed to
222 <stdout>. In particular the verbose flag will
223 have no effect.
224 -n, --name TEXT An alternate SVG badge text name to display on
225 the left-hand side of the badge.
226 --withname / --noname Indicates if a badge should be generated with
227 or without the left-hand side of the badge.
228 -w, --webshields / -l, --local Indicates if badges should be generated using
229 the shields.io HTTP API (default) or the local
230 SVG file template included.
231 -v, --verbose Use this flag to print details to stdout
232 during the badge generation process. Note that
233 this flag has no effect when '-' is used as
234 output, since the badge is written to
235 <stdout>. It also has no effect when the
236 silent flag `-s` is used.
237 -s, --silent When this flag is active nothing will be
238 written to stdout. Note that this flag has no
239 effect when '-' is used as the output file.
240 --help Show this message and exit.
241"""
242)
244ALL_COMMANDS = [TEST_CMD, COV_CMD, FLAKE8_CMD]
245ALL_COMMAND_NAMES = [c.name for c in ALL_COMMANDS]
248def test_help():
249 """Test that `genbadge` provides the right help"""
251 runner = CliRunner()
252 result = runner.invoke(genbadge_cmd, [], catch_exceptions=False)
254 assert result.exit_code == 0
255 print(result.output)
256 expected = """
257Usage: genbadge [OPTIONS] COMMAND [ARGS]...
259 Commandline utility to generate badges. To get help on each command use:
261 genbadge <cmd> --help
263Options:
264 --help Show this message and exit.
266Commands:
267 coverage Generate a badge for the coverage results (e.g. from a
268 coverage.xml).%s
269 flake8 Generate a badge for the flake8 results (e.g. from a flake8stats.txt
270 file).%s
271 tests Generate a badge for the test results (e.g. from a junit.xml).
272"""
273 if LooseVersion(click.__version__) < "8.": 273 ↛ 274line 273 didn't jump to line 274, because the condition on line 273 was never true
274 expected = expected % ("\n", "\n")
275 else:
276 expected = expected % ("", "")
277 assert "\n" + result.output == expected
280@pytest.mark.parametrize("cmd", ALL_COMMANDS, ids=str)
281def test_help_cmd(cmd):
282 """Test that the command-specific help message is correct"""
284 result = _invoke_genbadge([cmd.name, "--help"])
285 assert result.exit_code == 0
287 if LooseVersion(click.__version__) >= "8.": 287 ↛ 291line 287 didn't jump to line 291, because the condition on line 287 was never false
288 assert result.output == cmd.help_msg
289 else:
290 # the line wrapping seems to have changed, do not check for this old version.
291 pass
294@pytest.mark.parametrize("cmd", ALL_COMMANDS, ids=str)
295def test_file_not_found(monkeypatch, tmpdir, cmd):
296 """Test that the error message is nice when the input file is not found"""
298 currentfolder = Path(str(tmpdir))
299 monkeypatch.chdir(str(currentfolder))
301 # a) default input file: the error is raised by us as a click.exceptions.FileError (exit code 1)
302 result = _invoke_genbadge([cmd.name])
303 assert result.exit_code == 1
304 expected = """
305Error: Could not open file %r: File not found
306"""
307 # support various versions of click
308 if LooseVersion(click.__version__) < "8.": 308 ↛ 309line 308 didn't jump to line 309, because the condition on line 308 was never true
309 expected = expected.replace("%r", "%s")
310 assert "\n" + result.output == expected % cmd.default_infile
312 # b) different non-existent input file: the error is raised by click from click.File as a BadParameterError (code 2)
313 unknown_file = "unknown.file"
314 result = _invoke_genbadge([cmd.name, "-i", unknown_file])
315 assert result.exit_code == 2
316 expected = """
317Usage: genbadge {name} [OPTIONS]
318Try 'genbadge {name} --help' for help.
320Error: Invalid value for '-i' / '--input-file': %s: No such file or directory
321""".format(name=cmd.name)
322 # support various versions of click
323 if LooseVersion(click.__version__) < "8.": 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true
324 expected = expected % "Could not open file: %s"
325 else:
326 expected = expected % "%r"
327 assert "\n" + result.output == expected % unknown_file
330@pytest.mark.parametrize("outstream", [False, True], ids="outstream={}".format)
331@pytest.mark.parametrize("silent", [False, True], ids="silent={}".format)
332@pytest.mark.parametrize("verbose", [False, True], ids="verbose={}".format)
333@pytest.mark.parametrize("variant", ["default", "custom", "custom_shortargs", "custom_absolute"])
334@pytest.mark.parametrize("cmd", ALL_COMMANDS, ids=str)
335def test_any_command(monkeypatch, cmd, tmpdir, variant, outstream, silent, verbose):
336 """Test that `genbadge <cmd>` works consistently concerning the ios and output messages"""
338 # from pytest path to pathlib path
339 currentfolder = Path(str(tmpdir)) # tests_folder
341 # change the working directory to tmpdir
342 monkeypatch.chdir(str(currentfolder))
344 # create the various arguments. Use local template for faster exec
345 args = [cmd.name, "-l"]
346 if verbose:
347 args.append("--verbose")
348 if silent:
349 args.append("--silent")
350 if variant == "default":
351 if outstream:
352 pytest.skip("this test does not make sense")
353 infile = currentfolder / cmd.default_infile
354 outfile = currentfolder / cmd.default_outfile
355 infile_path_for_msg = str(infile.absolute().as_posix())
356 elif variant in ("custom", "custom_shortargs", "custom_absolute"): 356 ↛ 367line 356 didn't jump to line 367, because the condition on line 356 was never false
357 shortargs = "shortargs" in variant
358 absolute = "absolute" in variant
359 infile_path_for_msg = "foo/foo.xml"
360 infile = currentfolder / infile_path_for_msg
361 if absolute:
362 infile_path_for_msg = infile.absolute().as_posix()
363 outfile = currentfolder / "bar" / "bar-badge.svg"
364 args += ["-i" if shortargs else "--input-file", infile_path_for_msg]
365 args += ["-o" if shortargs else "--output-file", "-" if outstream else "bar/bar-badge.svg"]
366 else:
367 raise ValueError(variant)
368 outfile_path_for_msg = str(outfile.absolute().as_posix())
370 # copy the file from source
371 infile.parent.mkdir(parents=True, exist_ok=True)
372 copy(str(cmd.example_input_file), str(infile))
374 # execute "genbadge tests" with the appropriate arguments
375 print("Executing command in tmp folder %s" % (currentfolder,))
376 result = _invoke_genbadge(args)
377 assert result.exit_code == 0
379 # verify the output message
380 if not outstream:
381 if silent:
382 assert result.output == ""
383 elif verbose:
384 assert "\n" + result.output == cmd.example_output_msg_long % (infile_path_for_msg, outfile_path_for_msg)
385 else:
386 assert result.output == cmd.example_output_msg % outfile_path_for_msg
387 assert outfile.exists()
388 else:
389 assert result.output.startswith('<svg xmlns="http://www.w3.org/2000/svg" '
390 'xmlns:xlink="http://www.w3.org/1999/xlink" width=')
393@pytest.mark.parametrize("threshold", [-1, 0, 40, 40.1, 100, 101])
394@pytest.mark.parametrize("shortarg", [False, True])
395def test_threshold(threshold, shortarg, tmpdir):
397 # from pytest path to pathlib path
398 destfolder = Path(str(tmpdir))
399 badge_path = destfolder / "tests-badge.svg"
401 # define cli args (explicit input so that we do not fall into the python 2 issue with CliRunner IOError
402 args = ["tests", "-v", "-i", str(TEST_CMD.example_input_file), "-o", "%s" % badge_path]
403 args += ["-t" if shortarg else "--threshold", str(threshold)]
405 # execute "genbadge tests" with the appropriate arguments
406 result = _invoke_genbadge(args)
408 if 40.0 < threshold:
409 assert result.exit_code == 1
411 # verify the output message
412 assert "\n" + result.output == """
413Test statistics parsed successfully from '{}'
414 - Nb tests: Total (6) = Success (2) + Skipped (1) + Failed (2) + Errors (1)
415 - Success percentage: 40.00% (2 / 5) (Skipped tests are excluded)
417Error: Success percentage 40.0% is strictly lower than required threshold {}%
418""".format(str(TEST_CMD.example_input_file), float(threshold))
420 assert not badge_path.exists()
422 else:
423 assert result.exit_code == 0
425 # verify the output message
426 assert "\n" + result.output == TEST_CMD.example_output_msg_long % (str(TEST_CMD.example_input_file), str(badge_path.as_posix()))
428 assert badge_path.exists()
431@pytest.mark.parametrize("use_shields,shortarg",
432 [(None, None), (False, False), (False, True), (True, False), (True, True)])
433def test_local_remote(use_shields, shortarg, tmpdir):
435 if use_shields is False and sys.version_info < (3,) and platform.system != "Windows": 435 ↛ 436line 435 didn't jump to line 436, because the condition on line 435 was never true
436 pytest.skip("On Linux the embedded ttf font file is needed, and because of the path change pkg_resources does"
437 "not manage to find the file on python 2")
439 # from pytest path to pathlib path
440 destfolder = Path(str(tmpdir))
441 badge_path = destfolder / "tests-badge.svg"
443 # define cli args (explicit input so that we do not fall into the python 2 issue with CliRunner IOError
444 args = ["tests", "-v", "-i", str(TEST_CMD.example_input_file), "-o", "%s" % badge_path]
445 if use_shields is False:
446 args.append("-l" if shortarg else "--local")
447 if use_shields is True:
448 args.append("-w" if shortarg else "--webshields")
450 # execute "genbadge tests" with the appropriate arguments
451 result = _invoke_genbadge(args)
452 assert result.exit_code == 0
454 # verify the output message
455 assert "\n" + result.output == TEST_CMD.example_output_msg_long % (str(TEST_CMD.example_input_file), str(badge_path.as_posix()))
457 assert badge_path.exists()
460def _invoke_genbadge(args):
461 runner = CliRunner()
462 print("\n> genbadge %s" % (" ".join(args),))
463 result = runner.invoke(genbadge_cmd, args, catch_exceptions=False)
464 print(result.output)
465 return result