Skip to content

Commit c07ec34

Browse files
CadairCadair
authored
Merge pull request #128 from ConorMacBride/interactive-html
Interactive HTML summary
2 parents 4cbcf82 + ec56b30 commit c07ec34

19 files changed

+909
-143
lines changed

‎README.rst

+19-5
Original file line numberDiff line numberDiff line change
@@ -114,13 +114,24 @@ and the tests will pass if the images are the same. If you omit the
114114
runs, without checking the output images.
115115

116116

117-
Generating a Failure Summary
118-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
117+
Generating a Test Summary
118+
^^^^^^^^^^^^^^^^^^^^^^^^^
119119

120120
By specifying the ``--mpl-generate-summary=html`` CLI argument, a HTML summary
121-
page will be generated showing the baseline, diff and result image for each
122-
failing test. If no baseline images are configured, just the result images will
123-
be displayed. (See also, the **Results always** section below.)
121+
page will be generated showing the result, log entry and RMS of each test,
122+
and the hashes if configured. The baseline, diff and result image for each
123+
failing test will be shown. If **Results always** is configured
124+
(see section below), images for passing tests will also be shown.
125+
If no baseline images are configured, just the result images will
126+
be displayed.
127+
128+
+---------------+---------------+---------------+
129+
| |html all| | |html filter| | |html result| |
130+
+---------------+---------------+---------------+
131+
132+
As well as ``html``, ``basic-html`` can be specified for an alternative HTML
133+
summary which does not rely on JavaScript or external resources. A ``json``
134+
summary can also be saved. Multiple options can be specified comma-separated.
124135

125136
Options
126137
-------
@@ -301,6 +312,9 @@ install the latest version of the plugin then do::
301312
The reason for having to install the plugin first is to ensure that the
302313
plugin is correctly loaded as part of the test suite.
303314

315+
.. |html all| image:: images/html_all.png
316+
.. |html filter| image:: images/html_filter.png
317+
.. |html result| image:: images/html_result.png
304318
.. |expected| image:: images/baseline-coords_overlay_auto_coord_meta.png
305319
.. |actual| image:: images/coords_overlay_auto_coord_meta.png
306320
.. |diff| image:: images/coords_overlay_auto_coord_meta-failed-diff.png

‎images/html_all.png

60.1 KB
Loading

‎images/html_filter.png

56.6 KB
Loading

‎images/html_result.png

56.3 KB
Loading

‎pytest_mpl/plugin.py

+8-135
Original file line numberDiff line numberDiff line change
@@ -43,45 +43,16 @@
4343

4444
import pytest
4545

46-
SUPPORTED_FORMATS = {'html', 'json'}
46+
from pytest_mpl.summary.html import generate_summary_basic_html, generate_summary_html
47+
48+
SUPPORTED_FORMATS = {'html', 'json', 'basic-html'}
4749

4850
SHAPE_MISMATCH_ERROR = """Error: Image dimensions did not match.
4951
Expected shape: {expected_shape}
5052
{expected_path}
5153
Actual shape: {actual_shape}
5254
{actual_path}"""
5355

54-
HTML_INTRO = """
55-
<!DOCTYPE html>
56-
<html>
57-
<head>
58-
<style>
59-
table, th, td {
60-
border: 1px solid black;
61-
}
62-
.summary > div {
63-
padding: 0.5em;
64-
}
65-
tr.passed .status, .rms.passed, .hashes.passed {
66-
color: green;
67-
}
68-
tr.failed .status, .rms.failed, .hashes.failed {
69-
color: red;
70-
}
71-
</style>
72-
</head>
73-
<body>
74-
<h2>Image test comparison</h2>
75-
%summary%
76-
<table>
77-
<tr>
78-
<th>Test Name</th>
79-
<th>Baseline image</th>
80-
<th>Diff</th>
81-
<th>New image</th>
82-
</tr>
83-
"""
84-
8556

8657
def _download_file(baseline, filename):
8758
# Note that baseline can be a comma-separated list of URLs that we can
@@ -162,7 +133,7 @@ def pytest_addoption(parser):
162133
group.addoption('--mpl-generate-summary', action='store',
163134
help="Generate a summary report of any failed tests"
164135
", in --mpl-results-path. The type of the report should be "
165-
"specified. Supported types are `html` and `json`. "
136+
"specified. Supported types are `html`, `json` and `basic-html`. "
166137
"Multiple types can be specified separated by commas.")
167138

168139
results_path_help = "directory for test results, relative to location where py.test is run"
@@ -712,105 +683,6 @@ def item_function_wrapper(*args, **kwargs):
712683
else:
713684
item.obj = item_function_wrapper
714685

715-
def generate_stats(self):
716-
"""
717-
Generate a dictionary of summary statistics.
718-
"""
719-
stats = {'passed': 0, 'failed': 0, 'passed_baseline': 0, 'failed_baseline': 0, 'skipped': 0}
720-
for test in self._test_results.values():
721-
if test['status'] == 'passed':
722-
stats['passed'] += 1
723-
if test['rms'] is not None:
724-
stats['failed_baseline'] += 1
725-
elif test['status'] == 'failed':
726-
stats['failed'] += 1
727-
if test['rms'] is None:
728-
stats['passed_baseline'] += 1
729-
elif test['status'] == 'skipped':
730-
stats['skipped'] += 1
731-
else:
732-
raise ValueError(f"Unknown test status '{test['status']}'.")
733-
self._test_stats = stats
734-
735-
def generate_summary_html(self):
736-
"""
737-
Generate a simple HTML table of the failed test results
738-
"""
739-
html_file = self.results_dir / 'fig_comparison.html'
740-
with open(html_file, 'w') as f:
741-
742-
passed = f"{self._test_stats['passed']} passed"
743-
if self._test_stats['failed_baseline'] > 0:
744-
passed += (" hash comparison, although "
745-
f"{self._test_stats['failed_baseline']} "
746-
"of those have a different baseline image")
747-
748-
failed = f"{self._test_stats['failed']} failed"
749-
if self._test_stats['passed_baseline'] > 0:
750-
failed += (" hash comparison, although "
751-
f"{self._test_stats['passed_baseline']} "
752-
"of those have a matching baseline image")
753-
754-
f.write(HTML_INTRO.replace('%summary%', f'<p>{passed}.</p><p>{failed}.</p>'))
755-
756-
for test_name in sorted(self._test_results.keys()):
757-
summary = self._test_results[test_name]
758-
759-
if not self.results_always and summary['result_image'] is None:
760-
continue # Don't show test if no result image
761-
762-
if summary['rms'] is None and summary['tolerance'] is not None:
763-
rms = (f'<div class="rms passed">\n'
764-
f' <strong>RMS:</strong> '
765-
f' &lt; <span class="tolerance">{summary["tolerance"]}</span>\n'
766-
f'</div>')
767-
elif summary['rms'] is not None:
768-
rms = (f'<div class="rms failed">\n'
769-
f' <strong>RMS:</strong> '
770-
f' <span class="rms">{summary["rms"]}</span>\n'
771-
f'</div>')
772-
else:
773-
rms = ''
774-
775-
hashes = ''
776-
if summary['baseline_hash'] is not None:
777-
hashes += (f' <div class="baseline">Baseline: '
778-
f'{summary["baseline_hash"]}</div>\n')
779-
if summary['result_hash'] is not None:
780-
hashes += (f' <div class="result">Result: '
781-
f'{summary["result_hash"]}</div>\n')
782-
if len(hashes) > 0:
783-
if summary["baseline_hash"] == summary["result_hash"]:
784-
hash_result = 'passed'
785-
else:
786-
hash_result = 'failed'
787-
hashes = f'<div class="hashes {hash_result}">\n{hashes}</div>'
788-
789-
images = {}
790-
for image_type in ['baseline_image', 'diff_image', 'result_image']:
791-
if summary[image_type] is not None:
792-
images[image_type] = f'<img src="{summary[image_type]}" />'
793-
else:
794-
images[image_type] = ''
795-
796-
f.write(f'<tr class="{summary["status"]}">\n'
797-
' <td>\n'
798-
' <div class="summary">\n'
799-
f' <div class="test-name">{test_name}</div>\n'
800-
f' <div class="status">{summary["status"]}</div>\n'
801-
f' {rms}{hashes}\n'
802-
' </td>\n'
803-
f' <td>{images["baseline_image"]}</td>\n'
804-
f' <td>{images["diff_image"]}</td>\n'
805-
f' <td>{images["result_image"]}</td>\n'
806-
'</tr>\n\n')
807-
808-
f.write('</table>\n')
809-
f.write('</body>\n')
810-
f.write('</html>')
811-
812-
return html_file
813-
814686
def generate_summary_json(self):
815687
json_file = self.results_dir / 'results.json'
816688
with open(json_file, 'w') as f:
@@ -843,13 +715,14 @@ def pytest_unconfigure(self, config):
843715
if self._test_results[test_name][image_type] == '%EXISTS%':
844716
self._test_results[test_name][image_type] = str(directory / filename)
845717

846-
self.generate_stats()
847-
848718
if 'json' in self.generate_summary:
849719
summary = self.generate_summary_json()
850720
print(f"A JSON report can be found at: {summary}")
851721
if 'html' in self.generate_summary:
852-
summary = self.generate_summary_html()
722+
summary = generate_summary_html(self._test_results, self.results_dir)
723+
print(f"A summary of the failed tests can be found at: {summary}")
724+
if 'basic-html' in self.generate_summary:
725+
summary = generate_summary_basic_html(self._test_results, self.results_dir)
853726
print(f"A summary of the failed tests can be found at: {summary}")
854727

855728

‎pytest_mpl/summary/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)