add diff_pixels, diff_ratio for acceptence, similar as playwright node.js version

.github/workflows/ci.yml

@@ -1,42 +1,20 @@
 name: CI
 
 on:
-  push:
-    branches:
-      - main
-  pull_request:
+  push
 
 env:
   PLAYWRIGHT_BROWSERS_PATH: 0
 
 jobs:
-  infra:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -e .
-          python -m playwright install-deps
-          python -m playwright install
-      - name: Lint
-        uses: pre-commit/[email protected]
-
   test:
     name: Test
     timeout-minutes: 30
     strategy:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9, 3.10.4]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2

.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,40 @@
+name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+
+on:
+  push
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python 3.10.4
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.10.4
+      - name: Install pypa/build
+        run: >-
+          python -m
+          pip install
+          build
+          --user
+      - name: Build a binary wheel and a source tarball
+        run: >-
+          python -m
+          build
+          --sdist
+          --wheel
+          --outdir dist/
+#      - name: Publish distribution 📦 to Test PyPI
+#        uses: pypa/gh-action-pypi-publish@release/v1
+#        with:
+#          user: __token__
+#          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+#          repository_url: https://test.pypi.org/legacy/
+      - name: Publish distribution 📦 to PyPI
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

README.md

@@ -1,11 +1,20 @@
-# Pytest Plugin for Snapshot Testing with Playwright
+# Pytest Plugin for Visual Testing with Playwright
 
-This plugin enables snapshot testing in playwright like snapshotting screenshots of pages, element handles etc.
+Based on [pixelmatch-py](https://github.com/whtsky/pixelmatch-py) image comparison library.
+
+Expands `assert_snapshot` fixture from [pytest-playwright-snapshot](https://github.com/kumaraditya303/pytest-playwright-snapshot) library 
+
+## Main Features:
+- snapshots creation on the first run
+- visual review of mismatches
+- failing on `--update-snapshots` to make users manually review images
+- snapshot name is optional, `test_name[browser][os].png` is auto-generated by default
+- updated folder structure: `snapshots/file_name/test_name/test_name[browser][os].png`
 
 ## Installation
 
 ```bash
-$ pip install pytest-playwright-snapshot
+$ pip install pytest-playwright-visual
 ```
 
 ## Usage
@@ -17,27 +26,39 @@ Example:
 ```python
 def test_myapp(page, assert_snapshot):
     page.goto("https://example.com")
-    assert_snapshot(page.screenshot(), "example.png")
+    assert_snapshot(page.screenshot())
 ```
-
-Ths first time you run pytest, you will get error like
+Then, run pytest:
+```bash
+$ pytest
+```
+The first time you run pytest, snapshots will be created, and you will get the error:
 
 ```console
-Failed: Snapshot not found, use --update-snapshots to create it.
+Failed: --> New snapshot(s) created. Please review images
 ```
 
-As first you need to create golden snapshots to which this plugin will compare in future.
+The next run, the snapshots comparison will take place. 
 
-To create snapshots run:
+To update snapshots, run:
 
 ```bash
 $ pytest --update-snapshots
 ```
 
-This will create snapshots for your tests, after that you can run the tests are usual and this will compare the snapshots.
+After updating, tests will fail and you will need to review images.
+
+In case of a mismatch, `snapshot_tests_failures` folder will be created with `Actual_..`, `Expected_..` and `Diff_..` images generated.
+
+## Folder Structure Example
 
-There is `threshold` kwarg only option which can be used to set the threshold for the comparison of the screenshots and by default it is `0.1`
+![img_2.png](img_2.png)
 
+## API
+**assert_snapshot(page.screenshot(), threshold: float = 0.1, name='test_name[browser][os].png', fail_fast=False)**
+- `threshold` - sets the threshold for the comparison of the screenshots:`0` to `1`. Default is `0.1`
+- `name` - `.png` extensions only. Default is `test_name[browser][os].png` (recommended)
+- `fail_fast` - If `True`, will fail after first different pixel. `False` by default
 ## License
 
 Apache 2.0 LICENSE

pytest_playwright_visual/plugin.py

@@ -0,0 +1,75 @@
+import sys
+import os
+import shutil
+import math
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Callable
+import pytest
+from PIL import Image
+from pixelmatch.contrib.PIL import pixelmatch
+
+
+@pytest.fixture
+def assert_snapshot(pytestconfig: Any, request: Any, browser_name: str) -> Callable:
+    test_name = f"{str(Path(request.node.name))}[{str(sys.platform)}]"
+    test_dir = str(Path(request.node.name)).split('[', 1)[0]
+
+    def compare(img: bytes, *, diff_pixels: int = 0, diff_ratio: float = 0.0,
+                threshold: float = 0.1, name=f'{test_name}.png', fail_fast=False) -> None:
+        update_snapshot = pytestconfig.getoption("--update-snapshots")
+        test_file_name = str(os.path.basename(Path(request.node.fspath))).strip('.py')
+        filepath = (
+                Path(request.node.fspath).parent.resolve()
+                / 'snapshots'
+                / test_file_name
+                / test_dir
+        )
+        filepath.mkdir(parents=True, exist_ok=True)
+        file = filepath / name
+        # Create a dir where all snapshot test failures will go
+        results_dir_name = (Path(request.node.fspath).parent.resolve()
+                            / "snapshot_tests_failures")
+        test_results_dir = (results_dir_name
+                            / test_file_name / test_name)
+        # Remove a single test's past run dir with actual, diff and expected images
+        if test_results_dir.exists():
+            shutil.rmtree(test_results_dir)
+        if update_snapshot:
+            file.write_bytes(img)
+            pytest.fail("--> Snapshots updated. Please review images")
+        if not file.exists():
+            file.write_bytes(img)
+            # pytest.fail(
+            pytest.fail("--> New snapshot(s) created. Please review images")
+        img_a = Image.open(BytesIO(img))
+        img_b = Image.open(file)
+        img_diff = Image.new("RGBA", img_a.size)
+        mismatch = pixelmatch(img_a, img_b, img_diff, threshold=threshold, fail_fast=fail_fast)
+        if mismatch == 0:
+            return
+        if mismatch <= diff_pixels:
+            return
+        if (mismatch/math.prod(img_a.size)) <= diff_ratio:
+            return
+        else:
+            # Create new test_results folder
+            test_results_dir.mkdir(parents=True, exist_ok=True)
+            img_diff.save(f'{test_results_dir}/Diff_{name}')
+            img_a.save(f'{test_results_dir}/Actual_{name}')
+            img_b.save(f'{test_results_dir}/Expected_{name}')
+            print(f"threshold is {threshold}")
+            print(f"number of mismatch pixels is {mismatch}")
+            pytest.fail("--> Snapshots DO NOT match!")
+
+    return compare
+
+
+def pytest_addoption(parser: Any) -> None:
+    group = parser.getgroup("playwright-snapshot", "Playwright Snapshot")
+    group.addoption(
+        "--update-snapshots",
+        action="store_true",
+        default=False,
+        help="Update snapshots.",
+    )

setup.py

@@ -1,35 +1,35 @@
 from pathlib import Path
-
 from setuptools import setup
 
+
 setup(
-    name="pytest-playwright-snapshot",
-    author="Kumar Aditya",
-    author_email="",
-    description="A pytest wrapper for snapshot testing with playwright",
+    name="pytest-playwright-visual",
+    author="Symon Storozhenko",
+    author_email="[email protected]",
+    description="A pytest fixture for visual testing with Playwright",
     long_description=Path("README.md").read_text(),
     long_description_content_type="text/markdown",
-    url="https://github.com/kumaraditya303/pytest-playwright-snapshot",
-    packages=["pytest_playwright_snapshot"],
+    url="https://github.com/symon-storozhenko/pytest-playwright-visual",
+    packages=["pytest_playwright_visual"],
     include_package_data=True,
     install_requires=[
         "pytest_playwright>=0.1.2",
         "Pillow>=8.2.0",
-        "pixelmatch==0.2.3",
+        "pixelmatch>=0.3.0",
     ],
     entry_points={
-        "pytest11": ["playwright_snapshot = pytest_playwright_snapshot.plugin"]
+        "pytest11": ["playwright_visual = pytest_playwright_visual.plugin"]
     },
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
         "Framework :: Pytest",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     use_scm_version=True,
     setup_requires=["setuptools_scm"],
 )