Setup running e2e tests in Docker image (#70)

sfc-gh-gpijanski · web-flow · commit 3abafe831c21 · 2023-08-23T08:53:58.000+02:00
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -9,13 +9,13 @@ on:
       - "master"
 
 jobs:
-  build-examples-templates:
+  build-examples-templates-and-run-dockerized-e2e-tests:
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
-        node-version: 
+        node_version:
           # For details, see: https://nodejs.dev/en/about/releases/
           # Maintenance LTS. End Of Life: 2023-09-11
           - 16.x
@@ -26,18 +26,27 @@ jobs:
         component_lib_version:
           - current
           - develop
+        streamlit_version:
+          - latest
+          - nightly
 
-    name: Examples + Templates / node-version=${{ matrix.node-version }}  / component_lib_version=${{ matrix.component_lib_version }}
+    env:
+      NODE_VERSION: ${{ matrix.node_version }}
+      PYTHON_VERSION: 3.8  # Oldest version supported by Streamlit
+      STREAMLIT_VERSION: ${{ matrix.streamlit_version }}
+      COMPONENT_LIB_VERSION: ${{ matrix.component_lib_version }}
+
+    name: Examples + Templates / node_version=${{ matrix.node_version }} / streamlit_version=${{ matrix.streamlit_version }}  / component_lib_version=${{ matrix.component_lib_version }}
 
     steps:
     - uses: actions/checkout@v3
       with:
         persist-credentials: false
 
-    - name: Use Node.js ${{ matrix.node-version }}
+    - name: Use Node.js ${{ env.NODE_VERSION }}
       uses: actions/setup-node@v3
       with:
-        node-version: ${{ matrix.node-version }}
+        node-version: ${{ env.NODE_VERSION }}
 
     - name: Check dependencies for examples
       run: ./dev.py examples-check-deps
@@ -98,17 +107,17 @@ jobs:
         path: dist/*.whl
         if-no-files-found: error
 
-    - name: Install python dependencies
-      run: ./dev.py install-python-deps
-
-    - name: Install wheel packages
-      run: ./dev.py install-wheel-packages
+    - name: Set up Docker Buildx
+      if: matrix.node_version == '19.x'
+      uses: docker/setup-buildx-action@7703e82fbced3d0c9eec08dff4429c023a5fd9a9 # v2.9.1
 
-    - name: Install browsers
-      run: ./dev.py install-browsers
+    - name: Build docker images
+      if: matrix.node_version == '19.x'
+      run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}"
 
     - name: Run e2e tests
-      run: ./dev.py run-e2e
+      if: matrix.node_version == '19.x'
+      run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}"
 
   build-cookiecutter:
     runs-on: ubuntu-latest
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,41 @@
+# syntax=docker/dockerfile:1.4
+
+ARG PYTHON_VERSION="3.11.4"
+FROM python:${PYTHON_VERSION}-slim-bullseye
+
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+# Setup Pip
+ARG PIP_VERSION="23.2.1"
+ENV PIP_VERSION=${PIP_VERSION}
+
+RUN pip install --no-cache-dir --upgrade "pip==${PIP_VERSION}" && pip --version
+
+ENV PYTHONUNBUFFERED=1
+ENV PIP_ROOT_USER_ACTION=ignore
+RUN mkdir /component
+WORKDIR /component
+
+# Install streamlit and components
+ARG STREAMLIT_VERSION="latest"
+ENV E2E_STREAMLIT_VERSION=${STREAMLIT_VERSION}
+
+RUN <<"EOF"
+if [[ "${E2E_STREAMLIT_VERSION}" == "latest" ]]; then
+  pip install --no-cache-dir "streamlit"
+elif [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then
+  pip uninstall --yes streamlit
+  pip install --no-cache-dir "streamlit-nightly"
+else
+  pip install --no-cache-dir "streamlit==${E2E_STREAMLIT_VERSION}"
+fi
+
+# Coherence check
+installed_streamlit_version=$(python -c "import streamlit; print(streamlit.__version__)")
+echo "Installed Streamlit version: ${installed_streamlit_version}"
+if [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then
+  echo "${installed_streamlit_version}" | grep 'dev'
+else
+  echo "${installed_streamlit_version}" | grep -v 'dev'
+fi
+EOF
diff --git a/dev.py b/dev.py
@@ -14,6 +14,7 @@
 import json
 import sys
 import shutil
+import os
 
 THIS_DIRECTORY = Path(__file__).parent.absolute()
 EXAMPLE_DIRECTORIES = [d for d in (THIS_DIRECTORY / 'examples').iterdir() if d.is_dir()]
@@ -35,60 +36,85 @@ def run_verbose(cmd_args, *args, **kwargs):
 
 # Commands
 def cmd_all_npm_install(args):
-    """"Install all node dependencies for all examples"""
+    """Install all node dependencies for all examples"""
     for project_dir in EXAMPLE_DIRECTORIES + TEMPLATE_DIRECTORIES:
         frontend_dir = next(project_dir.glob("*/frontend/"))
         run_verbose(["npm", "install"], cwd=str(frontend_dir))
 
 
 def cmd_all_npm_build(args):
-    """"Build javascript code for all examples and templates"""
+    """Build javascript code for all examples and templates"""
     for project_dir in EXAMPLE_DIRECTORIES + TEMPLATE_DIRECTORIES:
         frontend_dir = next(project_dir.glob("*/frontend/"))
         run_verbose(["npm", "run", "build"], cwd=str(frontend_dir))
 
 
-def cmd_all_install_python_deps(args):
-    """"Install all dependencies needed to run e2e tests for all examples and templates"""
+def cmd_e2e_build_images(args):
+    """Build docker images for each component e2e tests"""
     for project_dir in EXAMPLE_DIRECTORIES + TEMPLATE_DIRECTORIES:
-        run_verbose(["pip", "install", "-e", ".[devel]"], cwd=str(project_dir))
+        e2e_dir = next(project_dir.glob("**/e2e/"), None)
+        if e2e_dir and os.listdir(e2e_dir):
+            # Define the image tag for the docker image
+            image_tag = (
+                f"component-template:py-{args.python_version}-st-{args.streamlit_version}-component-{project_dir.parts[-1]}"
+            )
+            # Build the docker image with specified build arguments
+            run_verbose(
+                [
+                    "docker",
+                    "build",
+                    ".",
+                    f"--build-arg=STREAMLIT_VERSION={args.streamlit_version}",
+                    f"--build-arg=PYTHON_VERSION={args.python_version}",
+                    f"--tag={image_tag}",
+                    "--progress=plain",
+                ],
+                env={**os.environ, "DOCKER_BUILDKIT": "1"},
+            )
 
 
-def cmd_all_install_wheel_packages(args):
-    """"Install wheel packages of all examples and templates for e2e tests"""
+def cmd_e2e_run(args):
+    """Run e2e tests for all examples and templates in separate docker images"""
     for project_dir in EXAMPLE_DIRECTORIES + TEMPLATE_DIRECTORIES:
-        wheel_files = list(project_dir.glob("dist/*.whl"))
-        if wheel_files:
-            wheel_file = wheel_files[0]
-            run_verbose(["pip", "install", str(wheel_file)], cwd=str(project_dir))
-        else:
-            print(f"No wheel files found in {project_dir}")
-
-
-def cmd_install_browsers(args):
-    """"Install multiple browsers to run e2e for all examples and templates"""
-    run_verbose(["playwright", "install", "webkit", "chromium", "firefox", "--with-deps"])
-
-
-def cmd_all_run_e2e(args):
-    """"Run e2e tests for all examples and templates"""
-    for project_dir in TEMPLATE_DIRECTORIES:
+        container_name = project_dir.parts[-1]
+        image_tag = (
+            f"component-template:py-{args.python_version}-st-{args.streamlit_version}-component-{container_name}"
+        )
         e2e_dir = next(project_dir.glob("**/e2e/"), None)
-        if e2e_dir:
-            with tempfile.TemporaryDirectory() as tmp_dir:
-                run_verbose(['python', '-m', 'venv', f"{tmp_dir}/venv"])
-                wheel_files = list(project_dir.glob("dist/*.whl"))
-                if wheel_files:
-                    wheel_file = wheel_files[0]
-                    run_verbose([f"{tmp_dir}/venv/bin/pip", "install", f"{str(wheel_file)}[devel]"], cwd=str(project_dir))
-                else:
-                    print(f"No wheel files found in {project_dir}")
-                run_verbose([f"{tmp_dir}/venv/bin/pytest", "-s", "--browser", "webkit", "--browser", "chromium", "--browser", "firefox", "--reruns", "5", str(e2e_dir)])
-
-    for project_dir in EXAMPLE_DIRECTORIES:
+        if e2e_dir and os.listdir(e2e_dir):
+            run_verbose([
+                "docker",
+                "run",
+                "--tty",
+                "--rm",
+                "--name", container_name,
+                "--volume", f"{e2e_dir.parent}/:/component/",
+                image_tag,
+                "/bin/sh", "-c",  # Run a shell command inside the container
+                "find /component/dist/ -name '*.whl' | xargs -I {} echo '{}[devel]' | xargs pip install && " # Install whl package and dev dependencies
+                f"playwright install webkit chromium firefox --with-deps && "  # Install browsers
+                f"pytest",  # Run pytest
+                "-s",
+                "--browser", "webkit",
+                "--browser", "chromium",
+                "--browser", "firefox",
+                "--reruns", "5",
+                "--capture=no",
+                "--setup-show"
+            ])
+
+
+def cmd_docker_images_cleanup(args):
+    """Cleanup docker images and containers"""
+    for project_dir in EXAMPLE_DIRECTORIES + TEMPLATE_DIRECTORIES:
+        container_name = project_dir.parts[-1]
+        image_name = (
+            f"component-template:py-{args.python_version}-st-{args.streamlit_version}-component-{container_name}"
+        )
         e2e_dir = next(project_dir.glob("**/e2e/"), None)
-        if e2e_dir:
-            run_verbose(["pytest", "-s", "--browser", "webkit", "--browser", "chromium", "--browser", "firefox", "--reruns", "5", str(e2e_dir)])
+        if e2e_dir and os.listdir(e2e_dir):
+            # Remove the associated Docker image
+            run_verbose(["docker", "rmi", image_name])
 
 
 def cmd_all_python_build_package(args):
@@ -104,8 +130,8 @@ def cmd_all_python_build_package(args):
 
 def check_deps(template_package_json, current_package_json):
     return (
-        check_deps_section(template_package_json, current_package_json, 'dependencies') +
-        check_deps_section(template_package_json, current_package_json, 'devDependencies')
+            check_deps_section(template_package_json, current_package_json, 'dependencies') +
+            check_deps_section(template_package_json, current_package_json, 'devDependencies')
     )
 
 
@@ -169,7 +195,8 @@ def cmd_check_templates_using_cookiecutter(args):
         replay_file_content = json.loads(cookiecutter_variant.replay_file.read_text())
 
         with tempfile.TemporaryDirectory() as output_dir:
-            print(f"Generating template with replay file: {cookiecutter_variant.replay_file.relative_to(THIS_DIRECTORY)}")
+            print(
+                f"Generating template with replay file: {cookiecutter_variant.replay_file.relative_to(THIS_DIRECTORY)}")
             run_verbose(
                 [
                     "cookiecutter",
@@ -243,10 +270,27 @@ def cmd_update_templates(args):
     "examples-check-deps": cmd_example_check_deps,
     "templates-check-not-modified": cmd_check_templates_using_cookiecutter,
     "templates-update": cmd_update_templates,
-    "install-python-deps": cmd_all_install_python_deps,
-    "install-wheel-packages": cmd_all_install_wheel_packages,
-    "install-browsers": cmd_install_browsers,
-    "run-e2e": cmd_all_run_e2e,
+    "e2e-build-images": cmd_e2e_build_images,
+    "e2e-run-tests": cmd_e2e_run,
+    "docker-images-cleanup": cmd_docker_images_cleanup
+}
+
+ARG_STREAMLIT_VERSION = ("--streamlit-version", "latest", "Streamlit version for which tests will be run.")
+ARG_PYTHON_VERSION = ("--python-version", os.environ.get("PYTHON_VERSION", "3.11.4"), "Python version for which tests will be run.")
+
+ARGUMENTS = {
+    "e2e-build-images": [
+        ARG_STREAMLIT_VERSION,
+        ARG_PYTHON_VERSION
+    ],
+    "e2e-run-tests": [
+        ARG_STREAMLIT_VERSION,
+        ARG_PYTHON_VERSION
+    ],
+    "docker-images-cleanup": [
+        (*ARG_STREAMLIT_VERSION[:2], f"Streamlit version used to create the Docker resources"),
+        (*ARG_PYTHON_VERSION[:2], f"Python version used to create the Docker resources")
+    ]
 }
 
 
@@ -256,7 +300,14 @@ def get_parser():
     subparsers = parser.add_subparsers(dest="subcommand", metavar="COMMAND")
     subparsers.required = True
     for command_name, command_fn in COMMANDS.items():
-        subparsers.add_parser(command_name, help=command_fn.__doc__).set_defaults(func=command_fn)
+        subparser = subparsers.add_parser(command_name, help=command_fn.__doc__)
+
+        if command_name in ARGUMENTS:
+            for arg_name, arg_default, arg_help in ARGUMENTS[command_name]:
+                subparser.add_argument(arg_name, default=arg_default, help=arg_help)
+
+        subparser.set_defaults(func=command_fn)
+
     return parser
 
 
diff --git a/examples/RadioButton/e2e/test_radio_button.py b/examples/RadioButton/e2e/test_radio_button.py
@@ -9,6 +9,7 @@
 ROOT_DIRECTORY = Path(__file__).parent.parent.absolute()
 BASIC_EXAMPLE_FILE = ROOT_DIRECTORY / "radio_button" / "example.py"
 
+
 @pytest.fixture(autouse=True, scope="module")
 def streamlit_app():
     with StreamlitRunner(BASIC_EXAMPLE_FILE) as runner:
