Merge pull request #68 from streamlit/STREAMLIT-3894-duplicate-e2e-setup-for-all-examples-and-components

Add test setup for each component

Author: sfc-gh-pbelczyk

README.md

@@ -54,7 +54,7 @@ $ npm run start  # Start the Webpack dev server
 ```
 $ cd template
 $ . venv/bin/activate  # activate the venv you created earlier
-$ streamlit run my_component/__init__.py  # run the example
+$ streamlit run my_component/example.py  # run the example
 ```
 * If all goes well, you should see something like this:
 ![Quickstart Success](quickstart.png)

cookiecutter/{{ cookiecutter.package_name }}/e2e/e2e_utils.py

@@ -0,0 +1,140 @@
+import contextlib
+import logging
+import os
+import shlex
+import socket
+import subprocess
+import sys
+import time
+import typing
+from contextlib import closing
+from tempfile import TemporaryFile
+
+import requests
+
+
+LOGGER = logging.getLogger(__file__)
+
+
+def _find_free_port():
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(("", 0))  # 0 means that the OS chooses a random port
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return int(s.getsockname()[1])  # [1] contains the randomly selected port number
+
+
+class AsyncSubprocess:
+    """A context manager. Wraps subprocess. Popen to capture output safely."""
+
+    def __init__(self, args, cwd=None, env=None):
+        self.args = args
+        self.cwd = cwd
+        self.env = env
+        self._proc = None
+        self._stdout_file = None
+
+    def terminate(self):
+        """Terminate the process and return its stdout/stderr in a string."""
+        if self._proc is not None:
+            self._proc.terminate()
+            self._proc.wait()
+            self._proc = None
+
+        # Read the stdout file and close it
+        stdout = None
+        if self._stdout_file is not None:
+            self._stdout_file.seek(0)
+            stdout = self._stdout_file.read()
+            self._stdout_file.close()
+            self._stdout_file = None
+
+        return stdout
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+
+    def start(self):
+        # Start the process and capture its stdout/stderr output to a temp
+        # file. We do this instead of using subprocess.PIPE (which causes the
+        # Popen object to capture the output to its own internal buffer),
+        # because large amounts of output can cause it to deadlock.
+        self._stdout_file = TemporaryFile("w+")
+        LOGGER.info("Running command: %s", shlex.join(self.args))
+        self._proc = subprocess.Popen(
+            self.args,
+            cwd=self.cwd,
+            stdout=self._stdout_file,
+            stderr=subprocess.STDOUT,
+            text=True,
+            env={**os.environ.copy(), **self.env} if self.env else None,
+        )
+
+    def stop(self):
+        if self._proc is not None:
+            self._proc.terminate()
+            self._proc = None
+        if self._stdout_file is not None:
+            self._stdout_file.close()
+            self._stdout_file = None
+
+
+class StreamlitRunner:
+    def __init__(
+            self, script_path: os.PathLike, server_port: typing.Optional[int] = None
+    ):
+        self._process = None
+        self.server_port = server_port
+        self.script_path = script_path
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.stop()
+
+    def start(self):
+        self.server_port = self.server_port or _find_free_port()
+        self._process = AsyncSubprocess(
+            [
+                sys.executable,
+                "-m",
+                "streamlit",
+                "run",
+                str(self.script_path),
+                f"--server.port={self.server_port}",
+                "--server.headless=true",
+                "--browser.gatherUsageStats=false",
+                "--global.developmentMode=false",
+            ]
+        )
+        self._process.start()
+        if not self.is_server_running():
+            self._process.stop()
+            raise RuntimeError("Application failed to start")
+
+    def stop(self):
+        self._process.stop()
+
+    def is_server_running(self, timeout: int = 30):
+        with requests.Session() as http_session:
+            start_time = time.time()
+            print("Start loop: ", start_time)
+            while True:
+                with contextlib.suppress(requests.RequestException):
+                    response = http_session.get(self.server_url + "/_stcore/health")
+                    if response.text == "ok":
+                        return True
+                time.sleep(3)
+                if time.time() - start_time > 60 * timeout:
+                    return False
+
+    @property
+    def server_url(self):
+        if not self.server_port:
+            raise RuntimeError("Unknown server port")
+        return f"http://localhost:{self.server_port}"

cookiecutter/{{ cookiecutter.package_name }}/e2e/test_template.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+import pytest
+
+from playwright.sync_api import Page, expect
+
+from e2e_utils import StreamlitRunner
+
+ROOT_DIRECTORY = Path(__file__).parent.parent.absolute()
+BASIC_EXAMPLE_FILE = ROOT_DIRECTORY / "my_component" / "example.py"
+
+@pytest.fixture(autouse=True, scope="module")
+def streamlit_app():
+    with StreamlitRunner(BASIC_EXAMPLE_FILE) as runner:
+        yield runner
+
+
+@pytest.fixture(autouse=True, scope="function")
+def go_to_app(page: Page, streamlit_app: StreamlitRunner):
+    page.goto(streamlit_app.server_url)
+    # Wait for app to load
+    page.get_by_role("img", name="Running...").is_hidden()
+
+
+def test_should_render_template(page: Page):
+    frame = page.frame_locator(
+        'iframe[title="my_component\\.my_component"] >> nth=0'
+    )
+
+    expect(page.get_by_text("You've clicked 0 times!").first).to_be_visible()
+
+    frame.get_by_role("button", name="Click me!").click()
+
+    expect(page.get_by_text("You've clicked 1 times!").first).to_be_visible()

cookiecutter/{{ cookiecutter.package_name }}/{{ cookiecutter.import_name }}/__init__.py

@@ -34,7 +34,7 @@
     # replace the `url` param with `path`, and point it to the component's
     # build directory:
     parent_dir = os.path.dirname(os.path.abspath(__file__))
-    build_dir = os.path.join(parent_dir, "frontend-react/build")
+    build_dir = os.path.join(parent_dir, "frontend/build")
     _component_func = components.declare_component("{{ cookiecutter.import_name }}", path=build_dir)
 
 
@@ -75,32 +75,3 @@ def {{ cookiecutter.import_name }}(name, key=None):
     # We could modify the value returned from the component if we wanted.
     # There's no need to do this in our simple example - but it's an option.
     return component_value
-
-
-# Add some test code to play with the component while it's in development.
-# During development, we can run this just as we would any other Streamlit
-# app: `$ streamlit run {{ cookiecutter.import_name }}/__init__.py`
-if not _RELEASE:
-    import streamlit as st
-
-    st.subheader("Component with constant args")
-
-    # Create an instance of our component with a constant `name` arg, and
-    # print its output value.
-    num_clicks = {{ cookiecutter.import_name }}("World")
-    st.markdown("You've clicked %s times!" % int(num_clicks))
-
-    st.markdown("---")
-    st.subheader("Component with variable args")
-
-    # Create a second instance of our component whose `name` arg will vary
-    # based on a text_input widget.
-    #
-    # We use the special "key" argument to assign a fixed identity to this
-    # component instance. By default, when a component's arguments change,
-    # it is considered a new instance and will be re-mounted on the frontend
-    # and lose its current state. In this case, we want to vary the component's
-    # "name" argument without having it get recreated.
-    name_input = st.text_input("Enter a name", value="Streamlit")
-    num_clicks = {{ cookiecutter.import_name }}(name_input, key="foo")
-    st.markdown("You've clicked %s times!" % int(num_clicks))

cookiecutter/{{ cookiecutter.package_name }}/{{ cookiecutter.import_name }}/example.py

@@ -0,0 +1,28 @@
+import streamlit as st
+from {{ cookiecutter.import_name }} import {{ cookiecutter.import_name }}
+
+# Add some test code to play with the component while it's in development.
+# During development, we can run this just as we would any other Streamlit
+# app: `$ streamlit run {{ cookiecutter.import_name }}/example.py`
+
+st.subheader("Component with constant args")
+
+# Create an instance of our component with a constant `name` arg, and
+# print its output value.
+num_clicks = {{ cookiecutter.import_name }}("World")
+st.markdown("You've clicked %s times!" % int(num_clicks))
+
+st.markdown("---")
+st.subheader("Component with variable args")
+
+# Create a second instance of our component whose `name` arg will vary
+# based on a text_input widget.
+#
+# We use the special "key" argument to assign a fixed identity to this
+# component instance. By default, when a component's arguments change,
+# it is considered a new instance and will be re-mounted on the frontend
+# and lose its current state. In this case, we want to vary the component's
+# "name" argument without having it get recreated.
+name_input = st.text_input("Enter a name", value="Streamlit")
+num_clicks = {{ cookiecutter.import_name }}(name_input, key="foo")
+st.markdown("You've clicked %s times!" % int(num_clicks))

dev.py

@@ -72,7 +72,20 @@ def cmd_install_browsers(args):
 
 def cmd_all_run_e2e(args):
     """"Run e2e tests for all examples and templates"""
-    for project_dir in EXAMPLE_DIRECTORIES + TEMPLATE_DIRECTORIES:
+    for project_dir in TEMPLATE_DIRECTORIES:
+        e2e_dir = next(project_dir.glob("**/e2e/"), None)
+        if e2e_dir:
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                run_verbose(['python', '-m', 'venv', f"{tmp_dir}/venv"])
+                wheel_files = list(project_dir.glob("dist/*.whl"))
+                if wheel_files:
+                    wheel_file = wheel_files[0]
+                    run_verbose([f"{tmp_dir}/venv/bin/pip", "install", f"{str(wheel_file)}[devel]"], cwd=str(project_dir))
+                else:
+                    print(f"No wheel files found in {project_dir}")
+                run_verbose([f"{tmp_dir}/venv/bin/pytest", "-s", "--browser", "webkit", "--browser", "chromium", "--browser", "firefox", "--reruns", "5", str(e2e_dir)])
+
+    for project_dir in EXAMPLE_DIRECTORIES:
         e2e_dir = next(project_dir.glob("**/e2e/"), None)
         if e2e_dir:
             run_verbose(["pytest", "-s", "--browser", "webkit", "--browser", "chromium", "--browser", "firefox", "--reruns", "5", str(e2e_dir)])

examples/CustomDataframe/custom_dataframe/__init__.py

@@ -1,6 +1,5 @@
 import os
 
-import streamlit as st
 import pandas as pd
 
 import streamlit.components.v1 as components
@@ -22,20 +21,3 @@
 
 def custom_dataframe(data, key=None):
     return _custom_dataframe(data=data, key=key, default=pd.DataFrame())
-
-
-# Test code to play with the component while it's in development.
-# During development, we can run this just as we would any other Streamlit
-# app: `$ streamlit run custom_dataframe/__init__.py`
-if not _RELEASE:
-    raw_data = {
-        "First Name": ["Jason", "Molly", "Tina", "Jake", "Amy"],
-        "Last Name": ["Miller", "Jacobson", "Ali", "Milner", "Smith"],
-        "Age": [42, 52, 36, 24, 73],
-    }
-
-    df = pd.DataFrame(raw_data, columns=["First Name", "Last Name", "Age"])
-    returned_df = custom_dataframe(df)
-
-    if not returned_df.empty:
-        st.table(returned_df)

examples/CustomDataframe/custom_dataframe/example.py

@@ -0,0 +1,18 @@
+import streamlit as st
+import pandas as pd
+from custom_dataframe import custom_dataframe
+
+# Test code to play with the component while it's in development.
+# During development, we can run this just as we would any other Streamlit
+# app: `$ streamlit run custom_dataframe/example.py`
+raw_data = {
+    "First Name": ["Jason", "Molly", "Tina", "Jake", "Amy"],
+    "Last Name": ["Miller", "Jacobson", "Ali", "Milner", "Smith"],
+    "Age": [42, 52, 36, 24, 73],
+}
+
+df = pd.DataFrame(raw_data, columns=["First Name", "Last Name", "Age"])
+returned_df = custom_dataframe(df)
+
+if not returned_df.empty:
+    st.table(returned_df)