Add test server and some top level Crawler tests

Pijukatel · Pijukatel · commit 17b99c423edc · 2025-07-25T16:27:10.000+02:00
diff --git a/pyproject.toml b/pyproject.toml
@@ -36,14 +36,15 @@ keywords = [
 dependencies = [
     "apify-client>=1.11.0",
     "apify-shared>=1.3.0",
-    "crawlee~=0.6.0",
+    "crawlee[parsel]~=0.6.0",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy
     # https://github.com/apify/apify-sdk-python/issues/460
     "lazy-object-proxy<1.11.0",
     "more_itertools>=10.2.0",
     "typing-extensions>=4.1.0",
+    "uvicorn",
     "websockets>=14.0",
 ]
 
diff --git a/tests/integration/actor_source_base/Dockerfile b/tests/integration/actor_source_base/Dockerfile
@@ -12,4 +12,4 @@ RUN echo "Python version:" \
  && echo "All installed Python packages:" \
  && pip freeze
 
-CMD ["python3", "-m", "src"]
+CMD ["sh", "-c", "python test_server.py & python -m src"]
diff --git a/tests/integration/actor_source_base/test_server.py b/tests/integration/actor_source_base/test_server.py
@@ -0,0 +1,91 @@
+# The Purpose of this server is to run benchmark independent of the network issues
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable, Coroutine
+from typing import Any
+
+from uvicorn import Config
+from uvicorn.server import Server
+from yarl import URL
+
+Receive = Callable[[], Awaitable[dict[str, Any]]]
+Send = Callable[[dict[str, Any]], Coroutine[None, None, None]]
+
+
+async def send_html_response(send: Send, html_content: bytes, status: int = 200) -> None:
+    """Send an HTML response to the client."""
+    await send(
+        {
+            'type': 'http.response.start',
+            'status': status,
+            'headers': [[b'content-type', b'text/html; charset=utf-8']],
+        }
+    )
+    await send({'type': 'http.response.body', 'body': html_content})
+
+
+async def app(scope: dict[str, Any], _: Receive, send: Send) -> None:
+    """Main ASGI application handler that routes requests to specific handlers.
+
+    Args:
+        scope: The ASGI connection scope.
+        _: The ASGI receive function.
+        send: The ASGI send function.
+    """
+    assert scope['type'] == 'http'
+    path = scope['path']
+
+    links = '\n'.join(f'<a href="{path}{i}">{path}{i}</a>' for i in range(10))
+    await send_html_response(
+        send,
+        f"""\
+<html><head>
+    <title>Title for {path} </title>
+</head>
+<body>
+    {links}
+</body></html>""".encode(),
+    )
+
+
+class TestServer(Server):
+    """A test HTTP server implementation based on Uvicorn Server."""
+
+    @property
+    def url(self) -> URL:
+        """Get the base URL of the server.
+
+        Returns:
+            A URL instance with the server's base URL.
+        """
+        protocol = 'https' if self.config.is_ssl else 'http'
+        return URL(f'{protocol}://{self.config.host}:{self.config.port}/')
+
+    async def serve(self) -> None:
+        """Run the server and set up restart capability.
+
+        Args:
+            sockets: Optional list of sockets to bind to.
+        """
+        self.restart_requested = asyncio.Event()
+
+        loop = asyncio.get_event_loop()
+        tasks = {
+            loop.create_task(super().serve()),
+        }
+        await asyncio.wait(tasks)
+
+
+if __name__ == '__main__':
+    asyncio.run(
+        TestServer(
+            config=Config(
+                app=app,
+                lifespan='off',
+                loop='asyncio',
+                port=8080,
+                log_config=None,
+                log_level=logging.CRITICAL,
+            )
+        ).serve()
+    )
diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py
@@ -400,12 +400,12 @@ async def main_server() -> None:
         async with Actor:
 
             class WebhookHandler(BaseHTTPRequestHandler):
-                def do_GET(self) -> None:  # noqa: N802
+                def do_GET(self) -> None:
                     self.send_response(200)
                     self.end_headers()
                     self.wfile.write(bytes('Hello, world!', encoding='utf-8'))
 
-                def do_POST(self) -> None:  # noqa: N802
+                def do_POST(self) -> None:
                     nonlocal webhook_body
                     content_length = self.headers.get('content-length')
                     length = int(content_length) if content_length else 0
diff --git a/tests/integration/test_crawlers_with_storages.py b/tests/integration/test_crawlers_with_storages.py
@@ -0,0 +1,87 @@
+from tests.integration.conftest import MakeActorFunction, RunActorFunction
+
+
+async def test_actor_on_platform_max_crawl_depth(
+    make_actor: MakeActorFunction,
+    run_actor: RunActorFunction,
+) -> None:
+    """
+    Test that the actor respects max_crawl_depth.
+
+    Test server is infinite server http://localhost:8080/{any_number} and each page has links to the next 10 pages.
+    For example: http://localhost:8080/ contains links:
+    http://localhost:8080/0, http://localhost:8080/1, ..., http://localhost:8080/9
+    """
+
+    async def main() -> None:
+        """The crawler entry point."""
+        import re
+
+        from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+        from apify import Actor
+
+        async with Actor:
+            crawler = ParselCrawler(max_crawl_depth=2)
+            finished = []
+            enqueue_pattern = re.compile(r'http://localhost:8080/2+$')
+
+            @crawler.router.default_handler
+            async def default_handler(context: ParselCrawlingContext) -> None:
+                """Default request handler."""
+                context.log.info(f'Processing {context.request.url} ...')
+                await context.enqueue_links(include=[enqueue_pattern])
+                await context.push_data({'Url': context.request.url})
+                finished.append(context.request.url)
+
+            await crawler.run(['http://localhost:8080/'])
+            assert finished == ['http://localhost:8080/', 'http://localhost:8080/2', 'http://localhost:8080/22']
+            # assert some dataset
+
+    actor = await make_actor(label='parsel-crawler', main_func=main)
+    run_result = await run_actor(actor)
+
+    assert run_result.status == 'SUCCEEDED'
+
+
+async def test_actor_on_platform_max_requests_per_crawl(
+    make_actor: MakeActorFunction,
+    run_actor: RunActorFunction,
+) -> None:
+    """
+    Test that the actor respects max_requests_per_crawl.
+
+    Test server is infinite server http://localhost:8080/{any_number} and each page has links to the next 10 pages.
+    For example: http://localhost:8080/ contains links:
+    http://localhost:8080/0, http://localhost:8080/1, ..., http://localhost:8080/9
+    """
+
+    async def main() -> None:
+        """The crawler entry point."""
+        from crawlee import ConcurrencySettings
+        from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+        from apify import Actor
+
+        async with Actor:
+            crawler = ParselCrawler(
+                max_requests_per_crawl=3, concurrency_settings=ConcurrencySettings(max_concurrency=1)
+            )
+            finished = []
+
+            @crawler.router.default_handler
+            async def default_handler(context: ParselCrawlingContext) -> None:
+                """Default request handler."""
+                context.log.info(f'Processing {context.request.url} ...')
+                await context.enqueue_links()
+                await context.push_data({'Url': context.request.url})
+                finished.append(context.request.url)
+
+            await crawler.run(['http://localhost:8080/'])
+            assert len(finished) == 3
+            # assert some dataset
+
+    actor = await make_actor(label='parsel-crawler', main_func=main)
+    run_result = await run_actor(actor)
+
+    assert run_result.status == 'SUCCEEDED'
diff --git a/uv.lock b/uv.lock
