Skip to content

Commit be6935d

Browse files
committed
Update retry test
1 parent 3a6afee commit be6935d

File tree

1 file changed

+12
-13
lines changed

1 file changed

+12
-13
lines changed

tests/integration/test_crawlers_with_storages.py

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
from crawlee._types import BasicCrawlingContext
21
from tests.integration.conftest import MakeActorFunction, RunActorFunction
32

43

@@ -76,31 +75,31 @@ async def test_actor_on_platform_max_request_retries(
7675
make_actor: MakeActorFunction,
7776
run_actor: RunActorFunction,
7877
) -> None:
79-
"""Test that the actor respects max_requests_per_crawl."""
78+
"""Test that the actor respects max_request_retries."""
8079

8180
async def main() -> None:
8281
"""The crawler entry point."""
82+
from crawlee._types import BasicCrawlingContext
8383
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
8484

8585
from apify import Actor
8686

8787
async with Actor:
88-
max_retries = 2
88+
max_retries = 3
8989
crawler = ParselCrawler(max_request_retries=max_retries)
90-
finished = []
91-
failed = []
90+
failed_counter = 0
9291

93-
@crawler.failed_request_handler
94-
async def failed_handler(context: BasicCrawlingContext, _: Exception) -> None:
95-
failed.add(context.request.url)
92+
@crawler.error_handler
93+
async def failed_handler(_: BasicCrawlingContext, __: Exception) -> None:
94+
nonlocal failed_counter
95+
failed_counter += 1
9696

9797
@crawler.router.default_handler
98-
async def default_handler(context: ParselCrawlingContext) -> None:
99-
finished.append(context.request.url)
98+
async def default_handler(_: ParselCrawlingContext) -> None:
99+
raise RuntimeError('Some error')
100100

101-
await crawler.run(['http://localhost:8080/non-existing-url'])
102-
assert len(finished) == 0
103-
assert len(failed) == max_retries + 1
101+
await crawler.run(['http://localhost:8080/'])
102+
assert failed_counter == max_retries, f'{failed_counter=}' # TODO max_retries + 1
104103

105104
actor = await make_actor(label='crawler-max-retries', main_func=main)
106105
run_result = await run_actor(actor)

0 commit comments

Comments
 (0)