Skip to content

feat: support for handling custom exceptions in middleware. #476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion taskiq/middlewares/simple_retry_middleware.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from logging import getLogger
from typing import Any
from typing import Any, Iterable, Optional

from taskiq.abc.middleware import TaskiqMiddleware
from taskiq.exceptions import NoResultError
Expand All @@ -18,10 +18,12 @@ def __init__(
default_retry_count: int = 3,
default_retry_label: bool = False,
no_result_on_retry: bool = True,
types_of_exceptions: Optional[Iterable[type[BaseException]]] = None,
) -> None:
self.default_retry_count = default_retry_count
self.default_retry_label = default_retry_label
self.no_result_on_retry = no_result_on_retry
self.types_of_exceptions = types_of_exceptions

async def on_error(
self,
Expand All @@ -42,6 +44,12 @@ async def on_error(
:param result: execution result.
:param exception: found exception.
"""
if self.types_of_exceptions is not None and not isinstance(
exception,
tuple(self.types_of_exceptions),
):
return

# Valid exception
if isinstance(exception, NoResultError):
return
Expand Down
11 changes: 10 additions & 1 deletion taskiq/middlewares/smart_retry_middleware.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import datetime
import random
from logging import getLogger
from typing import Any, Optional
from typing import Any, Iterable, Optional

from taskiq import ScheduleSource
from taskiq.abc.middleware import TaskiqMiddleware
Expand Down Expand Up @@ -35,6 +35,7 @@ def __init__(
use_delay_exponent: bool = False,
max_delay_exponent: float = 60,
schedule_source: Optional[ScheduleSource] = None,
types_of_exceptions: Optional[Iterable[type[BaseException]]] = None,
) -> None:
"""
Initialize retry middleware.
Expand All @@ -48,6 +49,7 @@ def __init__(
:param max_delay_exponent: Maximum allowed delay when using backoff.
:param schedule_source: Schedule source to use for scheduling.
If None, the default broker will be used.
:param types_of_exceptions: Types of exceptions to retry from.
"""
super().__init__()
self.default_retry_count = default_retry_count
Expand All @@ -58,6 +60,7 @@ def __init__(
self.use_delay_exponent = use_delay_exponent
self.max_delay_exponent = max_delay_exponent
self.schedule_source = schedule_source
self.types_of_exceptions = types_of_exceptions

if not isinstance(schedule_source, (ScheduleSource, type(None))):
raise TypeError(
Expand Down Expand Up @@ -138,6 +141,12 @@ async def on_error(
:param result: Execution result.
:param exception: Caught exception.
"""
if self.types_of_exceptions is not None and not isinstance(
exception,
tuple(self.types_of_exceptions),
):
return

if isinstance(exception, NoResultError):
return

Expand Down
108 changes: 107 additions & 1 deletion tests/middlewares/test_task_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pytest

from taskiq import InMemoryBroker, SimpleRetryMiddleware
from taskiq import InMemoryBroker, SimpleRetryMiddleware, SmartRetryMiddleware
from taskiq.exceptions import NoResultError


Expand Down Expand Up @@ -151,3 +151,109 @@ def run_task() -> str:

assert runs == 1
assert str(resp.error) == str(runs)


@pytest.mark.anyio
async def test_retry_of_custom_exc_types_of_simple_middleware() -> None:
# test that the passed error will be handled
broker = InMemoryBroker().with_middlewares(
SimpleRetryMiddleware(
no_result_on_retry=True,
default_retry_label=True,
types_of_exceptions=(KeyError, ValueError),
),
)
runs = 0

@broker.task(max_retries=10)
def run_task() -> None:
nonlocal runs

runs += 1

raise ValueError(runs)

task = await run_task.kiq()
resp = await task.wait_result(timeout=1)
with pytest.raises(ValueError):
resp.raise_for_error()

assert runs == 10

# test that an untransmitted error will not be handled
broker = InMemoryBroker().with_middlewares(
SimpleRetryMiddleware(
no_result_on_retry=True,
default_retry_label=True,
types_of_exceptions=(KeyError,),
),
)
runs = 0

@broker.task(max_retries=10)
def run_task2() -> None:
nonlocal runs

runs += 1

raise ValueError(runs)

task = await run_task2.kiq()
resp = await task.wait_result(timeout=1)
with pytest.raises(ValueError):
resp.raise_for_error()

assert runs == 1


@pytest.mark.anyio
async def test_retry_of_custom_exc_types_of_smart_middleware() -> None:
# test that the passed error will be handled
broker = InMemoryBroker().with_middlewares(
SmartRetryMiddleware(
no_result_on_retry=True,
default_retry_label=True,
types_of_exceptions=(KeyError, ValueError),
),
)
runs = 0

@broker.task(max_retries=10)
def run_task() -> None:
nonlocal runs

runs += 1

raise ValueError(runs)

task = await run_task.kiq()
resp = await task.wait_result(timeout=1)
with pytest.raises(ValueError):
resp.raise_for_error()

assert runs == 10

# test that an untransmitted error will not be handled
broker = InMemoryBroker().with_middlewares(
SmartRetryMiddleware(
no_result_on_retry=True,
default_retry_label=True,
types_of_exceptions=(KeyError,),
),
)
runs = 0

@broker.task(max_retries=10)
def run_task2() -> None:
nonlocal runs

runs += 1

raise ValueError(runs)

task = await run_task2.kiq()
resp = await task.wait_result(timeout=1)
with pytest.raises(ValueError):
resp.raise_for_error()

assert runs == 1
Loading