From b55eddfb2f76f88ce007c0b0b2f960ed2e482a9b Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 2 May 2022 16:22:10 +0100 Subject: [PATCH 01/45] Start work on retry --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 std/shared/src/main/scala/cats/effect/std/Retry.scala diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala new file mode 100644 index 0000000000..e69de29bb2 From 3b71e57fda6c59541b4b901ecee1e59b633d63f3 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 2 May 2022 17:12:51 +0100 Subject: [PATCH 02/45] Add cats-retry --- .../main/scala/cats/effect/std/Retry.scala | 710 ++++++++++++++++++ 1 file changed, 710 insertions(+) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index e69de29bb2..164dba6115 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -0,0 +1,710 @@ +package cats.effect.std +package retry +import cats.syntax.all._ + +object Fibonacci { + def fibonacci(n: Int): Long = { + if (n > 0) + fib(n)._1 + else + 0 + } + + // "Fast doubling" Fibonacci algorithm. + // See e.g. http://funloop.org/post/2017-04-14-computing-fibonacci-numbers.html for explanation. + private def fib(n: Int): (Long, Long) = n match { + case 0 => (0, 1) + case m => + val (a, b) = fib(m / 2) + val c = a * (b * 2 - a) + val d = a * a + b * b + if (n % 2 == 0) + (c, d) + else + (d, c + d) + } +} + +import scala.concurrent.duration.FiniteDuration + +sealed trait PolicyDecision + +object PolicyDecision { + case object GiveUp extends PolicyDecision + + final case class DelayAndRetry( + delay: FiniteDuration + ) extends PolicyDecision +} + +sealed trait RetryDetails { + def retriesSoFar: Int + def cumulativeDelay: FiniteDuration + def givingUp: Boolean + def upcomingDelay: Option[FiniteDuration] +} + +object RetryDetails { + final case class GivingUp( + totalRetries: Int, + totalDelay: FiniteDuration + ) extends RetryDetails { + val retriesSoFar: Int = totalRetries + val cumulativeDelay: FiniteDuration = totalDelay + val givingUp: Boolean = true + val upcomingDelay: Option[FiniteDuration] = None + } + + final case class WillDelayAndRetry( + nextDelay: FiniteDuration, + retriesSoFar: Int, + cumulativeDelay: FiniteDuration + ) extends RetryDetails { + val givingUp: Boolean = false + val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) + } +} + +import java.util.concurrent.TimeUnit + +import cats.Applicative +import retry.PolicyDecision._ + +import scala.concurrent.duration.{Duration, FiniteDuration} +import scala.util.Random + +object RetryPolicies { + private val LongMax: BigInt = BigInt(Long.MaxValue) + + /* + * Multiply the given duration by the given multiplier, but cap the result to + * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. + * + * Note: despite the "safe" in the name, we can still create an invalid + * FiniteDuration if the multiplier is negative. But an assumption of the library + * as a whole is that nobody would be silly enough to use negative delays. + */ + private def safeMultiply( + duration: FiniteDuration, + multiplier: Long + ): FiniteDuration = { + val durationNanos = BigInt(duration.toNanos) + val resultNanos = durationNanos * BigInt(multiplier) + val safeResultNanos = resultNanos min LongMax + FiniteDuration(safeResultNanos.toLong, TimeUnit.NANOSECONDS) + } + + /** Don't retry at all and always give up. Only really useful for combining with other policies. + */ + def alwaysGiveUp[M[_]: Applicative]: RetryPolicy[M] = + RetryPolicy.liftWithShow(Function.const(GiveUp), "alwaysGiveUp") + + /** Delay by a constant amount before each retry. Never give up. + */ + def constantDelay[M[_]: Applicative](delay: FiniteDuration): RetryPolicy[M] = + RetryPolicy.liftWithShow( + Function.const(DelayAndRetry(delay)), + show"constantDelay($delay)" + ) + + /** Each delay is twice as long as the previous one. Never give up. + */ + def exponentialBackoff[M[_]: Applicative]( + baseDelay: FiniteDuration + ): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + val delay = + safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) + DelayAndRetry(delay) + }, + show"exponentialBackOff(baseDelay=$baseDelay)" + ) + + /** Retry without delay, giving up after the given number of retries. + */ + def limitRetries[M[_]: Applicative](maxRetries: Int): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + if (status.retriesSoFar >= maxRetries) { + GiveUp + } else { + DelayAndRetry(Duration.Zero) + } + }, + show"limitRetries(maxRetries=$maxRetries)" + ) + + /** Delay(n) = Delay(n - 2) + Delay(n - 1) + * + * e.g. if `baseDelay` is 10 milliseconds, the delays before each retry will be + * 10 ms, 10 ms, 20 ms, 30ms, 50ms, 80ms, 130ms, ... + */ + def fibonacciBackoff[M[_]: Applicative]( + baseDelay: FiniteDuration + ): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + val delay = + safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) + DelayAndRetry(delay) + }, + show"fibonacciBackoff(baseDelay=$baseDelay)" + ) + + /** "Full jitter" backoff algorithm. + * See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ + def fullJitter[M[_]: Applicative](baseDelay: FiniteDuration): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + val e = Math.pow(2, status.retriesSoFar).toLong + val maxDelay = safeMultiply(baseDelay, e) + val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong + DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) + }, + show"fullJitter(baseDelay=$baseDelay)" + ) + + /** Set an upper bound on any individual delay produced by the given policy. + */ + def capDelay[M[_]: Applicative]( + cap: FiniteDuration, + policy: RetryPolicy[M] + ): RetryPolicy[M] = + policy.meet(constantDelay(cap)) + + /** Add an upper bound to a policy such that once the given time-delay + * amount per try has been reached or exceeded, the policy will stop + * retrying and give up. If you need to stop retrying once cumulative + * delay reaches a time-delay amount, use [[limitRetriesByCumulativeDelay]]. + */ + def limitRetriesByDelay[M[_]: Applicative]( + threshold: FiniteDuration, + policy: RetryPolicy[M] + ): RetryPolicy[M] = { + def decideNextRetry(status: RetryStatus): M[PolicyDecision] = + policy.decideNextRetry(status).map { + case r @ DelayAndRetry(delay) => + if (delay > threshold) GiveUp else r + case GiveUp => GiveUp + } + + RetryPolicy.withShow[M]( + decideNextRetry, + show"limitRetriesByDelay(threshold=$threshold, $policy)" + ) + } + + /** Add an upperbound to a policy such that once the cumulative delay + * over all retries has reached or exceeded the given limit, the + * policy will stop retrying and give up. + */ + def limitRetriesByCumulativeDelay[M[_]: Applicative]( + threshold: FiniteDuration, + policy: RetryPolicy[M] + ): RetryPolicy[M] = { + def decideNextRetry(status: RetryStatus): M[PolicyDecision] = + policy.decideNextRetry(status).map { + case r @ DelayAndRetry(delay) => + if (status.cumulativeDelay + delay >= threshold) GiveUp else r + case GiveUp => GiveUp + } + + RetryPolicy.withShow[M]( + decideNextRetry, + show"limitRetriesByCumulativeDelay(threshold=$threshold, $policy)" + ) + } +} + +import scala.concurrent.duration.{Duration, FiniteDuration} + +final case class RetryStatus( + retriesSoFar: Int, + cumulativeDelay: FiniteDuration, + previousDelay: Option[FiniteDuration] +) { + def addRetry(delay: FiniteDuration): RetryStatus = RetryStatus( + retriesSoFar = this.retriesSoFar + 1, + cumulativeDelay = this.cumulativeDelay + delay, + previousDelay = Some(delay) + ) +} + +object RetryStatus { + val NoRetriesYet = RetryStatus(0, Duration.Zero, None) +} + + + +import cats.effect.kernel.Temporal + +import scala.concurrent.duration.FiniteDuration + +trait Sleep[M[_]] { + def sleep(delay: FiniteDuration): M[Unit] +} + +object Sleep { + def apply[M[_]](implicit sleep: Sleep[M]): Sleep[M] = sleep + + implicit def sleepUsingTemporal[F[_]](implicit t: Temporal[F]): Sleep[F] = + (delay: FiniteDuration) => t.sleep(delay) +} + +object implicits extends syntax.AllSyntax + +import cats.{Monad, MonadError} + +import scala.concurrent.duration.FiniteDuration + +package object retry_ { + @deprecated("Use retryingOnFailures instead", "2.1.0") + def retryingM[A] = new RetryingOnFailuresPartiallyApplied[A] + def retryingOnFailures[A] = new RetryingOnFailuresPartiallyApplied[A] + + private def retryingOnFailuresImpl[M[_], A]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit], + status: RetryStatus, + a: A + )(implicit + M: Monad[M], + S: Sleep[M] + ): M[Either[RetryStatus, A]] = { + + def onFalse: M[Either[RetryStatus, A]] = for { + nextStep <- applyPolicy(policy, status) + _ <- onFailure(a, buildRetryDetails(status, nextStep)) + result <- nextStep match { + case NextStep.RetryAfterDelay(delay, updatedStatus) => + S.sleep(delay) *> + M.pure(Left(updatedStatus)) // continue recursion + case NextStep.GiveUp => + M.pure(Right(a)) // stop the recursion + } + } yield result + + wasSuccessful(a).ifM( + M.pure(Right(a)), // stop the recursion + onFalse + ) + } + + private[retry] class RetryingOnFailuresPartiallyApplied[A] { + def apply[M[_]]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit + M: Monad[M], + S: Sleep[M] + ): M[A] = M.tailRecM(RetryStatus.NoRetriesYet) { status => + action.flatMap { a => + retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + } + } + } + + def retryingOnSomeErrors[A] = new RetryingOnSomeErrorsPartiallyApplied[A] + + private def retryingOnSomeErrorsImpl[M[_], A, E]( + policy: RetryPolicy[M], + isWorthRetrying: E => M[Boolean], + onError: (E, RetryDetails) => M[Unit], + status: RetryStatus, + attempt: Either[E, A] + )(implicit + ME: MonadError[M, E], + S: Sleep[M] + ): M[Either[RetryStatus, A]] = attempt match { + case Left(error) => + isWorthRetrying(error).ifM( + for { + nextStep <- applyPolicy(policy, status) + _ <- onError(error, buildRetryDetails(status, nextStep)) + result <- nextStep match { + case NextStep.RetryAfterDelay(delay, updatedStatus) => + S.sleep(delay) *> + ME.pure(Left(updatedStatus)) // continue recursion + case NextStep.GiveUp => + ME.raiseError[A](error).map(Right(_)) // stop the recursion + } + } yield result, + ME.raiseError[A](error).map(Right(_)) // stop the recursion + ) + case Right(success) => + ME.pure(Right(success)) // stop the recursion + } + + private[retry] class RetryingOnSomeErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + isWorthRetrying: E => M[Boolean], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit + ME: MonadError[M, E], + S: Sleep[M] + ): M[A] = ME.tailRecM(RetryStatus.NoRetriesYet) { status => + ME.attempt(action).flatMap { attempt => + retryingOnSomeErrorsImpl( + policy, + isWorthRetrying, + onError, + status, + attempt + ) + } + } + } + + def retryingOnAllErrors[A] = new RetryingOnAllErrorsPartiallyApplied[A] + + private[retry] class RetryingOnAllErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit + ME: MonadError[M, E], + S: Sleep[M] + ): M[A] = + retryingOnSomeErrors[A].apply[M, E](policy, _ => ME.pure(true), onError)( + action + ) + } + + def retryingOnFailuresAndSomeErrors[A] = + new RetryingOnFailuresAndSomeErrorsPartiallyApplied[A] + + private[retry] class RetryingOnFailuresAndSomeErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + isWorthRetrying: E => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit + ME: MonadError[M, E], + S: Sleep[M] + ): M[A] = { + + ME.tailRecM(RetryStatus.NoRetriesYet) { status => + ME.attempt(action).flatMap { + case Right(a) => + retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + case attempt => + retryingOnSomeErrorsImpl( + policy, + isWorthRetrying, + onError, + status, + attempt + ) + } + } + } + } + + def retryingOnFailuresAndAllErrors[A] = + new RetryingOnFailuresAndAllErrorsPartiallyApplied[A] + + private[retry] class RetryingOnFailuresAndAllErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit + ME: MonadError[M, E], + S: Sleep[M] + ): M[A] = + retryingOnFailuresAndSomeErrors[A] + .apply[M, E]( + policy, + wasSuccessful, + _ => ME.pure(true), + onFailure, + onError + )( + action + ) + } + + def noop[M[_]: Monad, A]: (A, RetryDetails) => M[Unit] = + (_, _) => Monad[M].pure(()) + + private[retry] def applyPolicy[M[_]: Monad]( + policy: RetryPolicy[M], + retryStatus: RetryStatus + ): M[NextStep] = + policy.decideNextRetry(retryStatus).map { + case PolicyDecision.DelayAndRetry(delay) => + NextStep.RetryAfterDelay(delay, retryStatus.addRetry(delay)) + case PolicyDecision.GiveUp => + NextStep.GiveUp + } + + private[retry] def buildRetryDetails( + currentStatus: RetryStatus, + nextStep: NextStep + ): RetryDetails = + nextStep match { + case NextStep.RetryAfterDelay(delay, _) => + RetryDetails.WillDelayAndRetry( + delay, + currentStatus.retriesSoFar, + currentStatus.cumulativeDelay + ) + case NextStep.GiveUp => + RetryDetails.GivingUp( + currentStatus.retriesSoFar, + currentStatus.cumulativeDelay + ) + } + + private[retry] sealed trait NextStep + + private[retry] object NextStep { + case object GiveUp extends NextStep + + final case class RetryAfterDelay( + delay: FiniteDuration, + updatedStatus: RetryStatus + ) extends NextStep + } +} + + +trait AllSyntax extends RetrySyntax + + +import cats.{Monad, MonadError} + +trait RetrySyntax { + implicit final def retrySyntaxBase[M[_], A]( + action: => M[A] + ): RetryingOps[M, A] = + new RetryingOps[M, A](action) + + implicit final def retrySyntaxError[M[_], A, E]( + action: => M[A] + )(implicit M: MonadError[M, E]): RetryingErrorOps[M, A, E] = + new RetryingErrorOps[M, A, E](action) +} + +final class RetryingOps[M[_], A](action: => M[A]) { + @deprecated("Use retryingOnFailures instead", "2.1.0") + def retryingM[E]( + wasSuccessful: A => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit] + )(implicit + M: Monad[M], + S: Sleep[M] + ): M[A] = retryingOnFailures(wasSuccessful, policy, onFailure) + + def retryingOnFailures[E]( + wasSuccessful: A => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit] + )(implicit + M: Monad[M], + S: Sleep[M] + ): M[A] = + retry_.retryingOnFailures( + policy = policy, + wasSuccessful = wasSuccessful, + onFailure = onFailure + )(action) +} + +final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit + M: MonadError[M, E] +) { + def retryingOnAllErrors( + policy: RetryPolicy[M], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnAllErrors( + policy = policy, + onError = onError + )(action) + + def retryingOnSomeErrors( + isWorthRetrying: E => M[Boolean], + policy: RetryPolicy[M], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnSomeErrors( + policy = policy, + isWorthRetrying = isWorthRetrying, + onError = onError + )(action) + + def retryingOnFailuresAndAllErrors( + wasSuccessful: A => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnFailuresAndAllErrors( + policy = policy, + wasSuccessful = wasSuccessful, + onFailure = onFailure, + onError = onError + )(action) + + def retryingOnFailuresAndSomeErrors( + wasSuccessful: A => M[Boolean], + isWorthRetrying: E => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnFailuresAndSomeErrors( + policy = policy, + wasSuccessful = wasSuccessful, + isWorthRetrying = isWorthRetrying, + onFailure = onFailure, + onError = onError + )(action) +} + +// trait AllSyntax extends RetrySyntax + +import cats.{Apply, Applicative, Monad, Functor} +import cats.kernel.BoundedSemilattice +import retry.PolicyDecision._ + +import scala.concurrent.duration.Duration +import scala.concurrent.duration.FiniteDuration +import cats.arrow.FunctionK +import cats.Show + +case class RetryPolicy[M[_]]( + decideNextRetry: RetryStatus => M[PolicyDecision] +) { + def show: String = toString + + def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (GiveUp, pd) => pd + case (pd, _) => pd + }, + show"$show.followedBy($rp)" + ) + + /** Combine this schedule with another schedule, giving up when either of the schedules want to give up + * and choosing the maximum of the two delays when both of the schedules want to delay the next retry. + * The dual of the `meet` operation. + */ + def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp + }, + show"$show.join($rp)" + ) + + /** Combine this schedule with another schedule, giving up when both of the schedules want to give up + * and choosing the minimum of the two delays when both of the schedules want to delay the next retry. + * The dual of the `join` operation. + */ + def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp + }, + show"$show.meet($rp)" + ) + + def mapDelay( + f: FiniteDuration => FiniteDuration + )(implicit M: Functor[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map(decideNextRetry(status)) { + case GiveUp => GiveUp + case DelayAndRetry(d) => DelayAndRetry(f(d)) + }, + show"$show.mapDelay()" + ) + + def flatMapDelay( + f: FiniteDuration => M[FiniteDuration] + )(implicit M: Monad[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.flatMap(decideNextRetry(status)) { + case GiveUp => M.pure(GiveUp) + case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) + }, + show"$show.flatMapDelay()" + ) + + def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = + RetryPolicy.withShow( + status => nt(decideNextRetry(status)), + show"$show.mapK()" + ) +} + +object RetryPolicy { + def lift[M[_]]( + f: RetryStatus => PolicyDecision + )(implicit + M: Applicative[M] + ): RetryPolicy[M] = + RetryPolicy[M](decideNextRetry = retryStatus => M.pure(f(retryStatus))) + + def withShow[M[_]]( + decideNextRetry: RetryStatus => M[PolicyDecision], + pretty: => String + ): RetryPolicy[M] = + new RetryPolicy[M](decideNextRetry) { + override def show: String = pretty + override def toString: String = pretty + } + + def liftWithShow[M[_]: Applicative]( + decideNextRetry: RetryStatus => PolicyDecision, + pretty: => String + ): RetryPolicy[M] = + withShow(rs => Applicative[M].pure(decideNextRetry(rs)), pretty) + + implicit def boundedSemilatticeForRetryPolicy[M[_]](implicit + M: Applicative[M] + ): BoundedSemilattice[RetryPolicy[M]] = + new BoundedSemilattice[RetryPolicy[M]] { + override def empty: RetryPolicy[M] = + RetryPolicies.constantDelay[M](Duration.Zero) + + override def combine( + x: RetryPolicy[M], + y: RetryPolicy[M] + ): RetryPolicy[M] = x.join(y) + } + + implicit def showForRetryPolicy[M[_]]: Show[RetryPolicy[M]] = + Show.show(_.show) +} From 6a447b43b12e5031c3afad4e4f0a6906b0363690 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 2 May 2022 17:22:33 +0100 Subject: [PATCH 03/45] Simplify imports --- .../main/scala/cats/effect/std/Retry.scala | 266 +++++++++--------- 1 file changed, 126 insertions(+), 140 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 164dba6115..8a3162fd71 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -1,6 +1,15 @@ package cats.effect.std package retry + +import cats._ import cats.syntax.all._ +import cats.arrow.FunctionK +import cats.kernel.BoundedSemilattice +import cats.effect.kernel.Temporal +import retry.PolicyDecision._ +import scala.concurrent.duration.{Duration, FiniteDuration} +import scala.util.Random +import java.util.concurrent.TimeUnit object Fibonacci { def fibonacci(n: Int): Long = { @@ -25,7 +34,7 @@ object Fibonacci { } } -import scala.concurrent.duration.FiniteDuration + sealed trait PolicyDecision @@ -65,13 +74,122 @@ object RetryDetails { } } -import java.util.concurrent.TimeUnit -import cats.Applicative -import retry.PolicyDecision._ +case class RetryPolicy[M[_]]( + decideNextRetry: RetryStatus => M[PolicyDecision] +) { + def show: String = toString -import scala.concurrent.duration.{Duration, FiniteDuration} -import scala.util.Random + def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (GiveUp, pd) => pd + case (pd, _) => pd + }, + show"$show.followedBy($rp)" + ) + + /** Combine this schedule with another schedule, giving up when either of the schedules want to give up + * and choosing the maximum of the two delays when both of the schedules want to delay the next retry. + * The dual of the `meet` operation. + */ + def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp + }, + show"$show.join($rp)" + ) + + /** Combine this schedule with another schedule, giving up when both of the schedules want to give up + * and choosing the minimum of the two delays when both of the schedules want to delay the next retry. + * The dual of the `join` operation. + */ + def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp + }, + show"$show.meet($rp)" + ) + + def mapDelay( + f: FiniteDuration => FiniteDuration + )(implicit M: Functor[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map(decideNextRetry(status)) { + case GiveUp => GiveUp + case DelayAndRetry(d) => DelayAndRetry(f(d)) + }, + show"$show.mapDelay()" + ) + + def flatMapDelay( + f: FiniteDuration => M[FiniteDuration] + )(implicit M: Monad[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.flatMap(decideNextRetry(status)) { + case GiveUp => M.pure(GiveUp) + case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) + }, + show"$show.flatMapDelay()" + ) + + def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = + RetryPolicy.withShow( + status => nt(decideNextRetry(status)), + show"$show.mapK()" + ) +} + +object RetryPolicy { + def lift[M[_]]( + f: RetryStatus => PolicyDecision + )(implicit + M: Applicative[M] + ): RetryPolicy[M] = + RetryPolicy[M](decideNextRetry = retryStatus => M.pure(f(retryStatus))) + + def withShow[M[_]]( + decideNextRetry: RetryStatus => M[PolicyDecision], + pretty: => String + ): RetryPolicy[M] = + new RetryPolicy[M](decideNextRetry) { + override def show: String = pretty + override def toString: String = pretty + } + + def liftWithShow[M[_]: Applicative]( + decideNextRetry: RetryStatus => PolicyDecision, + pretty: => String + ): RetryPolicy[M] = + withShow(rs => Applicative[M].pure(decideNextRetry(rs)), pretty) + + implicit def boundedSemilatticeForRetryPolicy[M[_]](implicit + M: Applicative[M] + ): BoundedSemilattice[RetryPolicy[M]] = + new BoundedSemilattice[RetryPolicy[M]] { + override def empty: RetryPolicy[M] = + RetryPolicies.constantDelay[M](Duration.Zero) + + override def combine( + x: RetryPolicy[M], + y: RetryPolicy[M] + ): RetryPolicy[M] = x.join(y) + } + + implicit def showForRetryPolicy[M[_]]: Show[RetryPolicy[M]] = + Show.show(_.show) +} object RetryPolicies { private val LongMax: BigInt = BigInt(Long.MaxValue) @@ -218,7 +336,7 @@ object RetryPolicies { } } -import scala.concurrent.duration.{Duration, FiniteDuration} + final case class RetryStatus( retriesSoFar: Int, @@ -238,9 +356,6 @@ object RetryStatus { -import cats.effect.kernel.Temporal - -import scala.concurrent.duration.FiniteDuration trait Sleep[M[_]] { def sleep(delay: FiniteDuration): M[Unit] @@ -255,9 +370,6 @@ object Sleep { object implicits extends syntax.AllSyntax -import cats.{Monad, MonadError} - -import scala.concurrent.duration.FiniteDuration package object retry_ { @deprecated("Use retryingOnFailures instead", "2.1.0") @@ -490,7 +602,7 @@ package object retry_ { trait AllSyntax extends RetrySyntax -import cats.{Monad, MonadError} + trait RetrySyntax { implicit final def retrySyntaxBase[M[_], A]( @@ -582,129 +694,3 @@ final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit )(action) } -// trait AllSyntax extends RetrySyntax - -import cats.{Apply, Applicative, Monad, Functor} -import cats.kernel.BoundedSemilattice -import retry.PolicyDecision._ - -import scala.concurrent.duration.Duration -import scala.concurrent.duration.FiniteDuration -import cats.arrow.FunctionK -import cats.Show - -case class RetryPolicy[M[_]]( - decideNextRetry: RetryStatus => M[PolicyDecision] -) { - def show: String = toString - - def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (GiveUp, pd) => pd - case (pd, _) => pd - }, - show"$show.followedBy($rp)" - ) - - /** Combine this schedule with another schedule, giving up when either of the schedules want to give up - * and choosing the maximum of the two delays when both of the schedules want to delay the next retry. - * The dual of the `meet` operation. - */ - def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow[M]( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp - }, - show"$show.join($rp)" - ) - - /** Combine this schedule with another schedule, giving up when both of the schedules want to give up - * and choosing the minimum of the two delays when both of the schedules want to delay the next retry. - * The dual of the `join` operation. - */ - def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow[M]( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) - case (s @ DelayAndRetry(_), GiveUp) => s - case (GiveUp, s @ DelayAndRetry(_)) => s - case _ => GiveUp - }, - show"$show.meet($rp)" - ) - - def mapDelay( - f: FiniteDuration => FiniteDuration - )(implicit M: Functor[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.map(decideNextRetry(status)) { - case GiveUp => GiveUp - case DelayAndRetry(d) => DelayAndRetry(f(d)) - }, - show"$show.mapDelay()" - ) - - def flatMapDelay( - f: FiniteDuration => M[FiniteDuration] - )(implicit M: Monad[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.flatMap(decideNextRetry(status)) { - case GiveUp => M.pure(GiveUp) - case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) - }, - show"$show.flatMapDelay()" - ) - - def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = - RetryPolicy.withShow( - status => nt(decideNextRetry(status)), - show"$show.mapK()" - ) -} - -object RetryPolicy { - def lift[M[_]]( - f: RetryStatus => PolicyDecision - )(implicit - M: Applicative[M] - ): RetryPolicy[M] = - RetryPolicy[M](decideNextRetry = retryStatus => M.pure(f(retryStatus))) - - def withShow[M[_]]( - decideNextRetry: RetryStatus => M[PolicyDecision], - pretty: => String - ): RetryPolicy[M] = - new RetryPolicy[M](decideNextRetry) { - override def show: String = pretty - override def toString: String = pretty - } - - def liftWithShow[M[_]: Applicative]( - decideNextRetry: RetryStatus => PolicyDecision, - pretty: => String - ): RetryPolicy[M] = - withShow(rs => Applicative[M].pure(decideNextRetry(rs)), pretty) - - implicit def boundedSemilatticeForRetryPolicy[M[_]](implicit - M: Applicative[M] - ): BoundedSemilattice[RetryPolicy[M]] = - new BoundedSemilattice[RetryPolicy[M]] { - override def empty: RetryPolicy[M] = - RetryPolicies.constantDelay[M](Duration.Zero) - - override def combine( - x: RetryPolicy[M], - y: RetryPolicy[M] - ): RetryPolicy[M] = x.join(y) - } - - implicit def showForRetryPolicy[M[_]]: Show[RetryPolicy[M]] = - Show.show(_.show) -} From 77cc90654b2c7170dead815adf348aaad02f50ed Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 11 May 2022 00:40:12 +0100 Subject: [PATCH 04/45] Aggregate code by area --- .../main/scala/cats/effect/std/Retry.scala | 134 ++++++++---------- 1 file changed, 62 insertions(+), 72 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 8a3162fd71..e97ab722a2 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -11,29 +11,22 @@ import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.Random import java.util.concurrent.TimeUnit -object Fibonacci { - def fibonacci(n: Int): Long = { - if (n > 0) - fib(n)._1 - else - 0 - } - // "Fast doubling" Fibonacci algorithm. - // See e.g. http://funloop.org/post/2017-04-14-computing-fibonacci-numbers.html for explanation. - private def fib(n: Int): (Long, Long) = n match { - case 0 => (0, 1) - case m => - val (a, b) = fib(m / 2) - val c = a * (b * 2 - a) - val d = a * a + b * b - if (n % 2 == 0) - (c, d) - else - (d, c + d) - } +final case class RetryStatus( + retriesSoFar: Int, + cumulativeDelay: FiniteDuration, + previousDelay: Option[FiniteDuration] +) { + def addRetry(delay: FiniteDuration): RetryStatus = RetryStatus( + retriesSoFar = this.retriesSoFar + 1, + cumulativeDelay = this.cumulativeDelay + delay, + previousDelay = Some(delay) + ) } +object RetryStatus { + val NoRetriesYet = RetryStatus(0, Duration.Zero, None) +} sealed trait PolicyDecision @@ -46,35 +39,6 @@ object PolicyDecision { ) extends PolicyDecision } -sealed trait RetryDetails { - def retriesSoFar: Int - def cumulativeDelay: FiniteDuration - def givingUp: Boolean - def upcomingDelay: Option[FiniteDuration] -} - -object RetryDetails { - final case class GivingUp( - totalRetries: Int, - totalDelay: FiniteDuration - ) extends RetryDetails { - val retriesSoFar: Int = totalRetries - val cumulativeDelay: FiniteDuration = totalDelay - val givingUp: Boolean = true - val upcomingDelay: Option[FiniteDuration] = None - } - - final case class WillDelayAndRetry( - nextDelay: FiniteDuration, - retriesSoFar: Int, - cumulativeDelay: FiniteDuration - ) extends RetryDetails { - val givingUp: Boolean = false - val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) - } -} - - case class RetryPolicy[M[_]]( decideNextRetry: RetryStatus => M[PolicyDecision] ) { @@ -337,26 +301,6 @@ object RetryPolicies { } - -final case class RetryStatus( - retriesSoFar: Int, - cumulativeDelay: FiniteDuration, - previousDelay: Option[FiniteDuration] -) { - def addRetry(delay: FiniteDuration): RetryStatus = RetryStatus( - retriesSoFar = this.retriesSoFar + 1, - cumulativeDelay = this.cumulativeDelay + delay, - previousDelay = Some(delay) - ) -} - -object RetryStatus { - val NoRetriesYet = RetryStatus(0, Duration.Zero, None) -} - - - - trait Sleep[M[_]] { def sleep(delay: FiniteDuration): M[Unit] } @@ -370,6 +314,33 @@ object Sleep { object implicits extends syntax.AllSyntax +sealed trait RetryDetails { + def retriesSoFar: Int + def cumulativeDelay: FiniteDuration + def givingUp: Boolean + def upcomingDelay: Option[FiniteDuration] +} + +object RetryDetails { + final case class GivingUp( + totalRetries: Int, + totalDelay: FiniteDuration + ) extends RetryDetails { + val retriesSoFar: Int = totalRetries + val cumulativeDelay: FiniteDuration = totalDelay + val givingUp: Boolean = true + val upcomingDelay: Option[FiniteDuration] = None + } + + final case class WillDelayAndRetry( + nextDelay: FiniteDuration, + retriesSoFar: Int, + cumulativeDelay: FiniteDuration + ) extends RetryDetails { + val givingUp: Boolean = false + val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) + } +} package object retry_ { @deprecated("Use retryingOnFailures instead", "2.1.0") @@ -601,9 +572,6 @@ package object retry_ { trait AllSyntax extends RetrySyntax - - - trait RetrySyntax { implicit final def retrySyntaxBase[M[_], A]( action: => M[A] @@ -694,3 +662,25 @@ final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit )(action) } +object Fibonacci { + def fibonacci(n: Int): Long = { + if (n > 0) + fib(n)._1 + else + 0 + } + + // "Fast doubling" Fibonacci algorithm. + // See e.g. http://funloop.org/post/2017-04-14-computing-fibonacci-numbers.html for explanation. + private def fib(n: Int): (Long, Long) = n match { + case 0 => (0, 1) + case m => + val (a, b) = fib(m / 2) + val c = a * (b * 2 - a) + val d = a * a + b * b + if (n % 2 == 0) + (c, d) + else + (d, c + d) + } +} From 3ab660f2eea27cf63f7a766bf68901e9a7076594 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 11 May 2022 00:56:08 +0100 Subject: [PATCH 05/45] Start sketching cats-effect idiomatic Retry type --- .../main/scala/cats/effect/std/Retry.scala | 321 +++++++++++------- 1 file changed, 195 insertions(+), 126 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index e97ab722a2..5068771fe4 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -11,6 +11,97 @@ import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.Random import java.util.concurrent.TimeUnit +abstract class Retry[M[_]] { + def decideNextRetry: RetryStatus => M[PolicyDecision] + def show: String = toString + + def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (GiveUp, pd) => pd + case (pd, _) => pd + }, + show"$show.followedBy($rp)" + ) + + /** + * Combine this schedule with another schedule, giving up when either of the schedules want to + * give up and choosing the maximum of the two delays when both of the schedules want to delay + * the next retry. The dual of the `meet` operation. + */ + def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp + }, + show"$show.join($rp)" + ) + + /** + * Combine this schedule with another schedule, giving up when both of the schedules want to + * give up and choosing the minimum of the two delays when both of the schedules want to delay + * the next retry. The dual of the `join` operation. + */ + def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp + }, + show"$show.meet($rp)" + ) + + def mapDelay( + f: FiniteDuration => FiniteDuration + )(implicit M: Functor[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map(decideNextRetry(status)) { + case GiveUp => GiveUp + case DelayAndRetry(d) => DelayAndRetry(f(d)) + }, + show"$show.mapDelay()" + ) + + def flatMapDelay( + f: FiniteDuration => M[FiniteDuration] + )(implicit M: Monad[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.flatMap(decideNextRetry(status)) { + case GiveUp => M.pure(GiveUp) + case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) + }, + show"$show.flatMapDelay()" + ) + + def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = + RetryPolicy.withShow( + status => nt(decideNextRetry(status)), + show"$show.mapK()" + ) +} +object Retry { + final case class Status( + retriesSoFar: Int, + cumulativeDelay: FiniteDuration, + previousDelay: Option[FiniteDuration] + ) { + def addRetry(delay: FiniteDuration): Retry.Status = Retry.Status( + retriesSoFar = this.retriesSoFar + 1, + cumulativeDelay = this.cumulativeDelay + delay, + previousDelay = Some(delay) + ) + } + + val NoRetriesYet = Retry.Status(0, Duration.Zero, None) +} final case class RetryStatus( retriesSoFar: Int, @@ -28,7 +119,6 @@ object RetryStatus { val NoRetriesYet = RetryStatus(0, Duration.Zero, None) } - sealed trait PolicyDecision object PolicyDecision { @@ -49,37 +139,39 @@ case class RetryPolicy[M[_]]( status => M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { case (GiveUp, pd) => pd - case (pd, _) => pd + case (pd, _) => pd }, show"$show.followedBy($rp)" ) - /** Combine this schedule with another schedule, giving up when either of the schedules want to give up - * and choosing the maximum of the two delays when both of the schedules want to delay the next retry. - * The dual of the `meet` operation. - */ + /** + * Combine this schedule with another schedule, giving up when either of the schedules want to + * give up and choosing the maximum of the two delays when both of the schedules want to delay + * the next retry. The dual of the `meet` operation. + */ def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = RetryPolicy.withShow[M]( status => M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp + case _ => GiveUp }, show"$show.join($rp)" ) - /** Combine this schedule with another schedule, giving up when both of the schedules want to give up - * and choosing the minimum of the two delays when both of the schedules want to delay the next retry. - * The dual of the `join` operation. - */ + /** + * Combine this schedule with another schedule, giving up when both of the schedules want to + * give up and choosing the minimum of the two delays when both of the schedules want to delay + * the next retry. The dual of the `join` operation. + */ def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = RetryPolicy.withShow[M]( status => M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) - case (s @ DelayAndRetry(_), GiveUp) => s - case (GiveUp, s @ DelayAndRetry(_)) => s - case _ => GiveUp + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp }, show"$show.meet($rp)" ) @@ -90,7 +182,7 @@ case class RetryPolicy[M[_]]( RetryPolicy.withShow( status => M.map(decideNextRetry(status)) { - case GiveUp => GiveUp + case GiveUp => GiveUp case DelayAndRetry(d) => DelayAndRetry(f(d)) }, show"$show.mapDelay()" @@ -102,7 +194,7 @@ case class RetryPolicy[M[_]]( RetryPolicy.withShow( status => M.flatMap(decideNextRetry(status)) { - case GiveUp => M.pure(GiveUp) + case GiveUp => M.pure(GiveUp) case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) }, show"$show.flatMapDelay()" @@ -118,9 +210,7 @@ case class RetryPolicy[M[_]]( object RetryPolicy { def lift[M[_]]( f: RetryStatus => PolicyDecision - )(implicit - M: Applicative[M] - ): RetryPolicy[M] = + )(implicit M: Applicative[M]): RetryPolicy[M] = RetryPolicy[M](decideNextRetry = retryStatus => M.pure(f(retryStatus))) def withShow[M[_]]( @@ -128,7 +218,7 @@ object RetryPolicy { pretty: => String ): RetryPolicy[M] = new RetryPolicy[M](decideNextRetry) { - override def show: String = pretty + override def show: String = pretty override def toString: String = pretty } @@ -138,9 +228,8 @@ object RetryPolicy { ): RetryPolicy[M] = withShow(rs => Applicative[M].pure(decideNextRetry(rs)), pretty) - implicit def boundedSemilatticeForRetryPolicy[M[_]](implicit - M: Applicative[M] - ): BoundedSemilattice[RetryPolicy[M]] = + implicit def boundedSemilatticeForRetryPolicy[M[_]]( + implicit M: Applicative[M]): BoundedSemilattice[RetryPolicy[M]] = new BoundedSemilattice[RetryPolicy[M]] { override def empty: RetryPolicy[M] = RetryPolicies.constantDelay[M](Duration.Zero) @@ -170,27 +259,31 @@ object RetryPolicies { duration: FiniteDuration, multiplier: Long ): FiniteDuration = { - val durationNanos = BigInt(duration.toNanos) - val resultNanos = durationNanos * BigInt(multiplier) + val durationNanos = BigInt(duration.toNanos) + val resultNanos = durationNanos * BigInt(multiplier) val safeResultNanos = resultNanos min LongMax FiniteDuration(safeResultNanos.toLong, TimeUnit.NANOSECONDS) } - /** Don't retry at all and always give up. Only really useful for combining with other policies. - */ + /** + * Don't retry at all and always give up. Only really useful for combining with other + * policies. + */ def alwaysGiveUp[M[_]: Applicative]: RetryPolicy[M] = RetryPolicy.liftWithShow(Function.const(GiveUp), "alwaysGiveUp") - /** Delay by a constant amount before each retry. Never give up. - */ + /** + * Delay by a constant amount before each retry. Never give up. + */ def constantDelay[M[_]: Applicative](delay: FiniteDuration): RetryPolicy[M] = RetryPolicy.liftWithShow( Function.const(DelayAndRetry(delay)), show"constantDelay($delay)" ) - /** Each delay is twice as long as the previous one. Never give up. - */ + /** + * Each delay is twice as long as the previous one. Never give up. + */ def exponentialBackoff[M[_]: Applicative]( baseDelay: FiniteDuration ): RetryPolicy[M] = @@ -203,8 +296,9 @@ object RetryPolicies { show"exponentialBackOff(baseDelay=$baseDelay)" ) - /** Retry without delay, giving up after the given number of retries. - */ + /** + * Retry without delay, giving up after the given number of retries. + */ def limitRetries[M[_]: Applicative](maxRetries: Int): RetryPolicy[M] = RetryPolicy.liftWithShow( { status => @@ -217,11 +311,12 @@ object RetryPolicies { show"limitRetries(maxRetries=$maxRetries)" ) - /** Delay(n) = Delay(n - 2) + Delay(n - 1) - * - * e.g. if `baseDelay` is 10 milliseconds, the delays before each retry will be - * 10 ms, 10 ms, 20 ms, 30ms, 50ms, 80ms, 130ms, ... - */ + /** + * Delay(n) = Delay(n - 2) + Delay(n - 1) + * + * e.g. if `baseDelay` is 10 milliseconds, the delays before each retry will be 10 ms, 10 ms, + * 20 ms, 30ms, 50ms, 80ms, 130ms, ... + */ def fibonacciBackoff[M[_]: Applicative]( baseDelay: FiniteDuration ): RetryPolicy[M] = @@ -234,33 +329,36 @@ object RetryPolicies { show"fibonacciBackoff(baseDelay=$baseDelay)" ) - /** "Full jitter" backoff algorithm. - * See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ - */ + /** + * "Full jitter" backoff algorithm. See + * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ def fullJitter[M[_]: Applicative](baseDelay: FiniteDuration): RetryPolicy[M] = RetryPolicy.liftWithShow( { status => - val e = Math.pow(2, status.retriesSoFar).toLong - val maxDelay = safeMultiply(baseDelay, e) + val e = Math.pow(2, status.retriesSoFar).toLong + val maxDelay = safeMultiply(baseDelay, e) val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) }, show"fullJitter(baseDelay=$baseDelay)" ) - /** Set an upper bound on any individual delay produced by the given policy. - */ + /** + * Set an upper bound on any individual delay produced by the given policy. + */ def capDelay[M[_]: Applicative]( cap: FiniteDuration, policy: RetryPolicy[M] ): RetryPolicy[M] = policy.meet(constantDelay(cap)) - /** Add an upper bound to a policy such that once the given time-delay - * amount per try has been reached or exceeded, the policy will stop - * retrying and give up. If you need to stop retrying once cumulative - * delay reaches a time-delay amount, use [[limitRetriesByCumulativeDelay]]. - */ + /** + * Add an upper bound to a policy such that once the given time-delay amount per try + * has been reached or exceeded, the policy will stop retrying and give up. If you need to + * stop retrying once cumulative delay reaches a time-delay amount, use + * [[limitRetriesByCumulativeDelay]]. + */ def limitRetriesByDelay[M[_]: Applicative]( threshold: FiniteDuration, policy: RetryPolicy[M] @@ -278,10 +376,10 @@ object RetryPolicies { ) } - /** Add an upperbound to a policy such that once the cumulative delay - * over all retries has reached or exceeded the given limit, the - * policy will stop retrying and give up. - */ + /** + * Add an upperbound to a policy such that once the cumulative delay over all retries has + * reached or exceeded the given limit, the policy will stop retrying and give up. + */ def limitRetriesByCumulativeDelay[M[_]: Applicative]( threshold: FiniteDuration, policy: RetryPolicy[M] @@ -300,7 +398,6 @@ object RetryPolicies { } } - trait Sleep[M[_]] { def sleep(delay: FiniteDuration): M[Unit] } @@ -326,9 +423,9 @@ object RetryDetails { totalRetries: Int, totalDelay: FiniteDuration ) extends RetryDetails { - val retriesSoFar: Int = totalRetries - val cumulativeDelay: FiniteDuration = totalDelay - val givingUp: Boolean = true + val retriesSoFar: Int = totalRetries + val cumulativeDelay: FiniteDuration = totalDelay + val givingUp: Boolean = true val upcomingDelay: Option[FiniteDuration] = None } @@ -337,14 +434,14 @@ object RetryDetails { retriesSoFar: Int, cumulativeDelay: FiniteDuration ) extends RetryDetails { - val givingUp: Boolean = false + val givingUp: Boolean = false val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) } } package object retry_ { @deprecated("Use retryingOnFailures instead", "2.1.0") - def retryingM[A] = new RetryingOnFailuresPartiallyApplied[A] + def retryingM[A] = new RetryingOnFailuresPartiallyApplied[A] def retryingOnFailures[A] = new RetryingOnFailuresPartiallyApplied[A] private def retryingOnFailuresImpl[M[_], A]( @@ -353,14 +450,11 @@ package object retry_ { onFailure: (A, RetryDetails) => M[Unit], status: RetryStatus, a: A - )(implicit - M: Monad[M], - S: Sleep[M] - ): M[Either[RetryStatus, A]] = { + )(implicit M: Monad[M], S: Sleep[M]): M[Either[RetryStatus, A]] = { def onFalse: M[Either[RetryStatus, A]] = for { nextStep <- applyPolicy(policy, status) - _ <- onFailure(a, buildRetryDetails(status, nextStep)) + _ <- onFailure(a, buildRetryDetails(status, nextStep)) result <- nextStep match { case NextStep.RetryAfterDelay(delay, updatedStatus) => S.sleep(delay) *> @@ -383,13 +477,11 @@ package object retry_ { onFailure: (A, RetryDetails) => M[Unit] )( action: => M[A] - )(implicit - M: Monad[M], - S: Sleep[M] - ): M[A] = M.tailRecM(RetryStatus.NoRetriesYet) { status => - action.flatMap { a => - retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) - } + )(implicit M: Monad[M], S: Sleep[M]): M[A] = M.tailRecM(RetryStatus.NoRetriesYet) { + status => + action.flatMap { a => + retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + } } } @@ -401,15 +493,12 @@ package object retry_ { onError: (E, RetryDetails) => M[Unit], status: RetryStatus, attempt: Either[E, A] - )(implicit - ME: MonadError[M, E], - S: Sleep[M] - ): M[Either[RetryStatus, A]] = attempt match { + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[Either[RetryStatus, A]] = attempt match { case Left(error) => isWorthRetrying(error).ifM( for { nextStep <- applyPolicy(policy, status) - _ <- onError(error, buildRetryDetails(status, nextStep)) + _ <- onError(error, buildRetryDetails(status, nextStep)) result <- nextStep match { case NextStep.RetryAfterDelay(delay, updatedStatus) => S.sleep(delay) *> @@ -431,20 +520,18 @@ package object retry_ { onError: (E, RetryDetails) => M[Unit] )( action: => M[A] - )(implicit - ME: MonadError[M, E], - S: Sleep[M] - ): M[A] = ME.tailRecM(RetryStatus.NoRetriesYet) { status => - ME.attempt(action).flatMap { attempt => - retryingOnSomeErrorsImpl( - policy, - isWorthRetrying, - onError, - status, - attempt - ) + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = + ME.tailRecM(RetryStatus.NoRetriesYet) { status => + ME.attempt(action).flatMap { attempt => + retryingOnSomeErrorsImpl( + policy, + isWorthRetrying, + onError, + status, + attempt + ) + } } - } } def retryingOnAllErrors[A] = new RetryingOnAllErrorsPartiallyApplied[A] @@ -455,10 +542,7 @@ package object retry_ { onError: (E, RetryDetails) => M[Unit] )( action: => M[A] - )(implicit - ME: MonadError[M, E], - S: Sleep[M] - ): M[A] = + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = retryingOnSomeErrors[A].apply[M, E](policy, _ => ME.pure(true), onError)( action ) @@ -476,10 +560,7 @@ package object retry_ { onError: (E, RetryDetails) => M[Unit] )( action: => M[A] - )(implicit - ME: MonadError[M, E], - S: Sleep[M] - ): M[A] = { + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = { ME.tailRecM(RetryStatus.NoRetriesYet) { status => ME.attempt(action).flatMap { @@ -509,20 +590,16 @@ package object retry_ { onError: (E, RetryDetails) => M[Unit] )( action: => M[A] - )(implicit - ME: MonadError[M, E], - S: Sleep[M] - ): M[A] = - retryingOnFailuresAndSomeErrors[A] - .apply[M, E]( - policy, - wasSuccessful, - _ => ME.pure(true), - onFailure, - onError - )( - action - ) + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = + retryingOnFailuresAndSomeErrors[A].apply[M, E]( + policy, + wasSuccessful, + _ => ME.pure(true), + onFailure, + onError + )( + action + ) } def noop[M[_]: Monad, A]: (A, RetryDetails) => M[Unit] = @@ -569,7 +646,6 @@ package object retry_ { } } - trait AllSyntax extends RetrySyntax trait RetrySyntax { @@ -590,19 +666,14 @@ final class RetryingOps[M[_], A](action: => M[A]) { wasSuccessful: A => M[Boolean], policy: RetryPolicy[M], onFailure: (A, RetryDetails) => M[Unit] - )(implicit - M: Monad[M], - S: Sleep[M] - ): M[A] = retryingOnFailures(wasSuccessful, policy, onFailure) + )(implicit M: Monad[M], S: Sleep[M]): M[A] = + retryingOnFailures(wasSuccessful, policy, onFailure) def retryingOnFailures[E]( wasSuccessful: A => M[Boolean], policy: RetryPolicy[M], onFailure: (A, RetryDetails) => M[Unit] - )(implicit - M: Monad[M], - S: Sleep[M] - ): M[A] = + )(implicit M: Monad[M], S: Sleep[M]): M[A] = retry_.retryingOnFailures( policy = policy, wasSuccessful = wasSuccessful, @@ -610,9 +681,7 @@ final class RetryingOps[M[_], A](action: => M[A]) { )(action) } -final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit - M: MonadError[M, E] -) { +final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit M: MonadError[M, E]) { def retryingOnAllErrors( policy: RetryPolicy[M], onError: (E, RetryDetails) => M[Unit] @@ -676,8 +745,8 @@ object Fibonacci { case 0 => (0, 1) case m => val (a, b) = fib(m / 2) - val c = a * (b * 2 - a) - val d = a * a + b * b + val c = a * (b * 2 - a) + val d = a * a + b * b if (n % 2 == 0) (c, d) else From 58583b2cb18e67b689971cf7743b61d7527c5b4e Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 14 May 2022 16:34:58 +0100 Subject: [PATCH 06/45] Compiling version of cats-effect idiomatic retry --- .../main/scala/cats/effect/std/Retry.scala | 158 ++++++++++++------ 1 file changed, 104 insertions(+), 54 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 5068771fe4..2d92fcbc5e 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -11,98 +11,148 @@ import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.Random import java.util.concurrent.TimeUnit -abstract class Retry[M[_]] { - def decideNextRetry: RetryStatus => M[PolicyDecision] - def show: String = toString +abstract class Retry[F[_]] { + def decideNextRetry: Retry.Status => F[PolicyDecision] - def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (GiveUp, pd) => pd - case (pd, _) => pd - }, - show"$show.followedBy($rp)" - ) + def followedBy(rp: Retry[F]): Retry[F] /** * Combine this schedule with another schedule, giving up when either of the schedules want to * give up and choosing the maximum of the two delays when both of the schedules want to delay * the next retry. The dual of the `meet` operation. */ - def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow[M]( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp - }, - show"$show.join($rp)" - ) + def join(rp: Retry[F]): Retry[F] /** * Combine this schedule with another schedule, giving up when both of the schedules want to * give up and choosing the minimum of the two delays when both of the schedules want to delay * the next retry. The dual of the `join` operation. */ - def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow[M]( + def meet(rp: Retry[F]): Retry[F] + + def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] + + def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] + + def mapK[G[_]: Monad](f: F ~> G): Retry[G] +} +object Retry { + final case class Status( + retriesSoFar: Int, + cumulativeDelay: FiniteDuration, + previousDelay: Option[FiniteDuration] + ) { + def addRetry(delay: FiniteDuration): Retry.Status = Retry.Status( + retriesSoFar = this.retriesSoFar + 1, + cumulativeDelay = this.cumulativeDelay + delay, + previousDelay = Some(delay) + ) + } + + val NoRetriesYet = Retry.Status(0, Duration.Zero, None) + +// def lift[F[_]]( +// f: Retry.Status => PolicyDecision +// )(implicit F: Applicative[F]): Retry[F] = +// Retry[F](decideNextRetry = retryStatus => F.pure(f(retryStatus))) + +def withShow[F[_]: Monad]( + decideNextRetry: Retry.Status => F[PolicyDecision], + pretty: => String +): Retry[F] = + new RetryImpl[F](decideNextRetry, pretty) + +// def liftWithShow[F[_]: Applicative]( +// decideNextRetry: Retry.Status => PolicyDecision, +// pretty: => String +// ): Retry[F] = +// withShow(rs => Applicative[F].pure(decideNextRetry(rs)), pretty) + +// implicit def boundedSemilatticeForRetry[F[_]]( +// implicit F: Applicative[F]): BoundedSemilattice[Retry[F]] = +// new BoundedSemilattice[Retry[F]] { +// override def empty: Retry[F] = +// RetryPolicies.constantDelay[F](Duration.Zero) + +// override def combine( +// x: Retry[F], +// y: Retry[F] +// ): Retry[F] = x.join(y) +// } + +// implicit def showForRetry[F[_]]: Show[Retry[F]] = +// Show.show(_.show) + + private final case class RetryImpl[F[_]](decideNextRetry: Retry.Status => F[PolicyDecision], pretty: String)(implicit F: Monad[F]) extends Retry[F] { + + override def toString: String = pretty + + def followedBy(rp: Retry[F]): Retry[F] = + Retry.withShow( + status => + F.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (GiveUp, pd) => pd + case (pd, _) => pd + }, + s"$this.followedBy($rp)" + ) + + def join(rp: Retry[F]): Retry[F] = + Retry.withShow[F]( status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + F.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp + }, + s"$this.join($rp)" + ) + + def meet(rp: Retry[F]): Retry[F] = + Retry.withShow[F]( + status => + F.map2(decideNextRetry(status), rp.decideNextRetry(status)) { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) case (s @ DelayAndRetry(_), GiveUp) => s case (GiveUp, s @ DelayAndRetry(_)) => s case _ => GiveUp }, - show"$show.meet($rp)" + s"$this.meet($rp)" ) def mapDelay( f: FiniteDuration => FiniteDuration - )(implicit M: Functor[M]): RetryPolicy[M] = - RetryPolicy.withShow( + ): Retry[F] = + Retry.withShow( status => - M.map(decideNextRetry(status)) { + F.map(decideNextRetry(status)) { case GiveUp => GiveUp case DelayAndRetry(d) => DelayAndRetry(f(d)) }, - show"$show.mapDelay()" + s"$this.mapDelay()" ) def flatMapDelay( - f: FiniteDuration => M[FiniteDuration] - )(implicit M: Monad[M]): RetryPolicy[M] = - RetryPolicy.withShow( + f: FiniteDuration => F[FiniteDuration] + ): Retry[F] = + Retry.withShow( status => - M.flatMap(decideNextRetry(status)) { - case GiveUp => M.pure(GiveUp) - case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) + F.flatMap(decideNextRetry(status)) { + case GiveUp => F.pure(GiveUp) + case DelayAndRetry(d) => F.map(f(d))(DelayAndRetry(_)) }, - show"$show.flatMapDelay()" + s"$this.flatMapDelay()" ) - def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = - RetryPolicy.withShow( - status => nt(decideNextRetry(status)), - show"$show.mapK()" - ) -} -object Retry { - final case class Status( - retriesSoFar: Int, - cumulativeDelay: FiniteDuration, - previousDelay: Option[FiniteDuration] - ) { - def addRetry(delay: FiniteDuration): Retry.Status = Retry.Status( - retriesSoFar = this.retriesSoFar + 1, - cumulativeDelay = this.cumulativeDelay + delay, - previousDelay = Some(delay) + def mapK[G[_]: Monad](f: F ~> G): Retry[G] = + Retry.withShow( + status => f(decideNextRetry(status)), + s"$this.mapK()" ) } - - val NoRetriesYet = Retry.Status(0, Duration.Zero, None) } +/////// + final case class RetryStatus( retriesSoFar: Int, cumulativeDelay: FiniteDuration, From 840d042cf386b318cd29d5db08edbeddb8cf3a39 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 14 May 2022 16:55:05 +0100 Subject: [PATCH 07/45] Some renaming --- .../main/scala/cats/effect/std/Retry.scala | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 2d92fcbc5e..c3750d6fc4 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -12,23 +12,23 @@ import scala.util.Random import java.util.concurrent.TimeUnit abstract class Retry[F[_]] { - def decideNextRetry: Retry.Status => F[PolicyDecision] + def nextRetry: Retry.Status => F[PolicyDecision] - def followedBy(rp: Retry[F]): Retry[F] + def followedBy(r: Retry[F]): Retry[F] /** * Combine this schedule with another schedule, giving up when either of the schedules want to * give up and choosing the maximum of the two delays when both of the schedules want to delay * the next retry. The dual of the `meet` operation. */ - def join(rp: Retry[F]): Retry[F] + def join(r: Retry[F]): Retry[F] /** * Combine this schedule with another schedule, giving up when both of the schedules want to * give up and choosing the minimum of the two delays when both of the schedules want to delay * the next retry. The dual of the `join` operation. */ - def meet(rp: Retry[F]): Retry[F] + def meet(r: Retry[F]): Retry[F] def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] @@ -54,19 +54,19 @@ object Retry { // def lift[F[_]]( // f: Retry.Status => PolicyDecision // )(implicit F: Applicative[F]): Retry[F] = -// Retry[F](decideNextRetry = retryStatus => F.pure(f(retryStatus))) +// Retry[F](nextRetry = retryStatus => F.pure(f(retryStatus))) def withShow[F[_]: Monad]( - decideNextRetry: Retry.Status => F[PolicyDecision], + nextRetry: Retry.Status => F[PolicyDecision], pretty: => String ): Retry[F] = - new RetryImpl[F](decideNextRetry, pretty) + new RetryImpl[F](nextRetry, pretty) // def liftWithShow[F[_]: Applicative]( -// decideNextRetry: Retry.Status => PolicyDecision, +// nextRetry: Retry.Status => PolicyDecision, // pretty: => String // ): Retry[F] = -// withShow(rs => Applicative[F].pure(decideNextRetry(rs)), pretty) +// withShow(rs => Applicative[F].pure(nextRetry(rs)), pretty) // implicit def boundedSemilatticeForRetry[F[_]]( // implicit F: Applicative[F]): BoundedSemilattice[Retry[F]] = @@ -83,40 +83,40 @@ def withShow[F[_]: Monad]( // implicit def showForRetry[F[_]]: Show[Retry[F]] = // Show.show(_.show) - private final case class RetryImpl[F[_]](decideNextRetry: Retry.Status => F[PolicyDecision], pretty: String)(implicit F: Monad[F]) extends Retry[F] { + private final case class RetryImpl[F[_]](nextRetry: Retry.Status => F[PolicyDecision], pretty: String)(implicit F: Monad[F]) extends Retry[F] { override def toString: String = pretty - def followedBy(rp: Retry[F]): Retry[F] = + def followedBy(r: Retry[F]): Retry[F] = Retry.withShow( status => - F.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + F.map2(nextRetry(status), r.nextRetry(status)) { case (GiveUp, pd) => pd case (pd, _) => pd }, - s"$this.followedBy($rp)" + s"$this.followedBy($r)" ) - def join(rp: Retry[F]): Retry[F] = + def join(r: Retry[F]): Retry[F] = Retry.withShow[F]( status => - F.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + F.map2(nextRetry(status), r.nextRetry(status)) { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) case _ => GiveUp }, - s"$this.join($rp)" + s"$this.join($r)" ) - def meet(rp: Retry[F]): Retry[F] = + def meet(r: Retry[F]): Retry[F] = Retry.withShow[F]( status => - F.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + F.map2(nextRetry(status), r.nextRetry(status)) { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) case (s @ DelayAndRetry(_), GiveUp) => s case (GiveUp, s @ DelayAndRetry(_)) => s case _ => GiveUp }, - s"$this.meet($rp)" + s"$this.meet($r)" ) def mapDelay( @@ -124,7 +124,7 @@ def withShow[F[_]: Monad]( ): Retry[F] = Retry.withShow( status => - F.map(decideNextRetry(status)) { + F.map(nextRetry(status)) { case GiveUp => GiveUp case DelayAndRetry(d) => DelayAndRetry(f(d)) }, @@ -136,7 +136,7 @@ def withShow[F[_]: Monad]( ): Retry[F] = Retry.withShow( status => - F.flatMap(decideNextRetry(status)) { + F.flatMap(nextRetry(status)) { case GiveUp => F.pure(GiveUp) case DelayAndRetry(d) => F.map(f(d))(DelayAndRetry(_)) }, @@ -145,7 +145,7 @@ def withShow[F[_]: Monad]( def mapK[G[_]: Monad](f: F ~> G): Retry[G] = Retry.withShow( - status => f(decideNextRetry(status)), + status => f(nextRetry(status)), s"$this.mapK()" ) } From 5af2c8f806c9a9cb9f9f0a7a2f10d5e6089231a2 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 14 May 2022 16:58:32 +0100 Subject: [PATCH 08/45] Use cats syntax for Retry --- .../src/main/scala/cats/effect/std/Retry.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index c3750d6fc4..36cf18ebf0 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -83,14 +83,14 @@ def withShow[F[_]: Monad]( // implicit def showForRetry[F[_]]: Show[Retry[F]] = // Show.show(_.show) - private final case class RetryImpl[F[_]](nextRetry: Retry.Status => F[PolicyDecision], pretty: String)(implicit F: Monad[F]) extends Retry[F] { + private final case class RetryImpl[F[_]: Monad](nextRetry: Retry.Status => F[PolicyDecision], pretty: String) extends Retry[F] { override def toString: String = pretty def followedBy(r: Retry[F]): Retry[F] = Retry.withShow( status => - F.map2(nextRetry(status), r.nextRetry(status)) { + (nextRetry(status), r.nextRetry(status)).mapN { case (GiveUp, pd) => pd case (pd, _) => pd }, @@ -100,7 +100,7 @@ def withShow[F[_]: Monad]( def join(r: Retry[F]): Retry[F] = Retry.withShow[F]( status => - F.map2(nextRetry(status), r.nextRetry(status)) { + (nextRetry(status), r.nextRetry(status)).mapN { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) case _ => GiveUp }, @@ -110,7 +110,7 @@ def withShow[F[_]: Monad]( def meet(r: Retry[F]): Retry[F] = Retry.withShow[F]( status => - F.map2(nextRetry(status), r.nextRetry(status)) { + (nextRetry(status), r.nextRetry(status)).mapN { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) case (s @ DelayAndRetry(_), GiveUp) => s case (GiveUp, s @ DelayAndRetry(_)) => s @@ -124,7 +124,7 @@ def withShow[F[_]: Monad]( ): Retry[F] = Retry.withShow( status => - F.map(nextRetry(status)) { + nextRetry(status).map { case GiveUp => GiveUp case DelayAndRetry(d) => DelayAndRetry(f(d)) }, @@ -136,9 +136,9 @@ def withShow[F[_]: Monad]( ): Retry[F] = Retry.withShow( status => - F.flatMap(nextRetry(status)) { - case GiveUp => F.pure(GiveUp) - case DelayAndRetry(d) => F.map(f(d))(DelayAndRetry(_)) + nextRetry(status).flatMap { + case GiveUp => GiveUp.pure[F].widen[PolicyDecision] + case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) }, s"$this.flatMapDelay()" ) From 38c1f1ab4aa3a12f14f6b935bfa5c24dc60bdac0 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 14 May 2022 17:04:32 +0100 Subject: [PATCH 09/45] New constructors for Retry --- .../main/scala/cats/effect/std/Retry.scala | 91 ++++++++----------- 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 36cf18ebf0..b113de876c 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -51,22 +51,20 @@ object Retry { val NoRetriesYet = Retry.Status(0, Duration.Zero, None) -// def lift[F[_]]( -// f: Retry.Status => PolicyDecision -// )(implicit F: Applicative[F]): Retry[F] = -// Retry[F](nextRetry = retryStatus => F.pure(f(retryStatus))) - -def withShow[F[_]: Monad]( + def apply[F[_]: Monad]( nextRetry: Retry.Status => F[PolicyDecision], - pretty: => String -): Retry[F] = - new RetryImpl[F](nextRetry, pretty) + pretty: String = "" + ): Retry[F] = + new RetryImpl[F](nextRetry, pretty) -// def liftWithShow[F[_]: Applicative]( -// nextRetry: Retry.Status => PolicyDecision, -// pretty: => String -// ): Retry[F] = -// withShow(rs => Applicative[F].pure(nextRetry(rs)), pretty) + def lift[F[_]: Monad]( + nextRetry: Retry.Status => PolicyDecision, + pretty: String = "" + ): Retry[F] = + apply[F]( + status => nextRetry(status).pure[F], + pretty + ) // implicit def boundedSemilatticeForRetry[F[_]]( // implicit F: Applicative[F]): BoundedSemilattice[Retry[F]] = @@ -88,66 +86,57 @@ def withShow[F[_]: Monad]( override def toString: String = pretty def followedBy(r: Retry[F]): Retry[F] = - Retry.withShow( - status => - (nextRetry(status), r.nextRetry(status)).mapN { + Retry( + status => (nextRetry(status), r.nextRetry(status)).mapN { case (GiveUp, pd) => pd case (pd, _) => pd }, - s"$this.followedBy($r)" - ) + s"$this.followedBy($r)" + ) - def join(r: Retry[F]): Retry[F] = - Retry.withShow[F]( - status => - (nextRetry(status), r.nextRetry(status)).mapN { + def join(r: Retry[F]): Retry[F] = + Retry[F]( + status => (nextRetry(status), r.nextRetry(status)).mapN { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) case _ => GiveUp }, - s"$this.join($r)" - ) + s"$this.join($r)" + ) - def meet(r: Retry[F]): Retry[F] = - Retry.withShow[F]( - status => - (nextRetry(status), r.nextRetry(status)).mapN { + def meet(r: Retry[F]): Retry[F] = + Retry[F]( + status => (nextRetry(status), r.nextRetry(status)).mapN { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) case (s @ DelayAndRetry(_), GiveUp) => s case (GiveUp, s @ DelayAndRetry(_)) => s case _ => GiveUp }, - s"$this.meet($r)" - ) + s"$this.meet($r)" + ) - def mapDelay( - f: FiniteDuration => FiniteDuration - ): Retry[F] = - Retry.withShow( - status => - nextRetry(status).map { + def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = + Retry( + status => nextRetry(status).map { case GiveUp => GiveUp case DelayAndRetry(d) => DelayAndRetry(f(d)) }, - s"$this.mapDelay()" + s"$this.mapDelay()" ) - def flatMapDelay( - f: FiniteDuration => F[FiniteDuration] - ): Retry[F] = - Retry.withShow( - status => - nextRetry(status).flatMap { + def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = + Retry( + status => nextRetry(status).flatMap { case GiveUp => GiveUp.pure[F].widen[PolicyDecision] case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) }, - s"$this.flatMapDelay()" - ) + s"$this.flatMapDelay()" + ) - def mapK[G[_]: Monad](f: F ~> G): Retry[G] = - Retry.withShow( - status => f(nextRetry(status)), - s"$this.mapK()" - ) + def mapK[G[_]: Monad](f: F ~> G): Retry[G] = + Retry( + status => f(nextRetry(status)), + s"$this.mapK()" + ) } } From 7baf47a4e0d25e9a20f80885e540e7e782d0440b Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 14 May 2022 17:06:40 +0100 Subject: [PATCH 10/45] Make nextRetry into a method --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index b113de876c..b9dda48b17 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -12,7 +12,7 @@ import scala.util.Random import java.util.concurrent.TimeUnit abstract class Retry[F[_]] { - def nextRetry: Retry.Status => F[PolicyDecision] + def nextRetry(status: Retry.Status): F[PolicyDecision] def followedBy(r: Retry[F]): Retry[F] @@ -81,7 +81,10 @@ object Retry { // implicit def showForRetry[F[_]]: Show[Retry[F]] = // Show.show(_.show) - private final case class RetryImpl[F[_]: Monad](nextRetry: Retry.Status => F[PolicyDecision], pretty: String) extends Retry[F] { + private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[PolicyDecision], pretty: String) extends Retry[F] { + + override def nextRetry(status: Retry.Status): F[PolicyDecision] = + nextRetry_(status) override def toString: String = pretty From 037e8b46103027b7e407d2a2981ef6ddb3e4d740 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 14 May 2022 23:03:52 +0100 Subject: [PATCH 11/45] Add concrete retry strategies --- .../main/scala/cats/effect/std/OldRetry.scala | 666 ++++++++++++++++ .../main/scala/cats/effect/std/Retry.scala | 720 +++--------------- 2 files changed, 775 insertions(+), 611 deletions(-) create mode 100644 std/shared/src/main/scala/cats/effect/std/OldRetry.scala diff --git a/std/shared/src/main/scala/cats/effect/std/OldRetry.scala b/std/shared/src/main/scala/cats/effect/std/OldRetry.scala new file mode 100644 index 0000000000..037399e6ff --- /dev/null +++ b/std/shared/src/main/scala/cats/effect/std/OldRetry.scala @@ -0,0 +1,666 @@ +package cats.effect.std +package retry + +import cats._ +import cats.syntax.all._ +import cats.arrow.FunctionK +import cats.kernel.BoundedSemilattice +import cats.effect.kernel.Temporal +import retry.PolicyDecision._ +import scala.concurrent.duration.{Duration, FiniteDuration} +import scala.util.Random +import java.util.concurrent.TimeUnit + + +final case class RetryStatus( + retriesSoFar: Int, + cumulativeDelay: FiniteDuration, + previousDelay: Option[FiniteDuration] +) { + def addRetry(delay: FiniteDuration): RetryStatus = RetryStatus( + retriesSoFar = this.retriesSoFar + 1, + cumulativeDelay = this.cumulativeDelay + delay, + previousDelay = Some(delay) + ) +} + +object RetryStatus { + val NoRetriesYet = RetryStatus(0, Duration.Zero, None) +} + +sealed trait PolicyDecision + +object PolicyDecision { + case object GiveUp extends PolicyDecision + + final case class DelayAndRetry( + delay: FiniteDuration + ) extends PolicyDecision +} + +case class RetryPolicy[M[_]]( + decideNextRetry: RetryStatus => M[PolicyDecision] +) { + def show: String = toString + + def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (GiveUp, pd) => pd + case (pd, _) => pd + }, + show"$show.followedBy($rp)" + ) + + /** + * Combine this schedule with another schedule, giving up when either of the schedules want to + * give up and choosing the maximum of the two delays when both of the schedules want to delay + * the next retry. The dual of the `meet` operation. + */ + def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp + }, + show"$show.join($rp)" + ) + + /** + * Combine this schedule with another schedule, giving up when both of the schedules want to + * give up and choosing the minimum of the two delays when both of the schedules want to delay + * the next retry. The dual of the `join` operation. + */ + def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = + RetryPolicy.withShow[M]( + status => + M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp + }, + show"$show.meet($rp)" + ) + + def mapDelay( + f: FiniteDuration => FiniteDuration + )(implicit M: Functor[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.map(decideNextRetry(status)) { + case GiveUp => GiveUp + case DelayAndRetry(d) => DelayAndRetry(f(d)) + }, + show"$show.mapDelay()" + ) + + def flatMapDelay( + f: FiniteDuration => M[FiniteDuration] + )(implicit M: Monad[M]): RetryPolicy[M] = + RetryPolicy.withShow( + status => + M.flatMap(decideNextRetry(status)) { + case GiveUp => M.pure(GiveUp) + case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) + }, + show"$show.flatMapDelay()" + ) + + def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = + RetryPolicy.withShow( + status => nt(decideNextRetry(status)), + show"$show.mapK()" + ) +} + +object RetryPolicy { + def lift[M[_]]( + f: RetryStatus => PolicyDecision + )(implicit M: Applicative[M]): RetryPolicy[M] = + RetryPolicy[M](decideNextRetry = retryStatus => M.pure(f(retryStatus))) + + def withShow[M[_]]( + decideNextRetry: RetryStatus => M[PolicyDecision], + pretty: => String + ): RetryPolicy[M] = + new RetryPolicy[M](decideNextRetry) { + override def show: String = pretty + override def toString: String = pretty + } + + def liftWithShow[M[_]: Applicative]( + decideNextRetry: RetryStatus => PolicyDecision, + pretty: => String + ): RetryPolicy[M] = + withShow(rs => Applicative[M].pure(decideNextRetry(rs)), pretty) + + implicit def boundedSemilatticeForRetryPolicy[M[_]]( + implicit M: Applicative[M]): BoundedSemilattice[RetryPolicy[M]] = + new BoundedSemilattice[RetryPolicy[M]] { + override def empty: RetryPolicy[M] = + RetryPolicies.constantDelay[M](Duration.Zero) + + override def combine( + x: RetryPolicy[M], + y: RetryPolicy[M] + ): RetryPolicy[M] = x.join(y) + } + + implicit def showForRetryPolicy[M[_]]: Show[RetryPolicy[M]] = + Show.show(_.show) +} + +object RetryPolicies { + private val LongMax: BigInt = BigInt(Long.MaxValue) + + /* + * Multiply the given duration by the given multiplier, but cap the result to + * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. + * + * Note: despite the "safe" in the name, we can still create an invalid + * FiniteDuration if the multiplier is negative. But an assumption of the library + * as a whole is that nobody would be silly enough to use negative delays. + */ + private def safeMultiply( + duration: FiniteDuration, + multiplier: Long + ): FiniteDuration = { + val durationNanos = BigInt(duration.toNanos) + val resultNanos = durationNanos * BigInt(multiplier) + val safeResultNanos = resultNanos min LongMax + FiniteDuration(safeResultNanos.toLong, TimeUnit.NANOSECONDS) + } + + /** + * Don't retry at all and always give up. Only really useful for combining with other + * policies. + */ + def alwaysGiveUp[M[_]: Applicative]: RetryPolicy[M] = + RetryPolicy.liftWithShow(Function.const(GiveUp), "alwaysGiveUp") + + /** + * Delay by a constant amount before each retry. Never give up. + */ + def constantDelay[M[_]: Applicative](delay: FiniteDuration): RetryPolicy[M] = + RetryPolicy.liftWithShow( + Function.const(DelayAndRetry(delay)), + show"constantDelay($delay)" + ) + + /** + * Each delay is twice as long as the previous one. Never give up. + */ + def exponentialBackoff[M[_]: Applicative]( + baseDelay: FiniteDuration + ): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + val delay = + safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) + DelayAndRetry(delay) + }, + show"exponentialBackOff(baseDelay=$baseDelay)" + ) + + /** + * Retry without delay, giving up after the given number of retries. + */ + def limitRetries[M[_]: Applicative](maxRetries: Int): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + if (status.retriesSoFar >= maxRetries) { + GiveUp + } else { + DelayAndRetry(Duration.Zero) + } + }, + show"limitRetries(maxRetries=$maxRetries)" + ) + + /** + * Delay(n) = Delay(n - 2) + Delay(n - 1) + * + * e.g. if `baseDelay` is 10 milliseconds, the delays before each retry will be 10 ms, 10 ms, + * 20 ms, 30ms, 50ms, 80ms, 130ms, ... + */ + def fibonacciBackoff[M[_]: Applicative]( + baseDelay: FiniteDuration + ): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + val delay = + safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) + DelayAndRetry(delay) + }, + show"fibonacciBackoff(baseDelay=$baseDelay)" + ) + + /** + * "Full jitter" backoff algorithm. See + * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ + def fullJitter[M[_]: Applicative](baseDelay: FiniteDuration): RetryPolicy[M] = + RetryPolicy.liftWithShow( + { status => + val e = Math.pow(2, status.retriesSoFar).toLong + val maxDelay = safeMultiply(baseDelay, e) + val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong + DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) + }, + show"fullJitter(baseDelay=$baseDelay)" + ) + + /** + * Set an upper bound on any individual delay produced by the given policy. + */ + def capDelay[M[_]: Applicative]( + cap: FiniteDuration, + policy: RetryPolicy[M] + ): RetryPolicy[M] = + policy.meet(constantDelay(cap)) + + /** + * Add an upper bound to a policy such that once the given time-delay amount per try + * has been reached or exceeded, the policy will stop retrying and give up. If you need to + * stop retrying once cumulative delay reaches a time-delay amount, use + * [[limitRetriesByCumulativeDelay]]. + */ + def limitRetriesByDelay[M[_]: Applicative]( + threshold: FiniteDuration, + policy: RetryPolicy[M] + ): RetryPolicy[M] = { + def decideNextRetry(status: RetryStatus): M[PolicyDecision] = + policy.decideNextRetry(status).map { + case r @ DelayAndRetry(delay) => + if (delay > threshold) GiveUp else r + case GiveUp => GiveUp + } + + RetryPolicy.withShow[M]( + decideNextRetry, + show"limitRetriesByDelay(threshold=$threshold, $policy)" + ) + } + + /** + * Add an upperbound to a policy such that once the cumulative delay over all retries has + * reached or exceeded the given limit, the policy will stop retrying and give up. + */ + def limitRetriesByCumulativeDelay[M[_]: Applicative]( + threshold: FiniteDuration, + policy: RetryPolicy[M] + ): RetryPolicy[M] = { + def decideNextRetry(status: RetryStatus): M[PolicyDecision] = + policy.decideNextRetry(status).map { + case r @ DelayAndRetry(delay) => + if (status.cumulativeDelay + delay >= threshold) GiveUp else r + case GiveUp => GiveUp + } + + RetryPolicy.withShow[M]( + decideNextRetry, + show"limitRetriesByCumulativeDelay(threshold=$threshold, $policy)" + ) + } +} + +trait Sleep[M[_]] { + def sleep(delay: FiniteDuration): M[Unit] +} + +object Sleep { + def apply[M[_]](implicit sleep: Sleep[M]): Sleep[M] = sleep + + implicit def sleepUsingTemporal[F[_]](implicit t: Temporal[F]): Sleep[F] = + (delay: FiniteDuration) => t.sleep(delay) +} + +object implicits extends syntax.AllSyntax + +sealed trait RetryDetails { + def retriesSoFar: Int + def cumulativeDelay: FiniteDuration + def givingUp: Boolean + def upcomingDelay: Option[FiniteDuration] +} + +object RetryDetails { + final case class GivingUp( + totalRetries: Int, + totalDelay: FiniteDuration + ) extends RetryDetails { + val retriesSoFar: Int = totalRetries + val cumulativeDelay: FiniteDuration = totalDelay + val givingUp: Boolean = true + val upcomingDelay: Option[FiniteDuration] = None + } + + final case class WillDelayAndRetry( + nextDelay: FiniteDuration, + retriesSoFar: Int, + cumulativeDelay: FiniteDuration + ) extends RetryDetails { + val givingUp: Boolean = false + val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) + } +} + +package object retry_ { + @deprecated("Use retryingOnFailures instead", "2.1.0") + def retryingM[A] = new RetryingOnFailuresPartiallyApplied[A] + def retryingOnFailures[A] = new RetryingOnFailuresPartiallyApplied[A] + + private def retryingOnFailuresImpl[M[_], A]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit], + status: RetryStatus, + a: A + )(implicit M: Monad[M], S: Sleep[M]): M[Either[RetryStatus, A]] = { + + def onFalse: M[Either[RetryStatus, A]] = for { + nextStep <- applyPolicy(policy, status) + _ <- onFailure(a, buildRetryDetails(status, nextStep)) + result <- nextStep match { + case NextStep.RetryAfterDelay(delay, updatedStatus) => + S.sleep(delay) *> + M.pure(Left(updatedStatus)) // continue recursion + case NextStep.GiveUp => + M.pure(Right(a)) // stop the recursion + } + } yield result + + wasSuccessful(a).ifM( + M.pure(Right(a)), // stop the recursion + onFalse + ) + } + + private[retry] class RetryingOnFailuresPartiallyApplied[A] { + def apply[M[_]]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit M: Monad[M], S: Sleep[M]): M[A] = M.tailRecM(RetryStatus.NoRetriesYet) { + status => + action.flatMap { a => + retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + } + } + } + + def retryingOnSomeErrors[A] = new RetryingOnSomeErrorsPartiallyApplied[A] + + private def retryingOnSomeErrorsImpl[M[_], A, E]( + policy: RetryPolicy[M], + isWorthRetrying: E => M[Boolean], + onError: (E, RetryDetails) => M[Unit], + status: RetryStatus, + attempt: Either[E, A] + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[Either[RetryStatus, A]] = attempt match { + case Left(error) => + isWorthRetrying(error).ifM( + for { + nextStep <- applyPolicy(policy, status) + _ <- onError(error, buildRetryDetails(status, nextStep)) + result <- nextStep match { + case NextStep.RetryAfterDelay(delay, updatedStatus) => + S.sleep(delay) *> + ME.pure(Left(updatedStatus)) // continue recursion + case NextStep.GiveUp => + ME.raiseError[A](error).map(Right(_)) // stop the recursion + } + } yield result, + ME.raiseError[A](error).map(Right(_)) // stop the recursion + ) + case Right(success) => + ME.pure(Right(success)) // stop the recursion + } + + private[retry] class RetryingOnSomeErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + isWorthRetrying: E => M[Boolean], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = + ME.tailRecM(RetryStatus.NoRetriesYet) { status => + ME.attempt(action).flatMap { attempt => + retryingOnSomeErrorsImpl( + policy, + isWorthRetrying, + onError, + status, + attempt + ) + } + } + } + + def retryingOnAllErrors[A] = new RetryingOnAllErrorsPartiallyApplied[A] + + private[retry] class RetryingOnAllErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = + retryingOnSomeErrors[A].apply[M, E](policy, _ => ME.pure(true), onError)( + action + ) + } + + def retryingOnFailuresAndSomeErrors[A] = + new RetryingOnFailuresAndSomeErrorsPartiallyApplied[A] + + private[retry] class RetryingOnFailuresAndSomeErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + isWorthRetrying: E => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = { + + ME.tailRecM(RetryStatus.NoRetriesYet) { status => + ME.attempt(action).flatMap { + case Right(a) => + retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + case attempt => + retryingOnSomeErrorsImpl( + policy, + isWorthRetrying, + onError, + status, + attempt + ) + } + } + } + } + + def retryingOnFailuresAndAllErrors[A] = + new RetryingOnFailuresAndAllErrorsPartiallyApplied[A] + + private[retry] class RetryingOnFailuresAndAllErrorsPartiallyApplied[A] { + def apply[M[_], E]( + policy: RetryPolicy[M], + wasSuccessful: A => M[Boolean], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )( + action: => M[A] + )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = + retryingOnFailuresAndSomeErrors[A].apply[M, E]( + policy, + wasSuccessful, + _ => ME.pure(true), + onFailure, + onError + )( + action + ) + } + + def noop[M[_]: Monad, A]: (A, RetryDetails) => M[Unit] = + (_, _) => Monad[M].pure(()) + + private[retry] def applyPolicy[M[_]: Monad]( + policy: RetryPolicy[M], + retryStatus: RetryStatus + ): M[NextStep] = + policy.decideNextRetry(retryStatus).map { + case PolicyDecision.DelayAndRetry(delay) => + NextStep.RetryAfterDelay(delay, retryStatus.addRetry(delay)) + case PolicyDecision.GiveUp => + NextStep.GiveUp + } + + private[retry] def buildRetryDetails( + currentStatus: RetryStatus, + nextStep: NextStep + ): RetryDetails = + nextStep match { + case NextStep.RetryAfterDelay(delay, _) => + RetryDetails.WillDelayAndRetry( + delay, + currentStatus.retriesSoFar, + currentStatus.cumulativeDelay + ) + case NextStep.GiveUp => + RetryDetails.GivingUp( + currentStatus.retriesSoFar, + currentStatus.cumulativeDelay + ) + } + + private[retry] sealed trait NextStep + + private[retry] object NextStep { + case object GiveUp extends NextStep + + final case class RetryAfterDelay( + delay: FiniteDuration, + updatedStatus: RetryStatus + ) extends NextStep + } +} + +trait AllSyntax extends RetrySyntax + +trait RetrySyntax { + implicit final def retrySyntaxBase[M[_], A]( + action: => M[A] + ): RetryingOps[M, A] = + new RetryingOps[M, A](action) + + implicit final def retrySyntaxError[M[_], A, E]( + action: => M[A] + )(implicit M: MonadError[M, E]): RetryingErrorOps[M, A, E] = + new RetryingErrorOps[M, A, E](action) +} + +final class RetryingOps[M[_], A](action: => M[A]) { + @deprecated("Use retryingOnFailures instead", "2.1.0") + def retryingM[E]( + wasSuccessful: A => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit] + )(implicit M: Monad[M], S: Sleep[M]): M[A] = + retryingOnFailures(wasSuccessful, policy, onFailure) + + def retryingOnFailures[E]( + wasSuccessful: A => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit] + )(implicit M: Monad[M], S: Sleep[M]): M[A] = + retry_.retryingOnFailures( + policy = policy, + wasSuccessful = wasSuccessful, + onFailure = onFailure + )(action) +} + +final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit M: MonadError[M, E]) { + def retryingOnAllErrors( + policy: RetryPolicy[M], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnAllErrors( + policy = policy, + onError = onError + )(action) + + def retryingOnSomeErrors( + isWorthRetrying: E => M[Boolean], + policy: RetryPolicy[M], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnSomeErrors( + policy = policy, + isWorthRetrying = isWorthRetrying, + onError = onError + )(action) + + def retryingOnFailuresAndAllErrors( + wasSuccessful: A => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnFailuresAndAllErrors( + policy = policy, + wasSuccessful = wasSuccessful, + onFailure = onFailure, + onError = onError + )(action) + + def retryingOnFailuresAndSomeErrors( + wasSuccessful: A => M[Boolean], + isWorthRetrying: E => M[Boolean], + policy: RetryPolicy[M], + onFailure: (A, RetryDetails) => M[Unit], + onError: (E, RetryDetails) => M[Unit] + )(implicit S: Sleep[M]): M[A] = + retry_.retryingOnFailuresAndSomeErrors( + policy = policy, + wasSuccessful = wasSuccessful, + isWorthRetrying = isWorthRetrying, + onFailure = onFailure, + onError = onError + )(action) +} + +object Fibonacci { + def fibonacci(n: Int): Long = { + if (n > 0) + fib(n)._1 + else + 0 + } + + // "Fast doubling" Fibonacci algorithm. + // See e.g. http://funloop.org/post/2017-04-14-computing-fibonacci-numbers.html for explanation. + private def fib(n: Int): (Long, Long) = n match { + case 0 => (0, 1) + case m => + val (a, b) = fib(m / 2) + val c = a * (b * 2 - a) + val d = a * a + b * b + if (n % 2 == 0) + (c, d) + else + (d, c + d) + } +} + + diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index b9dda48b17..7e79626e20 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -49,7 +49,7 @@ object Retry { ) } - val NoRetriesYet = Retry.Status(0, Duration.Zero, None) + val noRetriesYet = Retry.Status(0, Duration.Zero, None) def apply[F[_]: Monad]( nextRetry: Retry.Status => F[PolicyDecision], @@ -66,259 +66,18 @@ object Retry { pretty ) -// implicit def boundedSemilatticeForRetry[F[_]]( -// implicit F: Applicative[F]): BoundedSemilattice[Retry[F]] = -// new BoundedSemilattice[Retry[F]] { -// override def empty: Retry[F] = -// RetryPolicies.constantDelay[F](Duration.Zero) - -// override def combine( -// x: Retry[F], -// y: Retry[F] -// ): Retry[F] = x.join(y) -// } - -// implicit def showForRetry[F[_]]: Show[Retry[F]] = -// Show.show(_.show) - - private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[PolicyDecision], pretty: String) extends Retry[F] { - - override def nextRetry(status: Retry.Status): F[PolicyDecision] = - nextRetry_(status) - - override def toString: String = pretty - - def followedBy(r: Retry[F]): Retry[F] = - Retry( - status => (nextRetry(status), r.nextRetry(status)).mapN { - case (GiveUp, pd) => pd - case (pd, _) => pd - }, - s"$this.followedBy($r)" - ) - - def join(r: Retry[F]): Retry[F] = - Retry[F]( - status => (nextRetry(status), r.nextRetry(status)).mapN { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp - }, - s"$this.join($r)" - ) - - def meet(r: Retry[F]): Retry[F] = - Retry[F]( - status => (nextRetry(status), r.nextRetry(status)).mapN { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) - case (s @ DelayAndRetry(_), GiveUp) => s - case (GiveUp, s @ DelayAndRetry(_)) => s - case _ => GiveUp - }, - s"$this.meet($r)" - ) - - def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = - Retry( - status => nextRetry(status).map { - case GiveUp => GiveUp - case DelayAndRetry(d) => DelayAndRetry(f(d)) - }, - s"$this.mapDelay()" - ) - - def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = - Retry( - status => nextRetry(status).flatMap { - case GiveUp => GiveUp.pure[F].widen[PolicyDecision] - case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) - }, - s"$this.flatMapDelay()" - ) - - def mapK[G[_]: Monad](f: F ~> G): Retry[G] = - Retry( - status => f(nextRetry(status)), - s"$this.mapK()" - ) - } -} - -/////// - -final case class RetryStatus( - retriesSoFar: Int, - cumulativeDelay: FiniteDuration, - previousDelay: Option[FiniteDuration] -) { - def addRetry(delay: FiniteDuration): RetryStatus = RetryStatus( - retriesSoFar = this.retriesSoFar + 1, - cumulativeDelay = this.cumulativeDelay + delay, - previousDelay = Some(delay) - ) -} - -object RetryStatus { - val NoRetriesYet = RetryStatus(0, Duration.Zero, None) -} - -sealed trait PolicyDecision - -object PolicyDecision { - case object GiveUp extends PolicyDecision - - final case class DelayAndRetry( - delay: FiniteDuration - ) extends PolicyDecision -} - -case class RetryPolicy[M[_]]( - decideNextRetry: RetryStatus => M[PolicyDecision] -) { - def show: String = toString - - def followedBy(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (GiveUp, pd) => pd - case (pd, _) => pd - }, - show"$show.followedBy($rp)" - ) - - /** - * Combine this schedule with another schedule, giving up when either of the schedules want to - * give up and choosing the maximum of the two delays when both of the schedules want to delay - * the next retry. The dual of the `meet` operation. - */ - def join(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow[M]( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp - }, - show"$show.join($rp)" - ) - - /** - * Combine this schedule with another schedule, giving up when both of the schedules want to - * give up and choosing the minimum of the two delays when both of the schedules want to delay - * the next retry. The dual of the `join` operation. - */ - def meet(rp: RetryPolicy[M])(implicit M: Apply[M]): RetryPolicy[M] = - RetryPolicy.withShow[M]( - status => - M.map2(decideNextRetry(status), rp.decideNextRetry(status)) { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) - case (s @ DelayAndRetry(_), GiveUp) => s - case (GiveUp, s @ DelayAndRetry(_)) => s - case _ => GiveUp - }, - show"$show.meet($rp)" - ) - - def mapDelay( - f: FiniteDuration => FiniteDuration - )(implicit M: Functor[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.map(decideNextRetry(status)) { - case GiveUp => GiveUp - case DelayAndRetry(d) => DelayAndRetry(f(d)) - }, - show"$show.mapDelay()" - ) - - def flatMapDelay( - f: FiniteDuration => M[FiniteDuration] - )(implicit M: Monad[M]): RetryPolicy[M] = - RetryPolicy.withShow( - status => - M.flatMap(decideNextRetry(status)) { - case GiveUp => M.pure(GiveUp) - case DelayAndRetry(d) => M.map(f(d))(DelayAndRetry(_)) - }, - show"$show.flatMapDelay()" - ) - - def mapK[N[_]](nt: FunctionK[M, N]): RetryPolicy[N] = - RetryPolicy.withShow( - status => nt(decideNextRetry(status)), - show"$show.mapK()" - ) -} - -object RetryPolicy { - def lift[M[_]]( - f: RetryStatus => PolicyDecision - )(implicit M: Applicative[M]): RetryPolicy[M] = - RetryPolicy[M](decideNextRetry = retryStatus => M.pure(f(retryStatus))) - - def withShow[M[_]]( - decideNextRetry: RetryStatus => M[PolicyDecision], - pretty: => String - ): RetryPolicy[M] = - new RetryPolicy[M](decideNextRetry) { - override def show: String = pretty - override def toString: String = pretty - } - - def liftWithShow[M[_]: Applicative]( - decideNextRetry: RetryStatus => PolicyDecision, - pretty: => String - ): RetryPolicy[M] = - withShow(rs => Applicative[M].pure(decideNextRetry(rs)), pretty) - - implicit def boundedSemilatticeForRetryPolicy[M[_]]( - implicit M: Applicative[M]): BoundedSemilattice[RetryPolicy[M]] = - new BoundedSemilattice[RetryPolicy[M]] { - override def empty: RetryPolicy[M] = - RetryPolicies.constantDelay[M](Duration.Zero) - - override def combine( - x: RetryPolicy[M], - y: RetryPolicy[M] - ): RetryPolicy[M] = x.join(y) - } - - implicit def showForRetryPolicy[M[_]]: Show[RetryPolicy[M]] = - Show.show(_.show) -} - -object RetryPolicies { - private val LongMax: BigInt = BigInt(Long.MaxValue) - - /* - * Multiply the given duration by the given multiplier, but cap the result to - * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. - * - * Note: despite the "safe" in the name, we can still create an invalid - * FiniteDuration if the multiplier is negative. But an assumption of the library - * as a whole is that nobody would be silly enough to use negative delays. - */ - private def safeMultiply( - duration: FiniteDuration, - multiplier: Long - ): FiniteDuration = { - val durationNanos = BigInt(duration.toNanos) - val resultNanos = durationNanos * BigInt(multiplier) - val safeResultNanos = resultNanos min LongMax - FiniteDuration(safeResultNanos.toLong, TimeUnit.NANOSECONDS) - } - /** * Don't retry at all and always give up. Only really useful for combining with other * policies. */ - def alwaysGiveUp[M[_]: Applicative]: RetryPolicy[M] = - RetryPolicy.liftWithShow(Function.const(GiveUp), "alwaysGiveUp") + def alwaysGiveUp[F[_]: Monad]: Retry[F] = + Retry.lift[F](Function.const(GiveUp), "alwaysGiveUp") /** * Delay by a constant amount before each retry. Never give up. */ - def constantDelay[M[_]: Applicative](delay: FiniteDuration): RetryPolicy[M] = - RetryPolicy.liftWithShow( + def constantDelay[F[_]: Monad](delay: FiniteDuration): Retry[F] = + Retry.lift[F]( Function.const(DelayAndRetry(delay)), show"constantDelay($delay)" ) @@ -326,10 +85,10 @@ object RetryPolicies { /** * Each delay is twice as long as the previous one. Never give up. */ - def exponentialBackoff[M[_]: Applicative]( + def exponentialBackoff[F[_]: Monad]( baseDelay: FiniteDuration - ): RetryPolicy[M] = - RetryPolicy.liftWithShow( + ): Retry[F] = + Retry.lift[F]( { status => val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) @@ -341,8 +100,8 @@ object RetryPolicies { /** * Retry without delay, giving up after the given number of retries. */ - def limitRetries[M[_]: Applicative](maxRetries: Int): RetryPolicy[M] = - RetryPolicy.liftWithShow( + def limitRetries[F[_]: Monad](maxRetries: Int): Retry[F] = + Retry.lift[F]( { status => if (status.retriesSoFar >= maxRetries) { GiveUp @@ -359,10 +118,10 @@ object RetryPolicies { * e.g. if `baseDelay` is 10 milliseconds, the delays before each retry will be 10 ms, 10 ms, * 20 ms, 30ms, 50ms, 80ms, 130ms, ... */ - def fibonacciBackoff[M[_]: Applicative]( + def fibonacciBackoff[F[_]: Monad]( baseDelay: FiniteDuration - ): RetryPolicy[M] = - RetryPolicy.liftWithShow( + ): Retry[F] = + Retry.lift[F]( { status => val delay = safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) @@ -375,8 +134,8 @@ object RetryPolicies { * "Full jitter" backoff algorithm. See * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ */ - def fullJitter[M[_]: Applicative](baseDelay: FiniteDuration): RetryPolicy[M] = - RetryPolicy.liftWithShow( + def fullJitter[F[_]: Monad](baseDelay: FiniteDuration): Retry[F] = + Retry.lift[F]( { status => val e = Math.pow(2, status.retriesSoFar).toLong val maxDelay = safeMultiply(baseDelay, e) @@ -389,10 +148,10 @@ object RetryPolicies { /** * Set an upper bound on any individual delay produced by the given policy. */ - def capDelay[M[_]: Applicative]( + def capDelay[F[_]: Monad]( cap: FiniteDuration, - policy: RetryPolicy[M] - ): RetryPolicy[M] = + policy: Retry[F] + ): Retry[F] = policy.meet(constantDelay(cap)) /** @@ -401,20 +160,20 @@ object RetryPolicies { * stop retrying once cumulative delay reaches a time-delay amount, use * [[limitRetriesByCumulativeDelay]]. */ - def limitRetriesByDelay[M[_]: Applicative]( + def limitRetriesByDelay[F[_]: Monad]( threshold: FiniteDuration, - policy: RetryPolicy[M] - ): RetryPolicy[M] = { - def decideNextRetry(status: RetryStatus): M[PolicyDecision] = - policy.decideNextRetry(status).map { + policy: Retry[F] + ): Retry[F] = { + def decideNextRetry(status: Retry.Status): F[PolicyDecision] = + policy.nextRetry(status).map { case r @ DelayAndRetry(delay) => if (delay > threshold) GiveUp else r case GiveUp => GiveUp } - RetryPolicy.withShow[M]( + Retry[F]( decideNextRetry, - show"limitRetriesByDelay(threshold=$threshold, $policy)" + s"limitRetriesByDelay(threshold=$threshold, $policy)" ) } @@ -422,376 +181,115 @@ object RetryPolicies { * Add an upperbound to a policy such that once the cumulative delay over all retries has * reached or exceeded the given limit, the policy will stop retrying and give up. */ - def limitRetriesByCumulativeDelay[M[_]: Applicative]( + def limitRetriesByCumulativeDelay[F[_]: Monad]( threshold: FiniteDuration, - policy: RetryPolicy[M] - ): RetryPolicy[M] = { - def decideNextRetry(status: RetryStatus): M[PolicyDecision] = - policy.decideNextRetry(status).map { + policy: Retry[F] + ): Retry[F] = { + def decideNextRetry(status: Retry.Status): F[PolicyDecision] = + policy.nextRetry(status).map { case r @ DelayAndRetry(delay) => if (status.cumulativeDelay + delay >= threshold) GiveUp else r case GiveUp => GiveUp } - RetryPolicy.withShow[M]( + Retry[F]( decideNextRetry, - show"limitRetriesByCumulativeDelay(threshold=$threshold, $policy)" + s"limitRetriesByCumulativeDelay(threshold=$threshold, $policy)" ) } -} - -trait Sleep[M[_]] { - def sleep(delay: FiniteDuration): M[Unit] -} - -object Sleep { - def apply[M[_]](implicit sleep: Sleep[M]): Sleep[M] = sleep - - implicit def sleepUsingTemporal[F[_]](implicit t: Temporal[F]): Sleep[F] = - (delay: FiniteDuration) => t.sleep(delay) -} +// implicit def boundedSemilatticeForRetry[F[_]]( +// implicit F: Monad[F]): BoundedSemilattice[Retry[F]] = +// new BoundedSemilattice[Retry[F]] { +// override def empty: Retry[F] = +// RetryPolicies.constantDelay[F](Duration.Zero) -object implicits extends syntax.AllSyntax +// override def combine( +// x: Retry[F], +// y: Retry[F] +// ): Retry[F] = x.join(y) +// } -sealed trait RetryDetails { - def retriesSoFar: Int - def cumulativeDelay: FiniteDuration - def givingUp: Boolean - def upcomingDelay: Option[FiniteDuration] -} +// implicit def showForRetry[F[_]]: Show[Retry[F]] = +// Show.show(_.show) -object RetryDetails { - final case class GivingUp( - totalRetries: Int, - totalDelay: FiniteDuration - ) extends RetryDetails { - val retriesSoFar: Int = totalRetries - val cumulativeDelay: FiniteDuration = totalDelay - val givingUp: Boolean = true - val upcomingDelay: Option[FiniteDuration] = None + /* + * Multiply the given duration by the given multiplier, but cap the result to + * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. + * + * Note: despite the "safe" in the name, we can still create an invalid + * FiniteDuration if the multiplier is negative. But an assumption of the library + * as a whole is that nobody would be silly enough to use negative delays. + */ + private def safeMultiply( + duration: FiniteDuration, + multiplier: Long + ): FiniteDuration = { + val longMax: BigInt = BigInt(Long.MaxValue) + val durationNanos = BigInt(duration.toNanos) + val resultNanos = durationNanos * BigInt(multiplier) + val safeResultNanos = resultNanos min longMax + FiniteDuration(safeResultNanos.toLong, TimeUnit.NANOSECONDS) } - final case class WillDelayAndRetry( - nextDelay: FiniteDuration, - retriesSoFar: Int, - cumulativeDelay: FiniteDuration - ) extends RetryDetails { - val givingUp: Boolean = false - val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) - } -} -package object retry_ { - @deprecated("Use retryingOnFailures instead", "2.1.0") - def retryingM[A] = new RetryingOnFailuresPartiallyApplied[A] - def retryingOnFailures[A] = new RetryingOnFailuresPartiallyApplied[A] - - private def retryingOnFailuresImpl[M[_], A]( - policy: RetryPolicy[M], - wasSuccessful: A => M[Boolean], - onFailure: (A, RetryDetails) => M[Unit], - status: RetryStatus, - a: A - )(implicit M: Monad[M], S: Sleep[M]): M[Either[RetryStatus, A]] = { - - def onFalse: M[Either[RetryStatus, A]] = for { - nextStep <- applyPolicy(policy, status) - _ <- onFailure(a, buildRetryDetails(status, nextStep)) - result <- nextStep match { - case NextStep.RetryAfterDelay(delay, updatedStatus) => - S.sleep(delay) *> - M.pure(Left(updatedStatus)) // continue recursion - case NextStep.GiveUp => - M.pure(Right(a)) // stop the recursion - } - } yield result + private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[PolicyDecision], pretty: String) extends Retry[F] { - wasSuccessful(a).ifM( - M.pure(Right(a)), // stop the recursion - onFalse - ) - } + override def nextRetry(status: Retry.Status): F[PolicyDecision] = + nextRetry_(status) - private[retry] class RetryingOnFailuresPartiallyApplied[A] { - def apply[M[_]]( - policy: RetryPolicy[M], - wasSuccessful: A => M[Boolean], - onFailure: (A, RetryDetails) => M[Unit] - )( - action: => M[A] - )(implicit M: Monad[M], S: Sleep[M]): M[A] = M.tailRecM(RetryStatus.NoRetriesYet) { - status => - action.flatMap { a => - retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) - } - } - } + override def toString: String = pretty - def retryingOnSomeErrors[A] = new RetryingOnSomeErrorsPartiallyApplied[A] - - private def retryingOnSomeErrorsImpl[M[_], A, E]( - policy: RetryPolicy[M], - isWorthRetrying: E => M[Boolean], - onError: (E, RetryDetails) => M[Unit], - status: RetryStatus, - attempt: Either[E, A] - )(implicit ME: MonadError[M, E], S: Sleep[M]): M[Either[RetryStatus, A]] = attempt match { - case Left(error) => - isWorthRetrying(error).ifM( - for { - nextStep <- applyPolicy(policy, status) - _ <- onError(error, buildRetryDetails(status, nextStep)) - result <- nextStep match { - case NextStep.RetryAfterDelay(delay, updatedStatus) => - S.sleep(delay) *> - ME.pure(Left(updatedStatus)) // continue recursion - case NextStep.GiveUp => - ME.raiseError[A](error).map(Right(_)) // stop the recursion - } - } yield result, - ME.raiseError[A](error).map(Right(_)) // stop the recursion + def followedBy(r: Retry[F]): Retry[F] = + Retry( + status => (nextRetry(status), r.nextRetry(status)).mapN { + case (GiveUp, pd) => pd + case (pd, _) => pd + }, + s"$this.followedBy($r)" ) - case Right(success) => - ME.pure(Right(success)) // stop the recursion - } - - private[retry] class RetryingOnSomeErrorsPartiallyApplied[A] { - def apply[M[_], E]( - policy: RetryPolicy[M], - isWorthRetrying: E => M[Boolean], - onError: (E, RetryDetails) => M[Unit] - )( - action: => M[A] - )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = - ME.tailRecM(RetryStatus.NoRetriesYet) { status => - ME.attempt(action).flatMap { attempt => - retryingOnSomeErrorsImpl( - policy, - isWorthRetrying, - onError, - status, - attempt - ) - } - } - } - def retryingOnAllErrors[A] = new RetryingOnAllErrorsPartiallyApplied[A] - - private[retry] class RetryingOnAllErrorsPartiallyApplied[A] { - def apply[M[_], E]( - policy: RetryPolicy[M], - onError: (E, RetryDetails) => M[Unit] - )( - action: => M[A] - )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = - retryingOnSomeErrors[A].apply[M, E](policy, _ => ME.pure(true), onError)( - action + def join(r: Retry[F]): Retry[F] = + Retry[F]( + status => (nextRetry(status), r.nextRetry(status)).mapN { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp + }, + s"$this.join($r)" ) - } - - def retryingOnFailuresAndSomeErrors[A] = - new RetryingOnFailuresAndSomeErrorsPartiallyApplied[A] - - private[retry] class RetryingOnFailuresAndSomeErrorsPartiallyApplied[A] { - def apply[M[_], E]( - policy: RetryPolicy[M], - wasSuccessful: A => M[Boolean], - isWorthRetrying: E => M[Boolean], - onFailure: (A, RetryDetails) => M[Unit], - onError: (E, RetryDetails) => M[Unit] - )( - action: => M[A] - )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = { - - ME.tailRecM(RetryStatus.NoRetriesYet) { status => - ME.attempt(action).flatMap { - case Right(a) => - retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) - case attempt => - retryingOnSomeErrorsImpl( - policy, - isWorthRetrying, - onError, - status, - attempt - ) - } - } - } - } - def retryingOnFailuresAndAllErrors[A] = - new RetryingOnFailuresAndAllErrorsPartiallyApplied[A] - - private[retry] class RetryingOnFailuresAndAllErrorsPartiallyApplied[A] { - def apply[M[_], E]( - policy: RetryPolicy[M], - wasSuccessful: A => M[Boolean], - onFailure: (A, RetryDetails) => M[Unit], - onError: (E, RetryDetails) => M[Unit] - )( - action: => M[A] - )(implicit ME: MonadError[M, E], S: Sleep[M]): M[A] = - retryingOnFailuresAndSomeErrors[A].apply[M, E]( - policy, - wasSuccessful, - _ => ME.pure(true), - onFailure, - onError - )( - action + def meet(r: Retry[F]): Retry[F] = + Retry[F]( + status => (nextRetry(status), r.nextRetry(status)).mapN { + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp + }, + s"$this.meet($r)" ) - } - def noop[M[_]: Monad, A]: (A, RetryDetails) => M[Unit] = - (_, _) => Monad[M].pure(()) - - private[retry] def applyPolicy[M[_]: Monad]( - policy: RetryPolicy[M], - retryStatus: RetryStatus - ): M[NextStep] = - policy.decideNextRetry(retryStatus).map { - case PolicyDecision.DelayAndRetry(delay) => - NextStep.RetryAfterDelay(delay, retryStatus.addRetry(delay)) - case PolicyDecision.GiveUp => - NextStep.GiveUp - } - - private[retry] def buildRetryDetails( - currentStatus: RetryStatus, - nextStep: NextStep - ): RetryDetails = - nextStep match { - case NextStep.RetryAfterDelay(delay, _) => - RetryDetails.WillDelayAndRetry( - delay, - currentStatus.retriesSoFar, - currentStatus.cumulativeDelay - ) - case NextStep.GiveUp => - RetryDetails.GivingUp( - currentStatus.retriesSoFar, - currentStatus.cumulativeDelay - ) - } - - private[retry] sealed trait NextStep - - private[retry] object NextStep { - case object GiveUp extends NextStep - - final case class RetryAfterDelay( - delay: FiniteDuration, - updatedStatus: RetryStatus - ) extends NextStep - } -} - -trait AllSyntax extends RetrySyntax - -trait RetrySyntax { - implicit final def retrySyntaxBase[M[_], A]( - action: => M[A] - ): RetryingOps[M, A] = - new RetryingOps[M, A](action) - - implicit final def retrySyntaxError[M[_], A, E]( - action: => M[A] - )(implicit M: MonadError[M, E]): RetryingErrorOps[M, A, E] = - new RetryingErrorOps[M, A, E](action) -} - -final class RetryingOps[M[_], A](action: => M[A]) { - @deprecated("Use retryingOnFailures instead", "2.1.0") - def retryingM[E]( - wasSuccessful: A => M[Boolean], - policy: RetryPolicy[M], - onFailure: (A, RetryDetails) => M[Unit] - )(implicit M: Monad[M], S: Sleep[M]): M[A] = - retryingOnFailures(wasSuccessful, policy, onFailure) - - def retryingOnFailures[E]( - wasSuccessful: A => M[Boolean], - policy: RetryPolicy[M], - onFailure: (A, RetryDetails) => M[Unit] - )(implicit M: Monad[M], S: Sleep[M]): M[A] = - retry_.retryingOnFailures( - policy = policy, - wasSuccessful = wasSuccessful, - onFailure = onFailure - )(action) -} - -final class RetryingErrorOps[M[_], A, E](action: => M[A])(implicit M: MonadError[M, E]) { - def retryingOnAllErrors( - policy: RetryPolicy[M], - onError: (E, RetryDetails) => M[Unit] - )(implicit S: Sleep[M]): M[A] = - retry_.retryingOnAllErrors( - policy = policy, - onError = onError - )(action) - - def retryingOnSomeErrors( - isWorthRetrying: E => M[Boolean], - policy: RetryPolicy[M], - onError: (E, RetryDetails) => M[Unit] - )(implicit S: Sleep[M]): M[A] = - retry_.retryingOnSomeErrors( - policy = policy, - isWorthRetrying = isWorthRetrying, - onError = onError - )(action) - - def retryingOnFailuresAndAllErrors( - wasSuccessful: A => M[Boolean], - policy: RetryPolicy[M], - onFailure: (A, RetryDetails) => M[Unit], - onError: (E, RetryDetails) => M[Unit] - )(implicit S: Sleep[M]): M[A] = - retry_.retryingOnFailuresAndAllErrors( - policy = policy, - wasSuccessful = wasSuccessful, - onFailure = onFailure, - onError = onError - )(action) - - def retryingOnFailuresAndSomeErrors( - wasSuccessful: A => M[Boolean], - isWorthRetrying: E => M[Boolean], - policy: RetryPolicy[M], - onFailure: (A, RetryDetails) => M[Unit], - onError: (E, RetryDetails) => M[Unit] - )(implicit S: Sleep[M]): M[A] = - retry_.retryingOnFailuresAndSomeErrors( - policy = policy, - wasSuccessful = wasSuccessful, - isWorthRetrying = isWorthRetrying, - onFailure = onFailure, - onError = onError - )(action) -} + def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = + Retry( + status => nextRetry(status).map { + case GiveUp => GiveUp + case DelayAndRetry(d) => DelayAndRetry(f(d)) + }, + s"$this.mapDelay()" + ) -object Fibonacci { - def fibonacci(n: Int): Long = { - if (n > 0) - fib(n)._1 - else - 0 - } + def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = + Retry( + status => nextRetry(status).flatMap { + case GiveUp => GiveUp.pure[F].widen[PolicyDecision] + case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) + }, + s"$this.flatMapDelay()" + ) - // "Fast doubling" Fibonacci algorithm. - // See e.g. http://funloop.org/post/2017-04-14-computing-fibonacci-numbers.html for explanation. - private def fib(n: Int): (Long, Long) = n match { - case 0 => (0, 1) - case m => - val (a, b) = fib(m / 2) - val c = a * (b * 2 - a) - val d = a * a + b * b - if (n % 2 == 0) - (c, d) - else - (d, c + d) + def mapK[G[_]: Monad](f: F ~> G): Retry[G] = + Retry( + status => f(nextRetry(status)), + s"$this.mapK()" + ) } } From 80efc5ed545f98d0824f5b7b219296994bfdf31b Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 28 Aug 2022 11:16:40 +0200 Subject: [PATCH 12/45] Add limit combinators as instance methods --- .../main/scala/cats/effect/std/Retry.scala | 107 +++++++++--------- 1 file changed, 53 insertions(+), 54 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 7e79626e20..0dc8f1d0f4 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -34,6 +34,25 @@ abstract class Retry[F[_]] { def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] + /** + * Set an upper bound on any individual delay produced by the given policy. + */ + def capDelay(cap: FiniteDuration): Retry[F] + + /** + * Add an upper bound to a policy such that once the given time-delay amount per try + * has been reached or exceeded, the policy will stop retrying and give up. If you need to + * stop retrying once cumulative delay reaches a time-delay amount, use + * [[limitRetriesByCumulativeDelay]]. + */ + def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] + + /** + * Add an upperbound to a policy such that once the cumulative delay over all retries has + * reached or exceeded the given limit, the policy will stop retrying and give up. + */ + def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] + def mapK[G[_]: Monad](f: F ~> G): Retry[G] } object Retry { @@ -145,58 +164,6 @@ object Retry { show"fullJitter(baseDelay=$baseDelay)" ) - /** - * Set an upper bound on any individual delay produced by the given policy. - */ - def capDelay[F[_]: Monad]( - cap: FiniteDuration, - policy: Retry[F] - ): Retry[F] = - policy.meet(constantDelay(cap)) - - /** - * Add an upper bound to a policy such that once the given time-delay amount per try - * has been reached or exceeded, the policy will stop retrying and give up. If you need to - * stop retrying once cumulative delay reaches a time-delay amount, use - * [[limitRetriesByCumulativeDelay]]. - */ - def limitRetriesByDelay[F[_]: Monad]( - threshold: FiniteDuration, - policy: Retry[F] - ): Retry[F] = { - def decideNextRetry(status: Retry.Status): F[PolicyDecision] = - policy.nextRetry(status).map { - case r @ DelayAndRetry(delay) => - if (delay > threshold) GiveUp else r - case GiveUp => GiveUp - } - - Retry[F]( - decideNextRetry, - s"limitRetriesByDelay(threshold=$threshold, $policy)" - ) - } - - /** - * Add an upperbound to a policy such that once the cumulative delay over all retries has - * reached or exceeded the given limit, the policy will stop retrying and give up. - */ - def limitRetriesByCumulativeDelay[F[_]: Monad]( - threshold: FiniteDuration, - policy: Retry[F] - ): Retry[F] = { - def decideNextRetry(status: Retry.Status): F[PolicyDecision] = - policy.nextRetry(status).map { - case r @ DelayAndRetry(delay) => - if (status.cumulativeDelay + delay >= threshold) GiveUp else r - case GiveUp => GiveUp - } - - Retry[F]( - decideNextRetry, - s"limitRetriesByCumulativeDelay(threshold=$threshold, $policy)" - ) - } // implicit def boundedSemilatticeForRetry[F[_]]( // implicit F: Monad[F]): BoundedSemilattice[Retry[F]] = // new BoundedSemilattice[Retry[F]] { @@ -286,10 +253,42 @@ object Retry { s"$this.flatMapDelay()" ) + def capDelay(cap: FiniteDuration): Retry[F] = + meet(constantDelay(cap)) + + def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] = { + def decideNextRetry(status: Retry.Status): F[PolicyDecision] = + nextRetry(status).map { + case r @ DelayAndRetry(delay) => + if (delay > threshold) GiveUp else r + case GiveUp => GiveUp + } + + Retry[F]( + decideNextRetry, + s"limitRetriesByDelay(threshold=$threshold, $this)" + ) + } + + def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] = { + def decideNextRetry(status: Retry.Status): F[PolicyDecision] = + nextRetry(status).map { + case r @ DelayAndRetry(delay) => + if (status.cumulativeDelay + delay >= threshold) GiveUp else r + case GiveUp => GiveUp + } + + Retry[F]( + decideNextRetry, + s"limitRetriesByCumulativeDelay(threshold=$threshold, $this)" + ) + } + def mapK[G[_]: Monad](f: F ~> G): Retry[G] = Retry( status => f(nextRetry(status)), s"$this.mapK()" ) - } -} + } + } + From 69c8cabac7141ffdacb37db5e5f5526d5cb7db89 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Tue, 30 Aug 2022 02:53:12 +0200 Subject: [PATCH 13/45] Add Retry.Decision --- .../main/scala/cats/effect/std/Retry.scala | 69 ++++++++++--------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 0dc8f1d0f4..86ef95899e 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -6,13 +6,12 @@ import cats.syntax.all._ import cats.arrow.FunctionK import cats.kernel.BoundedSemilattice import cats.effect.kernel.Temporal -import retry.PolicyDecision._ import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.Random import java.util.concurrent.TimeUnit abstract class Retry[F[_]] { - def nextRetry(status: Retry.Status): F[PolicyDecision] + def nextRetry(status: Retry.Status): F[Retry.Decision] def followedBy(r: Retry[F]): Retry[F] @@ -68,16 +67,22 @@ object Retry { ) } + sealed trait Decision + object Decision { + case object GiveUp extends Decision + final case class DelayAndRetry(delay: FiniteDuration) extends Decision + } + val noRetriesYet = Retry.Status(0, Duration.Zero, None) def apply[F[_]: Monad]( - nextRetry: Retry.Status => F[PolicyDecision], + nextRetry: Retry.Status => F[Retry.Decision], pretty: String = "" ): Retry[F] = new RetryImpl[F](nextRetry, pretty) def lift[F[_]: Monad]( - nextRetry: Retry.Status => PolicyDecision, + nextRetry: Retry.Status => Retry.Decision, pretty: String = "" ): Retry[F] = apply[F]( @@ -90,14 +95,14 @@ object Retry { * policies. */ def alwaysGiveUp[F[_]: Monad]: Retry[F] = - Retry.lift[F](Function.const(GiveUp), "alwaysGiveUp") + Retry.lift[F](Function.const(Decision.GiveUp), "alwaysDecision.GiveUp") /** * Delay by a constant amount before each retry. Never give up. */ def constantDelay[F[_]: Monad](delay: FiniteDuration): Retry[F] = Retry.lift[F]( - Function.const(DelayAndRetry(delay)), + Function.const(Decision.DelayAndRetry(delay)), show"constantDelay($delay)" ) @@ -111,7 +116,7 @@ object Retry { { status => val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) - DelayAndRetry(delay) + Decision.DelayAndRetry(delay) }, show"exponentialBackOff(baseDelay=$baseDelay)" ) @@ -123,9 +128,9 @@ object Retry { Retry.lift[F]( { status => if (status.retriesSoFar >= maxRetries) { - GiveUp + Decision.GiveUp } else { - DelayAndRetry(Duration.Zero) + Decision.DelayAndRetry(Duration.Zero) } }, show"limitRetries(maxRetries=$maxRetries)" @@ -144,7 +149,7 @@ object Retry { { status => val delay = safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) - DelayAndRetry(delay) + Decision.DelayAndRetry(delay) }, show"fibonacciBackoff(baseDelay=$baseDelay)" ) @@ -159,7 +164,7 @@ object Retry { val e = Math.pow(2, status.retriesSoFar).toLong val maxDelay = safeMultiply(baseDelay, e) val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong - DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) + Decision.DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) }, show"fullJitter(baseDelay=$baseDelay)" ) @@ -199,9 +204,9 @@ object Retry { } - private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[PolicyDecision], pretty: String) extends Retry[F] { + private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[Retry.Decision], pretty: String) extends Retry[F] { - override def nextRetry(status: Retry.Status): F[PolicyDecision] = + override def nextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry_(status) override def toString: String = pretty @@ -209,7 +214,7 @@ object Retry { def followedBy(r: Retry[F]): Retry[F] = Retry( status => (nextRetry(status), r.nextRetry(status)).mapN { - case (GiveUp, pd) => pd + case (Decision.GiveUp, pd) => pd case (pd, _) => pd }, s"$this.followedBy($r)" @@ -218,8 +223,8 @@ object Retry { def join(r: Retry[F]): Retry[F] = Retry[F]( status => (nextRetry(status), r.nextRetry(status)).mapN { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp + case (Decision.DelayAndRetry(a), Decision.DelayAndRetry(b)) => Decision.DelayAndRetry(a max b) + case _ => Decision.GiveUp }, s"$this.join($r)" ) @@ -227,10 +232,10 @@ object Retry { def meet(r: Retry[F]): Retry[F] = Retry[F]( status => (nextRetry(status), r.nextRetry(status)).mapN { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) - case (s @ DelayAndRetry(_), GiveUp) => s - case (GiveUp, s @ DelayAndRetry(_)) => s - case _ => GiveUp + case (Decision.DelayAndRetry(a), Decision.DelayAndRetry(b)) => Decision.DelayAndRetry(a min b) + case (s @ Decision.DelayAndRetry(_), Decision.GiveUp) => s + case (Decision.GiveUp, s @ Decision.DelayAndRetry(_)) => s + case _ => Decision.GiveUp }, s"$this.meet($r)" ) @@ -238,8 +243,8 @@ object Retry { def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = Retry( status => nextRetry(status).map { - case GiveUp => GiveUp - case DelayAndRetry(d) => DelayAndRetry(f(d)) + case Decision.GiveUp => Decision.GiveUp + case Decision.DelayAndRetry(d) => Decision.DelayAndRetry(f(d)) }, s"$this.mapDelay()" ) @@ -247,8 +252,8 @@ object Retry { def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = Retry( status => nextRetry(status).flatMap { - case GiveUp => GiveUp.pure[F].widen[PolicyDecision] - case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) + case Decision.GiveUp => Decision.GiveUp.pure[F].widen[Retry.Decision] + case Decision.DelayAndRetry(d) => f(d).map(Decision.DelayAndRetry(_)) }, s"$this.flatMapDelay()" ) @@ -257,11 +262,11 @@ object Retry { meet(constantDelay(cap)) def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] = { - def decideNextRetry(status: Retry.Status): F[PolicyDecision] = + def decideNextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry(status).map { - case r @ DelayAndRetry(delay) => - if (delay > threshold) GiveUp else r - case GiveUp => GiveUp + case r @ Decision.DelayAndRetry(delay) => + if (delay > threshold) Decision.GiveUp else r + case Decision.GiveUp => Decision.GiveUp } Retry[F]( @@ -271,11 +276,11 @@ object Retry { } def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] = { - def decideNextRetry(status: Retry.Status): F[PolicyDecision] = + def decideNextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry(status).map { - case r @ DelayAndRetry(delay) => - if (status.cumulativeDelay + delay >= threshold) GiveUp else r - case GiveUp => GiveUp + case r @ Decision.DelayAndRetry(delay) => + if (status.cumulativeDelay + delay >= threshold) Decision.GiveUp else r + case Decision.GiveUp => Decision.GiveUp } Retry[F]( From 2a2ad213d7e3680226cb7f6bb32f3a38962f4657 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Tue, 30 Aug 2022 02:54:31 +0200 Subject: [PATCH 14/45] Use Retry.Decision cases unqualified --- .../main/scala/cats/effect/std/Retry.scala | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 86ef95899e..2a6d678b68 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -73,6 +73,8 @@ object Retry { final case class DelayAndRetry(delay: FiniteDuration) extends Decision } + import Decision._ + val noRetriesYet = Retry.Status(0, Duration.Zero, None) def apply[F[_]: Monad]( @@ -95,14 +97,14 @@ object Retry { * policies. */ def alwaysGiveUp[F[_]: Monad]: Retry[F] = - Retry.lift[F](Function.const(Decision.GiveUp), "alwaysDecision.GiveUp") + Retry.lift[F](Function.const(GiveUp), "alwaysGiveUp") /** * Delay by a constant amount before each retry. Never give up. */ def constantDelay[F[_]: Monad](delay: FiniteDuration): Retry[F] = Retry.lift[F]( - Function.const(Decision.DelayAndRetry(delay)), + Function.const(DelayAndRetry(delay)), show"constantDelay($delay)" ) @@ -116,7 +118,7 @@ object Retry { { status => val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) - Decision.DelayAndRetry(delay) + DelayAndRetry(delay) }, show"exponentialBackOff(baseDelay=$baseDelay)" ) @@ -128,9 +130,9 @@ object Retry { Retry.lift[F]( { status => if (status.retriesSoFar >= maxRetries) { - Decision.GiveUp + GiveUp } else { - Decision.DelayAndRetry(Duration.Zero) + DelayAndRetry(Duration.Zero) } }, show"limitRetries(maxRetries=$maxRetries)" @@ -149,7 +151,7 @@ object Retry { { status => val delay = safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) - Decision.DelayAndRetry(delay) + DelayAndRetry(delay) }, show"fibonacciBackoff(baseDelay=$baseDelay)" ) @@ -164,7 +166,7 @@ object Retry { val e = Math.pow(2, status.retriesSoFar).toLong val maxDelay = safeMultiply(baseDelay, e) val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong - Decision.DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) + DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) }, show"fullJitter(baseDelay=$baseDelay)" ) @@ -214,7 +216,7 @@ object Retry { def followedBy(r: Retry[F]): Retry[F] = Retry( status => (nextRetry(status), r.nextRetry(status)).mapN { - case (Decision.GiveUp, pd) => pd + case (GiveUp, pd) => pd case (pd, _) => pd }, s"$this.followedBy($r)" @@ -223,8 +225,8 @@ object Retry { def join(r: Retry[F]): Retry[F] = Retry[F]( status => (nextRetry(status), r.nextRetry(status)).mapN { - case (Decision.DelayAndRetry(a), Decision.DelayAndRetry(b)) => Decision.DelayAndRetry(a max b) - case _ => Decision.GiveUp + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) + case _ => GiveUp }, s"$this.join($r)" ) @@ -232,10 +234,10 @@ object Retry { def meet(r: Retry[F]): Retry[F] = Retry[F]( status => (nextRetry(status), r.nextRetry(status)).mapN { - case (Decision.DelayAndRetry(a), Decision.DelayAndRetry(b)) => Decision.DelayAndRetry(a min b) - case (s @ Decision.DelayAndRetry(_), Decision.GiveUp) => s - case (Decision.GiveUp, s @ Decision.DelayAndRetry(_)) => s - case _ => Decision.GiveUp + case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) + case (s @ DelayAndRetry(_), GiveUp) => s + case (GiveUp, s @ DelayAndRetry(_)) => s + case _ => GiveUp }, s"$this.meet($r)" ) @@ -243,8 +245,8 @@ object Retry { def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = Retry( status => nextRetry(status).map { - case Decision.GiveUp => Decision.GiveUp - case Decision.DelayAndRetry(d) => Decision.DelayAndRetry(f(d)) + case GiveUp => GiveUp + case DelayAndRetry(d) => DelayAndRetry(f(d)) }, s"$this.mapDelay()" ) @@ -252,8 +254,8 @@ object Retry { def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = Retry( status => nextRetry(status).flatMap { - case Decision.GiveUp => Decision.GiveUp.pure[F].widen[Retry.Decision] - case Decision.DelayAndRetry(d) => f(d).map(Decision.DelayAndRetry(_)) + case GiveUp => GiveUp.pure[F].widen[Retry.Decision] + case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) }, s"$this.flatMapDelay()" ) @@ -264,9 +266,9 @@ object Retry { def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] = { def decideNextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry(status).map { - case r @ Decision.DelayAndRetry(delay) => - if (delay > threshold) Decision.GiveUp else r - case Decision.GiveUp => Decision.GiveUp + case r @ DelayAndRetry(delay) => + if (delay > threshold) GiveUp else r + case GiveUp => GiveUp } Retry[F]( @@ -278,9 +280,9 @@ object Retry { def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] = { def decideNextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry(status).map { - case r @ Decision.DelayAndRetry(delay) => - if (status.cumulativeDelay + delay >= threshold) Decision.GiveUp else r - case Decision.GiveUp => Decision.GiveUp + case r @ DelayAndRetry(delay) => + if (status.cumulativeDelay + delay >= threshold) GiveUp else r + case GiveUp => GiveUp } Retry[F]( From 2402b03c72989074738eaaab98f875bebdf82f58 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 31 Aug 2022 15:17:47 +0200 Subject: [PATCH 15/45] Add generic retry combinator --- .../main/scala/cats/effect/std/Retry.scala | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 2a6d678b68..86fd051591 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -77,6 +77,152 @@ object Retry { val noRetriesYet = Retry.Status(0, Duration.Zero, None) + sealed trait RetryDetails { + def retriesSoFar: Int + def cumulativeDelay: FiniteDuration + def givingUp: Boolean + def upcomingDelay: Option[FiniteDuration] + } + + object RetryDetails { + final case class GivingUp( + totalRetries: Int, + totalDelay: FiniteDuration + ) extends RetryDetails { + val retriesSoFar: Int = totalRetries + val cumulativeDelay: FiniteDuration = totalDelay + val givingUp: Boolean = true + val upcomingDelay: Option[FiniteDuration] = None + } + + final case class WillDelayAndRetry( + nextDelay: FiniteDuration, + retriesSoFar: Int, + cumulativeDelay: FiniteDuration + ) extends RetryDetails { + val givingUp: Boolean = false + val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) + } + } + + def retry[F[_]: Temporal, A]( + policy: Retry[F], + action: F[A], + wasSuccessful: A => F[Boolean], + isWorthRetrying: Throwable => F[Boolean], + onFailure: (A, RetryDetails) => F[Unit], + onError: (Throwable, RetryDetails) => F[Unit] + ): F[A] = { + + def applyPolicy( + policy: Retry[F], + retryStatus: Retry.Status + ): F[NextStep] = + policy.nextRetry(retryStatus).map { + case Decision.DelayAndRetry(delay) => + NextStep.RetryAfterDelay(delay, retryStatus.addRetry(delay)) + case Decision.GiveUp => + NextStep.GiveUp + } + + def buildRetryDetails( + currentStatus: Retry.Status, + nextStep: NextStep + ): RetryDetails = + nextStep match { + case NextStep.RetryAfterDelay(delay, _) => + RetryDetails.WillDelayAndRetry( + delay, + currentStatus.retriesSoFar, + currentStatus.cumulativeDelay + ) + case NextStep.GiveUp => + RetryDetails.GivingUp( + currentStatus.retriesSoFar, + currentStatus.cumulativeDelay + ) + } + + sealed trait NextStep + + object NextStep { + case object GiveUp extends NextStep + + final case class RetryAfterDelay( + delay: FiniteDuration, + updatedStatus: Retry.Status + ) extends NextStep + } + + def retryingOnFailuresImpl( + policy: Retry[F], + wasSuccessful: A => F[Boolean], + onFailure: (A, RetryDetails) => F[Unit], + status: Retry.Status, + a: A + ): F[Either[Retry.Status, A]] = { + + def onFalse: F[Either[Retry.Status, A]] = for { + nextStep <- applyPolicy(policy, status) + _ <- onFailure(a, buildRetryDetails(status, nextStep)) + result <- nextStep match { + case NextStep.RetryAfterDelay(delay, updatedStatus) => + Temporal[F].sleep(delay) *> + updatedStatus.asLeft.pure[F] // continue recursion + case NextStep.GiveUp => + a.asRight.pure[F] // stop the recursion + } + } yield result + + wasSuccessful(a).ifM( + a.asRight.pure[F], + onFalse + ) + } + + def retryingOnSomeErrorsImpl[A]( + policy: Retry[F], + isWorthRetrying: Throwable => F[Boolean], + onError: (Throwable, RetryDetails) => F[Unit], + status: Retry.Status, + attempt: Either[Throwable, A] + ): F[Either[Retry.Status, A]] = attempt match { + case Left(error) => + isWorthRetrying(error).ifM( + for { + nextStep <- applyPolicy(policy, status) + _ <- onError(error, buildRetryDetails(status, nextStep)) + result <- nextStep match { + case NextStep.RetryAfterDelay(delay, updatedStatus) => + Temporal[F].sleep(delay) *> + updatedStatus.asLeft.pure[F] // continue recursion + case NextStep.GiveUp => + Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion + } + } yield result, + Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion + ) + case Right(success) => + success.asRight.pure[F] // stop the recursion + } + + + Temporal[F].tailRecM(Retry.noRetriesYet) { status => + action.attempt.flatMap { + case Right(a) => + retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + case attempt => + retryingOnSomeErrorsImpl( + policy, + isWorthRetrying, + onError, + status, + attempt + ) + } + } + } + def apply[F[_]: Monad]( nextRetry: Retry.Status => F[Retry.Decision], pretty: String = "" @@ -263,6 +409,7 @@ object Retry { def capDelay(cap: FiniteDuration): Retry[F] = meet(constantDelay(cap)) + // TODO inline these decideNextRetry definitions def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] = { def decideNextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry(status).map { From c61e5d87226f42bcb3a908761271ed1d0f741765 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 31 Aug 2022 15:41:08 +0200 Subject: [PATCH 16/45] Only retry on failures --- .../main/scala/cats/effect/std/Retry.scala | 30 +------------------ 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 86fd051591..9786c6e304 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -108,9 +108,7 @@ object Retry { def retry[F[_]: Temporal, A]( policy: Retry[F], action: F[A], - wasSuccessful: A => F[Boolean], isWorthRetrying: Throwable => F[Boolean], - onFailure: (A, RetryDetails) => F[Unit], onError: (Throwable, RetryDetails) => F[Unit] ): F[A] = { @@ -154,32 +152,6 @@ object Retry { ) extends NextStep } - def retryingOnFailuresImpl( - policy: Retry[F], - wasSuccessful: A => F[Boolean], - onFailure: (A, RetryDetails) => F[Unit], - status: Retry.Status, - a: A - ): F[Either[Retry.Status, A]] = { - - def onFalse: F[Either[Retry.Status, A]] = for { - nextStep <- applyPolicy(policy, status) - _ <- onFailure(a, buildRetryDetails(status, nextStep)) - result <- nextStep match { - case NextStep.RetryAfterDelay(delay, updatedStatus) => - Temporal[F].sleep(delay) *> - updatedStatus.asLeft.pure[F] // continue recursion - case NextStep.GiveUp => - a.asRight.pure[F] // stop the recursion - } - } yield result - - wasSuccessful(a).ifM( - a.asRight.pure[F], - onFalse - ) - } - def retryingOnSomeErrorsImpl[A]( policy: Retry[F], isWorthRetrying: Throwable => F[Boolean], @@ -210,7 +182,7 @@ object Retry { Temporal[F].tailRecM(Retry.noRetriesYet) { status => action.attempt.flatMap { case Right(a) => - retryingOnFailuresImpl(policy, wasSuccessful, onFailure, status, a) + a.asRight.pure[F] case attempt => retryingOnSomeErrorsImpl( policy, From a88821e17167757fcb8a2105c28f124bc71f4751 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 31 Aug 2022 15:59:43 +0200 Subject: [PATCH 17/45] Simplify NextStep type --- .../main/scala/cats/effect/std/Retry.scala | 50 ++++++++----------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 9786c6e304..2fae571e15 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -105,6 +105,7 @@ object Retry { } } + // TODO replace the name policy def retry[F[_]: Temporal, A]( policy: Retry[F], action: F[A], @@ -114,44 +115,33 @@ object Retry { def applyPolicy( policy: Retry[F], - retryStatus: Retry.Status + status: Retry.Status ): F[NextStep] = - policy.nextRetry(retryStatus).map { - case Decision.DelayAndRetry(delay) => - NextStep.RetryAfterDelay(delay, retryStatus.addRetry(delay)) - case Decision.GiveUp => - NextStep.GiveUp + policy.nextRetry(status).map { + case decision @ Decision.DelayAndRetry(delay) => + NextStep(status.addRetry(delay), decision) + case decision @ Decision.GiveUp => + NextStep(status, decision) } - def buildRetryDetails( - currentStatus: Retry.Status, - nextStep: NextStep - ): RetryDetails = - nextStep match { - case NextStep.RetryAfterDelay(delay, _) => + def buildRetryDetails(nextStep: NextStep): RetryDetails = { + nextStep.decision match { + case Decision.DelayAndRetry(delay) => RetryDetails.WillDelayAndRetry( delay, - currentStatus.retriesSoFar, - currentStatus.cumulativeDelay + nextStep.status.retriesSoFar, + nextStep.status.cumulativeDelay ) - case NextStep.GiveUp => + case Decision.GiveUp => RetryDetails.GivingUp( - currentStatus.retriesSoFar, - currentStatus.cumulativeDelay + nextStep.status.retriesSoFar, + nextStep.status.cumulativeDelay ) } - - sealed trait NextStep - - object NextStep { - case object GiveUp extends NextStep - - final case class RetryAfterDelay( - delay: FiniteDuration, - updatedStatus: Retry.Status - ) extends NextStep } + case class NextStep(status: Retry.Status, decision: Retry.Decision) + def retryingOnSomeErrorsImpl[A]( policy: Retry[F], isWorthRetrying: Throwable => F[Boolean], @@ -163,12 +153,12 @@ object Retry { isWorthRetrying(error).ifM( for { nextStep <- applyPolicy(policy, status) - _ <- onError(error, buildRetryDetails(status, nextStep)) + _ <- onError(error, buildRetryDetails(nextStep)) result <- nextStep match { - case NextStep.RetryAfterDelay(delay, updatedStatus) => + case NextStep(updatedStatus, Decision.DelayAndRetry(delay)) => Temporal[F].sleep(delay) *> updatedStatus.asLeft.pure[F] // continue recursion - case NextStep.GiveUp => + case NextStep(_, GiveUp) => Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion } } yield result, From a5a7e74edc320b229eb3fd3043ad5f221873b04e Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 31 Aug 2022 16:03:46 +0200 Subject: [PATCH 18/45] Remove RetryDetails --- .../main/scala/cats/effect/std/Retry.scala | 50 ++----------------- 1 file changed, 3 insertions(+), 47 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 2fae571e15..c03792f56e 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -77,40 +77,12 @@ object Retry { val noRetriesYet = Retry.Status(0, Duration.Zero, None) - sealed trait RetryDetails { - def retriesSoFar: Int - def cumulativeDelay: FiniteDuration - def givingUp: Boolean - def upcomingDelay: Option[FiniteDuration] - } - - object RetryDetails { - final case class GivingUp( - totalRetries: Int, - totalDelay: FiniteDuration - ) extends RetryDetails { - val retriesSoFar: Int = totalRetries - val cumulativeDelay: FiniteDuration = totalDelay - val givingUp: Boolean = true - val upcomingDelay: Option[FiniteDuration] = None - } - - final case class WillDelayAndRetry( - nextDelay: FiniteDuration, - retriesSoFar: Int, - cumulativeDelay: FiniteDuration - ) extends RetryDetails { - val givingUp: Boolean = false - val upcomingDelay: Option[FiniteDuration] = Some(nextDelay) - } - } - // TODO replace the name policy def retry[F[_]: Temporal, A]( policy: Retry[F], action: F[A], isWorthRetrying: Throwable => F[Boolean], - onError: (Throwable, RetryDetails) => F[Unit] + onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] ): F[A] = { def applyPolicy( @@ -124,28 +96,12 @@ object Retry { NextStep(status, decision) } - def buildRetryDetails(nextStep: NextStep): RetryDetails = { - nextStep.decision match { - case Decision.DelayAndRetry(delay) => - RetryDetails.WillDelayAndRetry( - delay, - nextStep.status.retriesSoFar, - nextStep.status.cumulativeDelay - ) - case Decision.GiveUp => - RetryDetails.GivingUp( - nextStep.status.retriesSoFar, - nextStep.status.cumulativeDelay - ) - } - } - case class NextStep(status: Retry.Status, decision: Retry.Decision) def retryingOnSomeErrorsImpl[A]( policy: Retry[F], isWorthRetrying: Throwable => F[Boolean], - onError: (Throwable, RetryDetails) => F[Unit], + onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit], status: Retry.Status, attempt: Either[Throwable, A] ): F[Either[Retry.Status, A]] = attempt match { @@ -153,7 +109,7 @@ object Retry { isWorthRetrying(error).ifM( for { nextStep <- applyPolicy(policy, status) - _ <- onError(error, buildRetryDetails(nextStep)) + _ <- onError(error, nextStep.status, nextStep.decision) result <- nextStep match { case NextStep(updatedStatus, Decision.DelayAndRetry(delay)) => Temporal[F].sleep(delay) *> From 94b3cf8556ce5cec10f254fb6fff0ce3d9238315 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Wed, 31 Aug 2022 16:16:04 +0200 Subject: [PATCH 19/45] Remove for comprehension --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index c03792f56e..d274bae023 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -107,17 +107,16 @@ object Retry { ): F[Either[Retry.Status, A]] = attempt match { case Left(error) => isWorthRetrying(error).ifM( - for { - nextStep <- applyPolicy(policy, status) - _ <- onError(error, nextStep.status, nextStep.decision) - result <- nextStep match { + applyPolicy(policy, status).flatMap { nextStep => + onError(error, nextStep.status, nextStep.decision) >> + (nextStep match { case NextStep(updatedStatus, Decision.DelayAndRetry(delay)) => Temporal[F].sleep(delay) *> updatedStatus.asLeft.pure[F] // continue recursion case NextStep(_, GiveUp) => Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion - } - } yield result, + }) + }, Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion ) case Right(success) => From 51f611512b1c549b4dab3c1b96d4047c639a2451 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 3 Sep 2022 03:51:40 +0200 Subject: [PATCH 20/45] Eliminate the NextStep helper class --- .../main/scala/cats/effect/std/Retry.scala | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index d274bae023..f7f9d424da 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -85,19 +85,6 @@ object Retry { onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] ): F[A] = { - def applyPolicy( - policy: Retry[F], - status: Retry.Status - ): F[NextStep] = - policy.nextRetry(status).map { - case decision @ Decision.DelayAndRetry(delay) => - NextStep(status.addRetry(delay), decision) - case decision @ Decision.GiveUp => - NextStep(status, decision) - } - - case class NextStep(status: Retry.Status, decision: Retry.Decision) - def retryingOnSomeErrorsImpl[A]( policy: Retry[F], isWorthRetrying: Throwable => F[Boolean], @@ -107,15 +94,22 @@ object Retry { ): F[Either[Retry.Status, A]] = attempt match { case Left(error) => isWorthRetrying(error).ifM( - applyPolicy(policy, status).flatMap { nextStep => - onError(error, nextStep.status, nextStep.decision) >> - (nextStep match { - case NextStep(updatedStatus, Decision.DelayAndRetry(delay)) => - Temporal[F].sleep(delay) *> - updatedStatus.asLeft.pure[F] // continue recursion - case NextStep(_, GiveUp) => - Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion - }) + policy.nextRetry(status) // TODO the map below will become the definition of Retry + .map { + case decision @ Decision.DelayAndRetry(delay) => + (status.addRetry(delay), decision) + case decision @ Decision.GiveUp => + (status, decision) + } + .flatTap { case (status, decision) => onError(error, status, decision) } + .flatMap { case (status, decision) => + decision match { + case Decision.DelayAndRetry(delay) => + Temporal[F].sleep(delay) *> + status.asLeft.pure[F] // continue recursion + case Decision.GiveUp => + Temporal[F].raiseError[A](error).map(_.asRight) // stop the recursion + } }, Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion ) From 329ea139f6557c42c71e8244792c233a6922c73b Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 3 Sep 2022 03:57:44 +0200 Subject: [PATCH 21/45] Remove pretty printing functionality --- .../main/scala/cats/effect/std/Retry.scala | 69 +++++++------------ 1 file changed, 23 insertions(+), 46 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index f7f9d424da..024704b038 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -135,34 +135,28 @@ object Retry { } def apply[F[_]: Monad]( - nextRetry: Retry.Status => F[Retry.Decision], - pretty: String = "" + nextRetry: Retry.Status => F[Retry.Decision] ): Retry[F] = - new RetryImpl[F](nextRetry, pretty) + new RetryImpl[F](nextRetry) def lift[F[_]: Monad]( - nextRetry: Retry.Status => Retry.Decision, - pretty: String = "" + nextRetry: Retry.Status => Retry.Decision ): Retry[F] = - apply[F]( - status => nextRetry(status).pure[F], - pretty - ) + apply[F](status => nextRetry(status).pure[F]) /** * Don't retry at all and always give up. Only really useful for combining with other * policies. */ def alwaysGiveUp[F[_]: Monad]: Retry[F] = - Retry.lift[F](Function.const(GiveUp), "alwaysGiveUp") + Retry.lift[F](Function.const(GiveUp)) /** * Delay by a constant amount before each retry. Never give up. */ def constantDelay[F[_]: Monad](delay: FiniteDuration): Retry[F] = Retry.lift[F]( - Function.const(DelayAndRetry(delay)), - show"constantDelay($delay)" + Function.const(DelayAndRetry(delay)) ) /** @@ -176,8 +170,7 @@ object Retry { val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) DelayAndRetry(delay) - }, - show"exponentialBackOff(baseDelay=$baseDelay)" + } ) /** @@ -191,8 +184,7 @@ object Retry { } else { DelayAndRetry(Duration.Zero) } - }, - show"limitRetries(maxRetries=$maxRetries)" + } ) /** @@ -209,8 +201,7 @@ object Retry { val delay = safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) DelayAndRetry(delay) - }, - show"fibonacciBackoff(baseDelay=$baseDelay)" + } ) /** @@ -224,8 +215,7 @@ object Retry { val maxDelay = safeMultiply(baseDelay, e) val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) - }, - show"fullJitter(baseDelay=$baseDelay)" + } ) // implicit def boundedSemilatticeForRetry[F[_]]( @@ -240,9 +230,6 @@ object Retry { // ): Retry[F] = x.join(y) // } -// implicit def showForRetry[F[_]]: Show[Retry[F]] = -// Show.show(_.show) - /* * Multiply the given duration by the given multiplier, but cap the result to * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. @@ -263,20 +250,17 @@ object Retry { } - private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[Retry.Decision], pretty: String) extends Retry[F] { + private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[Retry.Decision]) extends Retry[F] { - override def nextRetry(status: Retry.Status): F[Retry.Decision] = + def nextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry_(status) - override def toString: String = pretty - def followedBy(r: Retry[F]): Retry[F] = Retry( status => (nextRetry(status), r.nextRetry(status)).mapN { case (GiveUp, pd) => pd case (pd, _) => pd - }, - s"$this.followedBy($r)" + } ) def join(r: Retry[F]): Retry[F] = @@ -284,8 +268,7 @@ object Retry { status => (nextRetry(status), r.nextRetry(status)).mapN { case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) case _ => GiveUp - }, - s"$this.join($r)" + } ) def meet(r: Retry[F]): Retry[F] = @@ -295,8 +278,7 @@ object Retry { case (s @ DelayAndRetry(_), GiveUp) => s case (GiveUp, s @ DelayAndRetry(_)) => s case _ => GiveUp - }, - s"$this.meet($r)" + } ) def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = @@ -304,8 +286,7 @@ object Retry { status => nextRetry(status).map { case GiveUp => GiveUp case DelayAndRetry(d) => DelayAndRetry(f(d)) - }, - s"$this.mapDelay()" + } ) def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = @@ -313,8 +294,7 @@ object Retry { status => nextRetry(status).flatMap { case GiveUp => GiveUp.pure[F].widen[Retry.Decision] case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) - }, - s"$this.flatMapDelay()" + } ) def capDelay(cap: FiniteDuration): Retry[F] = @@ -330,8 +310,7 @@ object Retry { } Retry[F]( - decideNextRetry, - s"limitRetriesByDelay(threshold=$threshold, $this)" + decideNextRetry ) } @@ -344,16 +323,14 @@ object Retry { } Retry[F]( - decideNextRetry, - s"limitRetriesByCumulativeDelay(threshold=$threshold, $this)" + decideNextRetry ) } def mapK[G[_]: Monad](f: F ~> G): Retry[G] = - Retry( - status => f(nextRetry(status)), - s"$this.mapK()" - ) - } + Retry(status => f(nextRetry(status))) + + override def toString: String = "Retry(...)" + } } From c47278b67cac8a8a03de599a9ae8550ba9308719 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 3 Sep 2022 04:11:57 +0200 Subject: [PATCH 22/45] Refactor --- .../main/scala/cats/effect/std/Retry.scala | 154 ++++++++---------- 1 file changed, 64 insertions(+), 90 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 024704b038..27d0572aa1 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -149,15 +149,13 @@ object Retry { * policies. */ def alwaysGiveUp[F[_]: Monad]: Retry[F] = - Retry.lift[F](Function.const(GiveUp)) + Retry.lift[F](_ => GiveUp) /** * Delay by a constant amount before each retry. Never give up. */ def constantDelay[F[_]: Monad](delay: FiniteDuration): Retry[F] = - Retry.lift[F]( - Function.const(DelayAndRetry(delay)) - ) + Retry.lift[F](_ => DelayAndRetry(delay)) /** * Each delay is twice as long as the previous one. Never give up. @@ -165,27 +163,20 @@ object Retry { def exponentialBackoff[F[_]: Monad]( baseDelay: FiniteDuration ): Retry[F] = - Retry.lift[F]( - { status => - val delay = - safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) - DelayAndRetry(delay) - } - ) + Retry.lift[F] { status => + val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) + DelayAndRetry(delay) + } + /** * Retry without delay, giving up after the given number of retries. */ def limitRetries[F[_]: Monad](maxRetries: Int): Retry[F] = - Retry.lift[F]( - { status => - if (status.retriesSoFar >= maxRetries) { - GiveUp - } else { - DelayAndRetry(Duration.Zero) - } - } - ) + Retry.lift[F] { status => + if (status.retriesSoFar >= maxRetries) GiveUp + else DelayAndRetry(Duration.Zero) + } /** * Delay(n) = Delay(n - 2) + Delay(n - 1) @@ -196,27 +187,23 @@ object Retry { def fibonacciBackoff[F[_]: Monad]( baseDelay: FiniteDuration ): Retry[F] = - Retry.lift[F]( - { status => + Retry.lift[F] { status => val delay = safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) DelayAndRetry(delay) - } - ) + } /** * "Full jitter" backoff algorithm. See * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ */ def fullJitter[F[_]: Monad](baseDelay: FiniteDuration): Retry[F] = - Retry.lift[F]( - { status => - val e = Math.pow(2, status.retriesSoFar).toLong - val maxDelay = safeMultiply(baseDelay, e) - val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong - DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) - } - ) + Retry.lift[F] { status => + val e = Math.pow(2, status.retriesSoFar).toLong + val maxDelay = safeMultiply(baseDelay, e) + val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong + DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) + } // implicit def boundedSemilatticeForRetry[F[_]]( // implicit F: Monad[F]): BoundedSemilattice[Retry[F]] = @@ -255,76 +242,64 @@ object Retry { def nextRetry(status: Retry.Status): F[Retry.Decision] = nextRetry_(status) - def followedBy(r: Retry[F]): Retry[F] = - Retry( - status => (nextRetry(status), r.nextRetry(status)).mapN { - case (GiveUp, pd) => pd - case (pd, _) => pd - } - ) + def followedBy(r: Retry[F]) = Retry { status => + (nextRetry(status), r.nextRetry(status)).mapN { + case (GiveUp, decision) => decision + case (decision, _) => decision + } + } - def join(r: Retry[F]): Retry[F] = - Retry[F]( - status => (nextRetry(status), r.nextRetry(status)).mapN { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a max b) - case _ => GiveUp - } - ) - - def meet(r: Retry[F]): Retry[F] = - Retry[F]( - status => (nextRetry(status), r.nextRetry(status)).mapN { - case (DelayAndRetry(a), DelayAndRetry(b)) => DelayAndRetry(a min b) - case (s @ DelayAndRetry(_), GiveUp) => s - case (GiveUp, s @ DelayAndRetry(_)) => s - case _ => GiveUp - } - ) + def join(r: Retry[F]) = Retry[F] { status => + (nextRetry(status), r.nextRetry(status)).mapN { + case (DelayAndRetry(t1), DelayAndRetry(t2)) => DelayAndRetry(t1 max t2) + case _ => GiveUp + } + } - def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] = - Retry( - status => nextRetry(status).map { - case GiveUp => GiveUp - case DelayAndRetry(d) => DelayAndRetry(f(d)) - } - ) - def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] = - Retry( - status => nextRetry(status).flatMap { - case GiveUp => GiveUp.pure[F].widen[Retry.Decision] - case DelayAndRetry(d) => f(d).map(DelayAndRetry(_)) - } - ) + def meet(r: Retry[F]) = Retry { status => + (nextRetry(status), r.nextRetry(status)).mapN { + case (DelayAndRetry(t1), DelayAndRetry(t2)) => DelayAndRetry(t1 min t2) + case (retrying @ DelayAndRetry(_), GiveUp) => retrying + case (GiveUp, retrying @ DelayAndRetry(_)) => retrying + case _ => GiveUp + } + } + + def mapDelay(f: FiniteDuration => FiniteDuration) = Retry { status => + nextRetry(status).map { + case GiveUp => GiveUp + case DelayAndRetry(delay) => DelayAndRetry(f(delay)) + } + } + + def flatMapDelay(f: FiniteDuration => F[FiniteDuration]) = Retry { status => + nextRetry(status).flatMap { + case GiveUp => GiveUp.pure[F].widen[Retry.Decision] + case DelayAndRetry(delay) => f(delay).map(DelayAndRetry(_)) + } + } def capDelay(cap: FiniteDuration): Retry[F] = meet(constantDelay(cap)) // TODO inline these decideNextRetry definitions - def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] = { - def decideNextRetry(status: Retry.Status): F[Retry.Decision] = - nextRetry(status).map { - case r @ DelayAndRetry(delay) => - if (delay > threshold) GiveUp else r - case GiveUp => GiveUp - } - - Retry[F]( - decideNextRetry - ) + def limitRetriesByDelay(threshold: FiniteDuration) = Retry { status => + nextRetry(status).map { + case retrying @ DelayAndRetry(delay) => + if (delay > threshold) GiveUp else retrying + case GiveUp => GiveUp + } } - def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] = { - def decideNextRetry(status: Retry.Status): F[Retry.Decision] = + def limitRetriesByCumulativeDelay(threshold: FiniteDuration) = + Retry { status => nextRetry(status).map { - case r @ DelayAndRetry(delay) => - if (status.cumulativeDelay + delay >= threshold) GiveUp else r + case retrying @ DelayAndRetry(delay) => + if (status.cumulativeDelay + delay >= threshold) GiveUp + else retrying case GiveUp => GiveUp } - - Retry[F]( - decideNextRetry - ) } def mapK[G[_]: Monad](f: F ~> G): Retry[G] = @@ -333,4 +308,3 @@ object Retry { override def toString: String = "Retry(...)" } } - From 182037d807c11b97dc2d0cbbfd4d282b396e4c43 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 3 Sep 2022 04:17:51 +0200 Subject: [PATCH 23/45] Simplify retry logic --- .../main/scala/cats/effect/std/Retry.scala | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 27d0572aa1..6c732a4427 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -85,7 +85,7 @@ object Retry { onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] ): F[A] = { - def retryingOnSomeErrorsImpl[A]( + def logic[A]( policy: Retry[F], isWorthRetrying: Throwable => F[Boolean], onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit], @@ -94,23 +94,19 @@ object Retry { ): F[Either[Retry.Status, A]] = attempt match { case Left(error) => isWorthRetrying(error).ifM( - policy.nextRetry(status) // TODO the map below will become the definition of Retry - .map { + policy + .nextRetry(status) + .flatMap { case decision @ Decision.DelayAndRetry(delay) => - (status.addRetry(delay), decision) + val newStatus = status.addRetry(delay) + + onError(error, newStatus, decision) >> + Temporal[F].sleep(delay) >> + newStatus.asLeft.pure[F] // continue recursion case decision @ Decision.GiveUp => - (status, decision) - } - .flatTap { case (status, decision) => onError(error, status, decision) } - .flatMap { case (status, decision) => - decision match { - case Decision.DelayAndRetry(delay) => - Temporal[F].sleep(delay) *> - status.asLeft.pure[F] // continue recursion - case Decision.GiveUp => - Temporal[F].raiseError[A](error).map(_.asRight) // stop the recursion - } - }, + onError(error, status, decision) >> + Temporal[F].raiseError[A](error).map(_.asRight) // stop the recursion + }, Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion ) case Right(success) => @@ -123,7 +119,7 @@ object Retry { case Right(a) => a.asRight.pure[F] case attempt => - retryingOnSomeErrorsImpl( + logic( policy, isWorthRetrying, onError, From f5ef9ef9dea43da974a470f67e9480e43f76a2f0 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 3 Sep 2022 04:27:09 +0200 Subject: [PATCH 24/45] Further simplify retry logic --- .../main/scala/cats/effect/std/Retry.scala | 79 ++++++++----------- 1 file changed, 31 insertions(+), 48 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 6c732a4427..8832a9a483 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -56,16 +56,10 @@ abstract class Retry[F[_]] { } object Retry { final case class Status( - retriesSoFar: Int, - cumulativeDelay: FiniteDuration, - previousDelay: Option[FiniteDuration] - ) { - def addRetry(delay: FiniteDuration): Retry.Status = Retry.Status( - retriesSoFar = this.retriesSoFar + 1, - cumulativeDelay = this.cumulativeDelay + delay, - previousDelay = Some(delay) - ) - } + retriesSoFar: Int, + cumulativeDelay: FiniteDuration, + previousDelay: Option[FiniteDuration] + ) sealed trait Decision object Decision { @@ -85,47 +79,36 @@ object Retry { onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] ): F[A] = { - def logic[A]( - policy: Retry[F], - isWorthRetrying: Throwable => F[Boolean], - onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit], - status: Retry.Status, - attempt: Either[Throwable, A] - ): F[Either[Retry.Status, A]] = attempt match { - case Left(error) => - isWorthRetrying(error).ifM( - policy - .nextRetry(status) - .flatMap { - case decision @ Decision.DelayAndRetry(delay) => - val newStatus = status.addRetry(delay) - - onError(error, newStatus, decision) >> - Temporal[F].sleep(delay) >> - newStatus.asLeft.pure[F] // continue recursion - case decision @ Decision.GiveUp => - onError(error, status, decision) >> - Temporal[F].raiseError[A](error).map(_.asRight) // stop the recursion - }, - Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion - ) - case Right(success) => - success.asRight.pure[F] // stop the recursion - } - + def logic[A](status: Retry.Status, attempt: Either[Throwable, A]): F[Either[Retry.Status, A]] = + attempt match { + case Left(error) => + isWorthRetrying(error).ifM( + policy + .nextRetry(status) + .flatMap { + case decision @ Decision.DelayAndRetry(delay) => + val newStatus = Retry.Status( + retriesSoFar = status.retriesSoFar + 1, + cumulativeDelay = status.cumulativeDelay + delay, + previousDelay = delay.some + ) + + onError(error, newStatus, decision) >> + Temporal[F].sleep(delay).as(newStatus.asLeft) // continue recursion + case decision @ Decision.GiveUp => + onError(error, status, decision) >> + Temporal[F].raiseError[A](error).map(_.asRight) // stop the recursion + }, + Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion + ) + case Right(success) => + success.asRight.pure[F] // stop the recursion + } Temporal[F].tailRecM(Retry.noRetriesYet) { status => action.attempt.flatMap { - case Right(a) => - a.asRight.pure[F] - case attempt => - logic( - policy, - isWorthRetrying, - onError, - status, - attempt - ) + case Right(a) => a.asRight.pure[F] + case attempt => logic(status, attempt ) } } } From cfae71dd314450076446ca077a53b492080f1a94 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sat, 3 Sep 2022 04:31:02 +0200 Subject: [PATCH 25/45] Use simple recursion instead of tailRecM --- .../main/scala/cats/effect/std/Retry.scala | 54 ++++++++----------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 8832a9a483..8b40682d79 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -78,39 +78,31 @@ object Retry { isWorthRetrying: Throwable => F[Boolean], onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] ): F[A] = { - - def logic[A](status: Retry.Status, attempt: Either[Throwable, A]): F[Either[Retry.Status, A]] = - attempt match { - case Left(error) => - isWorthRetrying(error).ifM( - policy - .nextRetry(status) - .flatMap { - case decision @ Decision.DelayAndRetry(delay) => - val newStatus = Retry.Status( - retriesSoFar = status.retriesSoFar + 1, - cumulativeDelay = status.cumulativeDelay + delay, - previousDelay = delay.some - ) - - onError(error, newStatus, decision) >> - Temporal[F].sleep(delay).as(newStatus.asLeft) // continue recursion - case decision @ Decision.GiveUp => - onError(error, status, decision) >> - Temporal[F].raiseError[A](error).map(_.asRight) // stop the recursion - }, - Temporal[F].raiseError[A](error).map(Right(_)) // stop the recursion - ) - case Right(success) => - success.asRight.pure[F] // stop the recursion + def loop(status: Retry.Status): F[A] = + action.handleErrorWith { error => + isWorthRetrying(error).ifM( + policy + .nextRetry(status) + .flatMap { + case decision @ Decision.DelayAndRetry(delay) => + val newStatus = Retry.Status( + retriesSoFar = status.retriesSoFar + 1, + cumulativeDelay = status.cumulativeDelay + delay, + previousDelay = delay.some + ) + + onError(error, newStatus, decision) >> + Temporal[F].sleep(delay) >> + loop(newStatus) + case decision @ Decision.GiveUp => + onError(error, status, decision) >> + Temporal[F].raiseError[A](error) + }, + Temporal[F].raiseError[A](error) + ) } - Temporal[F].tailRecM(Retry.noRetriesYet) { status => - action.attempt.flatMap { - case Right(a) => a.asRight.pure[F] - case attempt => logic(status, attempt ) - } - } + loop(noRetriesYet) } def apply[F[_]: Monad]( From c19550fbab80ebabb2c71af663029e30208a6eb6 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 4 Sep 2022 13:54:25 +0200 Subject: [PATCH 26/45] Remove references to the name policy --- .../src/main/scala/cats/effect/std/Retry.scala | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 8b40682d79..f95206f43d 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -34,21 +34,21 @@ abstract class Retry[F[_]] { def flatMapDelay(f: FiniteDuration => F[FiniteDuration]): Retry[F] /** - * Set an upper bound on any individual delay produced by the given policy. + * Set an upper bound on any individual delay produced by the given Retry. */ def capDelay(cap: FiniteDuration): Retry[F] /** - * Add an upper bound to a policy such that once the given time-delay amount per try - * has been reached or exceeded, the policy will stop retrying and give up. If you need to + * Add an upper bound to a Retry such that once the given time-delay amount per try + * has been reached or exceeded, the Retry will stop retrying and give up. If you need to * stop retrying once cumulative delay reaches a time-delay amount, use * [[limitRetriesByCumulativeDelay]]. */ def limitRetriesByDelay(threshold: FiniteDuration): Retry[F] /** - * Add an upperbound to a policy such that once the cumulative delay over all retries has - * reached or exceeded the given limit, the policy will stop retrying and give up. + * Add an upperbound to a Retry such that once the cumulative delay over all retries has + * reached or exceeded the given limit, the Retry will stop retrying and give up. */ def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] @@ -71,9 +71,8 @@ object Retry { val noRetriesYet = Retry.Status(0, Duration.Zero, None) - // TODO replace the name policy def retry[F[_]: Temporal, A]( - policy: Retry[F], + r: Retry[F], action: F[A], isWorthRetrying: Throwable => F[Boolean], onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] @@ -81,7 +80,7 @@ object Retry { def loop(status: Retry.Status): F[A] = action.handleErrorWith { error => isWorthRetrying(error).ifM( - policy + r .nextRetry(status) .flatMap { case decision @ Decision.DelayAndRetry(delay) => From 50e640604980a1025459ff307dfc36d282e66620 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 4 Sep 2022 14:51:11 +0200 Subject: [PATCH 27/45] Remove stale todo --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index f95206f43d..03a65eda91 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -253,7 +253,6 @@ object Retry { def capDelay(cap: FiniteDuration): Retry[F] = meet(constantDelay(cap)) - // TODO inline these decideNextRetry definitions def limitRetriesByDelay(threshold: FiniteDuration) = Retry { status => nextRetry(status).map { case retrying @ DelayAndRetry(delay) => From 4e41a1cc3368eefd7a417167bd92687428875ff1 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 4 Sep 2022 15:04:16 +0200 Subject: [PATCH 28/45] Bundle error selection inside retry --- .../main/scala/cats/effect/std/Retry.scala | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 03a65eda91..84c58c4450 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -11,7 +11,7 @@ import scala.util.Random import java.util.concurrent.TimeUnit abstract class Retry[F[_]] { - def nextRetry(status: Retry.Status): F[Retry.Decision] + def nextRetry(status: Retry.Status, error: Throwable): F[Retry.Decision] def followedBy(r: Retry[F]): Retry[F] @@ -81,7 +81,7 @@ object Retry { action.handleErrorWith { error => isWorthRetrying(error).ifM( r - .nextRetry(status) + .nextRetry(status, error) .flatMap { case decision @ Decision.DelayAndRetry(delay) => val newStatus = Retry.Status( @@ -105,14 +105,19 @@ object Retry { } def apply[F[_]: Monad]( - nextRetry: Retry.Status => F[Retry.Decision] + nextRetry: (Retry.Status, Throwable) => F[Retry.Decision] ): Retry[F] = new RetryImpl[F](nextRetry) + def liftF[F[_]: Monad]( + nextRetry: Retry.Status => F[Retry.Decision] + ): Retry[F] = + Retry((status, _) => nextRetry(status)) + def lift[F[_]: Monad]( nextRetry: Retry.Status => Retry.Decision ): Retry[F] = - apply[F](status => nextRetry(status).pure[F]) + liftF[F](status => nextRetry(status).pure[F]) /** * Don't retry at all and always give up. Only really useful for combining with other @@ -207,28 +212,28 @@ object Retry { } - private final case class RetryImpl[F[_]: Monad](nextRetry_ : Retry.Status => F[Retry.Decision]) extends Retry[F] { + private final case class RetryImpl[F[_]: Monad](nextRetry_ : (Retry.Status, Throwable) => F[Retry.Decision]) extends Retry[F] { - def nextRetry(status: Retry.Status): F[Retry.Decision] = - nextRetry_(status) + def nextRetry(status: Retry.Status, error: Throwable): F[Retry.Decision] = + nextRetry_(status, error) - def followedBy(r: Retry[F]) = Retry { status => - (nextRetry(status), r.nextRetry(status)).mapN { + def followedBy(r: Retry[F]) = Retry { (status, error) => + (nextRetry(status, error), r.nextRetry(status, error)).mapN { case (GiveUp, decision) => decision case (decision, _) => decision } } - def join(r: Retry[F]) = Retry[F] { status => - (nextRetry(status), r.nextRetry(status)).mapN { + def join(r: Retry[F]) = Retry[F] { (status, error) => + (nextRetry(status, error), r.nextRetry(status, error)).mapN { case (DelayAndRetry(t1), DelayAndRetry(t2)) => DelayAndRetry(t1 max t2) case _ => GiveUp } } - def meet(r: Retry[F]) = Retry { status => - (nextRetry(status), r.nextRetry(status)).mapN { + def meet(r: Retry[F]) = Retry { (status, error) => + (nextRetry(status, error), r.nextRetry(status, error)).mapN { case (DelayAndRetry(t1), DelayAndRetry(t2)) => DelayAndRetry(t1 min t2) case (retrying @ DelayAndRetry(_), GiveUp) => retrying case (GiveUp, retrying @ DelayAndRetry(_)) => retrying @@ -236,25 +241,29 @@ object Retry { } } - def mapDelay(f: FiniteDuration => FiniteDuration) = Retry { status => - nextRetry(status).map { + def mapDelay(f: FiniteDuration => FiniteDuration) = Retry { (status, error) => + nextRetry(status, error).map { case GiveUp => GiveUp case DelayAndRetry(delay) => DelayAndRetry(f(delay)) } } - def flatMapDelay(f: FiniteDuration => F[FiniteDuration]) = Retry { status => - nextRetry(status).flatMap { + def flatMapDelay(f: FiniteDuration => F[FiniteDuration]) = Retry { (status, error) => + nextRetry(status, error).flatMap { case GiveUp => GiveUp.pure[F].widen[Retry.Decision] case DelayAndRetry(delay) => f(delay).map(DelayAndRetry(_)) } } + // TODO + // This implementation doesn't work, a retry that only retries on specific errors + // will always retry once capped since constant delay never gives up, and meet only gives up if + // both retries do def capDelay(cap: FiniteDuration): Retry[F] = meet(constantDelay(cap)) - def limitRetriesByDelay(threshold: FiniteDuration) = Retry { status => - nextRetry(status).map { + def limitRetriesByDelay(threshold: FiniteDuration) = Retry { (status, error) => + nextRetry(status, error).map { case retrying @ DelayAndRetry(delay) => if (delay > threshold) GiveUp else retrying case GiveUp => GiveUp @@ -262,8 +271,8 @@ object Retry { } def limitRetriesByCumulativeDelay(threshold: FiniteDuration) = - Retry { status => - nextRetry(status).map { + Retry { (status, error) => + nextRetry(status, error).map { case retrying @ DelayAndRetry(delay) => if (status.cumulativeDelay + delay >= threshold) GiveUp else retrying @@ -272,7 +281,7 @@ object Retry { } def mapK[G[_]: Monad](f: F ~> G): Retry[G] = - Retry(status => f(nextRetry(status))) + Retry((status, error) => f(nextRetry(status, error))) override def toString: String = "Retry(...)" } From 424778765ae82c6dfe927b3f4c0afff4c3663b8e Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 4 Sep 2022 15:37:13 +0200 Subject: [PATCH 29/45] Embed error selection into Retry --- .../main/scala/cats/effect/std/Retry.scala | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 84c58c4450..184208752e 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -52,6 +52,11 @@ abstract class Retry[F[_]] { */ def limitRetriesByCumulativeDelay(threshold: FiniteDuration): Retry[F] + def selectError(f: Throwable => Boolean): Retry[F] + + def selectErrorWith(f: Throwable => F[Boolean]): Retry[F] + + def mapK[G[_]: Monad](f: F ~> G): Retry[G] } object Retry { @@ -74,31 +79,27 @@ object Retry { def retry[F[_]: Temporal, A]( r: Retry[F], action: F[A], - isWorthRetrying: Throwable => F[Boolean], onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] ): F[A] = { def loop(status: Retry.Status): F[A] = action.handleErrorWith { error => - isWorthRetrying(error).ifM( - r - .nextRetry(status, error) - .flatMap { - case decision @ Decision.DelayAndRetry(delay) => - val newStatus = Retry.Status( - retriesSoFar = status.retriesSoFar + 1, - cumulativeDelay = status.cumulativeDelay + delay, - previousDelay = delay.some - ) - - onError(error, newStatus, decision) >> - Temporal[F].sleep(delay) >> - loop(newStatus) - case decision @ Decision.GiveUp => - onError(error, status, decision) >> - Temporal[F].raiseError[A](error) - }, - Temporal[F].raiseError[A](error) - ) + r + .nextRetry(status, error) + .flatMap { + case decision @ Decision.DelayAndRetry(delay) => + val newStatus = Retry.Status( + retriesSoFar = status.retriesSoFar + 1, + cumulativeDelay = status.cumulativeDelay + delay, + previousDelay = delay.some + ) + + onError(error, newStatus, decision) >> + Temporal[F].sleep(delay) >> + loop(newStatus) + case decision @ Decision.GiveUp => + onError(error, status, decision) >> + Temporal[F].raiseError[A](error) + } } loop(noRetriesYet) @@ -280,6 +281,20 @@ object Retry { } } + def selectError(f: Throwable => Boolean): Retry[F] = + Retry { (status, error) => + if(f(error)) nextRetry(status, error) + else GiveUp.pure[F].widen[Decision] + } + + def selectErrorWith(f: Throwable => F[Boolean]): Retry[F] = + Retry { (status, error) => + f(error).flatMap { + case true => nextRetry(status, error) + case false => GiveUp.pure[F].widen[Decision] + } + } + def mapK[G[_]: Monad](f: F ~> G): Retry[G] = Retry((status, error) => f(nextRetry(status, error))) From 9d369687989d8a5770ac84bec8dc483979d11eaf Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 4 Sep 2022 15:44:43 +0200 Subject: [PATCH 30/45] Working implementation of capDelay --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 184208752e..3bc1be21a1 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -256,12 +256,8 @@ object Retry { } } - // TODO - // This implementation doesn't work, a retry that only retries on specific errors - // will always retry once capped since constant delay never gives up, and meet only gives up if - // both retries do def capDelay(cap: FiniteDuration): Retry[F] = - meet(constantDelay(cap)) + mapDelay(delay => delay.min(cap)) def limitRetriesByDelay(threshold: FiniteDuration) = Retry { (status, error) => nextRetry(status, error).map { From fe5b422a14581da3d8931d43b581edb82d733b38 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 5 Sep 2022 00:02:37 +0200 Subject: [PATCH 31/45] Add flatTap --- .../main/scala/cats/effect/std/Retry.scala | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 3bc1be21a1..5c1f03e7f0 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -56,6 +56,7 @@ abstract class Retry[F[_]] { def selectErrorWith(f: Throwable => F[Boolean]): Retry[F] + def flatTap(f: (Throwable, Retry.Decision, Retry.Status) => F[Unit]): Retry[F] def mapK[G[_]: Monad](f: F ~> G): Retry[G] } @@ -76,28 +77,21 @@ object Retry { val noRetriesYet = Retry.Status(0, Duration.Zero, None) - def retry[F[_]: Temporal, A]( - r: Retry[F], - action: F[A], - onError: (Throwable, Retry.Status, Retry.Decision) => F[Unit] - ): F[A] = { + def retry[F[_]: Temporal, A](r: Retry[F], action: F[A]): F[A] = { def loop(status: Retry.Status): F[A] = action.handleErrorWith { error => r .nextRetry(status, error) .flatMap { - case decision @ Decision.DelayAndRetry(delay) => + case DelayAndRetry(delay) => val newStatus = Retry.Status( retriesSoFar = status.retriesSoFar + 1, cumulativeDelay = status.cumulativeDelay + delay, previousDelay = delay.some ) - onError(error, newStatus, decision) >> - Temporal[F].sleep(delay) >> - loop(newStatus) - case decision @ Decision.GiveUp => - onError(error, status, decision) >> + Temporal[F].sleep(delay) >> loop(newStatus) + case GiveUp => Temporal[F].raiseError[A](error) } } @@ -291,6 +285,22 @@ object Retry { } } + def flatTap(f: (Throwable, Retry.Decision, Retry.Status) => F[Unit]): Retry[F] = + Retry { (status, error) => + nextRetry(status, error).flatTap { + case decision @ DelayAndRetry(delay) => + val newStatus = Retry.Status( + retriesSoFar = status.retriesSoFar + 1, + cumulativeDelay = status.cumulativeDelay + delay, + previousDelay = delay.some + ) + + f(error, decision, newStatus) + case decision @ GiveUp => + f(error, decision, status) + } + } + def mapK[G[_]: Monad](f: F ~> G): Retry[G] = Retry((status, error) => f(nextRetry(status, error))) From e32d87fb143f50261f27db8a67a77afd956c6d84 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 5 Sep 2022 00:14:19 +0200 Subject: [PATCH 32/45] Reintroduce addRetry method on Status --- .../main/scala/cats/effect/std/Retry.scala | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 5c1f03e7f0..9f09c58bcc 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -65,7 +65,13 @@ object Retry { retriesSoFar: Int, cumulativeDelay: FiniteDuration, previousDelay: Option[FiniteDuration] - ) + ) { + def addRetry(delay: FiniteDuration) = Retry.Status( + retriesSoFar = retriesSoFar + 1, + cumulativeDelay = cumulativeDelay + delay, + previousDelay = delay.some + ) + } sealed trait Decision object Decision { @@ -84,13 +90,7 @@ object Retry { .nextRetry(status, error) .flatMap { case DelayAndRetry(delay) => - val newStatus = Retry.Status( - retriesSoFar = status.retriesSoFar + 1, - cumulativeDelay = status.cumulativeDelay + delay, - previousDelay = delay.some - ) - - Temporal[F].sleep(delay) >> loop(newStatus) + Temporal[F].sleep(delay) >> loop(status.addRetry(delay)) case GiveUp => Temporal[F].raiseError[A](error) } @@ -289,13 +289,7 @@ object Retry { Retry { (status, error) => nextRetry(status, error).flatTap { case decision @ DelayAndRetry(delay) => - val newStatus = Retry.Status( - retriesSoFar = status.retriesSoFar + 1, - cumulativeDelay = status.cumulativeDelay + delay, - previousDelay = delay.some - ) - - f(error, decision, newStatus) + f(error, decision, status.addRetry(delay)) case decision @ GiveUp => f(error, decision, status) } From e0f37f0ebe85b435c4d23e44f720ca637854462a Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 5 Sep 2022 00:21:21 +0200 Subject: [PATCH 33/45] Rename join/meet to and/or --- .../src/main/scala/cats/effect/std/Retry.scala | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 9f09c58bcc..0319144acf 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -16,18 +16,16 @@ abstract class Retry[F[_]] { def followedBy(r: Retry[F]): Retry[F] /** - * Combine this schedule with another schedule, giving up when either of the schedules want to - * give up and choosing the maximum of the two delays when both of the schedules want to delay - * the next retry. The dual of the `meet` operation. + * Combine this schedule with another schedule, retrying when both schedules want to retry + * and choosing the maximum of the two delays. */ - def join(r: Retry[F]): Retry[F] + def and(r: Retry[F]): Retry[F] /** - * Combine this schedule with another schedule, giving up when both of the schedules want to - * give up and choosing the minimum of the two delays when both of the schedules want to delay - * the next retry. The dual of the `join` operation. + * Combine this schedule with another schedule, retrying when either schedule wants to retry + * and choosing the minimum of the two delays when both schedules want to retry. */ - def meet(r: Retry[F]): Retry[F] + def or(r: Retry[F]): Retry[F] def mapDelay(f: FiniteDuration => FiniteDuration): Retry[F] @@ -219,7 +217,7 @@ object Retry { } } - def join(r: Retry[F]) = Retry[F] { (status, error) => + def and(r: Retry[F]) = Retry[F] { (status, error) => (nextRetry(status, error), r.nextRetry(status, error)).mapN { case (DelayAndRetry(t1), DelayAndRetry(t2)) => DelayAndRetry(t1 max t2) case _ => GiveUp @@ -227,7 +225,7 @@ object Retry { } - def meet(r: Retry[F]) = Retry { (status, error) => + def or(r: Retry[F]) = Retry { (status, error) => (nextRetry(status, error), r.nextRetry(status, error)).mapN { case (DelayAndRetry(t1), DelayAndRetry(t2)) => DelayAndRetry(t1 min t2) case (retrying @ DelayAndRetry(_), GiveUp) => retrying From a60822033fa84c10c9d047b8196b31a17753b20f Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Mon, 5 Sep 2022 01:58:51 +0200 Subject: [PATCH 34/45] Make limitRetries into a combinator --- .../src/main/scala/cats/effect/std/Retry.scala | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 0319144acf..4c244e8c7a 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -36,6 +36,8 @@ abstract class Retry[F[_]] { */ def capDelay(cap: FiniteDuration): Retry[F] + def limitRetries(maxRetries: Int): Retry[F] + /** * Add an upper bound to a Retry such that once the given time-delay amount per try * has been reached or exceeded, the Retry will stop retrying and give up. If you need to @@ -136,16 +138,6 @@ object Retry { DelayAndRetry(delay) } - - /** - * Retry without delay, giving up after the given number of retries. - */ - def limitRetries[F[_]: Monad](maxRetries: Int): Retry[F] = - Retry.lift[F] { status => - if (status.retriesSoFar >= maxRetries) GiveUp - else DelayAndRetry(Duration.Zero) - } - /** * Delay(n) = Delay(n - 2) + Delay(n - 1) * @@ -204,7 +196,6 @@ object Retry { FiniteDuration(safeResultNanos.toLong, TimeUnit.NANOSECONDS) } - private final case class RetryImpl[F[_]: Monad](nextRetry_ : (Retry.Status, Throwable) => F[Retry.Decision]) extends Retry[F] { def nextRetry(status: Retry.Status, error: Throwable): F[Retry.Decision] = @@ -251,6 +242,11 @@ object Retry { def capDelay(cap: FiniteDuration): Retry[F] = mapDelay(delay => delay.min(cap)) + def limitRetries(maxRetries: Int): Retry[F] = Retry { (status, error) => + if (status.retriesSoFar >= maxRetries) GiveUp.pure[F].widen[Decision] + else nextRetry(status, error) + } + def limitRetriesByDelay(threshold: FiniteDuration) = Retry { (status, error) => nextRetry(status, error).map { case retrying @ DelayAndRetry(delay) => From 0cf72df092ecdbc2c9e32289c9c61461ae049149 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:25:55 +0200 Subject: [PATCH 35/45] Remove unused import --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 4c244e8c7a..1864a0b47c 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -3,8 +3,7 @@ package retry import cats._ import cats.syntax.all._ -import cats.arrow.FunctionK -import cats.kernel.BoundedSemilattice +//import cats.kernel.BoundedSemilattice import cats.effect.kernel.Temporal import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.Random From 0eca605247fe1a4c7d5af23df51524508da76c70 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:32:12 +0200 Subject: [PATCH 36/45] Inline fibonacci code --- .../main/scala/cats/effect/std/Retry.scala | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 1864a0b47c..1c5f48d2be 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -145,12 +145,38 @@ object Retry { */ def fibonacciBackoff[F[_]: Monad]( baseDelay: FiniteDuration - ): Retry[F] = + ): Retry[F] = { + // "Fast doubling" Fibonacci algorithm. + // See e.g. http://funloop.org/post/2017-04-14-computing-fibonacci-numbers.html for explanation. + def fib(n: Int): (Long, Long) = n match { + case 0 => (0, 1) + case m => + val (a, b) = fib(m / 2) + val c = a * (b * 2 - a) + val d = a * a + b * b + if (n % 2 == 0) + (c, d) + else + (d, c + d) + } + + // TODO + // this can probably be eliminated (after tests) since it only + // exists for that > 0 test, which is a condition that cannot + // happen at use site + def fibonacci(n: Int): Long = { + if (n > 0) + fib(n)._1 + else + 0 + } + Retry.lift[F] { status => val delay = - safeMultiply(baseDelay, Fibonacci.fibonacci(status.retriesSoFar + 1)) + safeMultiply(baseDelay, fibonacci(status.retriesSoFar + 1)) DelayAndRetry(delay) } + } /** * "Full jitter" backoff algorithm. See From 3bdab782ee4cc80e48d4b74e0fb7a672e25889ff Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:32:41 +0200 Subject: [PATCH 37/45] No need for a retry package --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 1c5f48d2be..64e18aa2c1 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -1,5 +1,4 @@ package cats.effect.std -package retry import cats._ import cats.syntax.all._ From ff0a3232a167ac93ce71ff7ab66d7f53a7237e9a Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:33:39 +0200 Subject: [PATCH 38/45] No implicit numeric widening --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 64e18aa2c1..d7101b1c5a 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -132,7 +132,7 @@ object Retry { baseDelay: FiniteDuration ): Retry[F] = Retry.lift[F] { status => - val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar).toLong) + val delay = safeMultiply(baseDelay, Math.pow(2, status.retriesSoFar.toDouble).toLong) DelayAndRetry(delay) } @@ -183,7 +183,7 @@ object Retry { */ def fullJitter[F[_]: Monad](baseDelay: FiniteDuration): Retry[F] = Retry.lift[F] { status => - val e = Math.pow(2, status.retriesSoFar).toLong + val e = Math.pow(2, status.retriesSoFar.toDouble).toLong val maxDelay = safeMultiply(baseDelay, e) val delayNanos = (maxDelay.toNanos * Random.nextDouble()).toLong DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) From 1291c8ca167a67030b9a4ee62abff543b66a5537 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:37:41 +0200 Subject: [PATCH 39/45] Verify assumption on negative delays --- std/shared/src/main/scala/cats/effect/std/Retry.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index d7101b1c5a..50428c31a3 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -204,15 +204,12 @@ object Retry { /* * Multiply the given duration by the given multiplier, but cap the result to * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. - * - * Note: despite the "safe" in the name, we can still create an invalid - * FiniteDuration if the multiplier is negative. But an assumption of the library - * as a whole is that nobody would be silly enough to use negative delays. */ private def safeMultiply( duration: FiniteDuration, multiplier: Long ): FiniteDuration = { + assert(multiplier > 0, "Don't use a negative delay") val longMax: BigInt = BigInt(Long.MaxValue) val durationNanos = BigInt(duration.toNanos) val resultNanos = durationNanos * BigInt(multiplier) From aec3767db0e6b89b998c7359182099ab5510f891 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:47:26 +0200 Subject: [PATCH 40/45] Remove BoundedSemiLattice[Retry] for now --- .../src/main/scala/cats/effect/std/Retry.scala | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/std/shared/src/main/scala/cats/effect/std/Retry.scala b/std/shared/src/main/scala/cats/effect/std/Retry.scala index 50428c31a3..24e39c03e3 100644 --- a/std/shared/src/main/scala/cats/effect/std/Retry.scala +++ b/std/shared/src/main/scala/cats/effect/std/Retry.scala @@ -2,7 +2,6 @@ package cats.effect.std import cats._ import cats.syntax.all._ -//import cats.kernel.BoundedSemilattice import cats.effect.kernel.Temporal import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.Random @@ -189,18 +188,6 @@ object Retry { DelayAndRetry(new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS)) } -// implicit def boundedSemilatticeForRetry[F[_]]( -// implicit F: Monad[F]): BoundedSemilattice[Retry[F]] = -// new BoundedSemilattice[Retry[F]] { -// override def empty: Retry[F] = -// RetryPolicies.constantDelay[F](Duration.Zero) - -// override def combine( -// x: Retry[F], -// y: Retry[F] -// ): Retry[F] = x.join(y) -// } - /* * Multiply the given duration by the given multiplier, but cap the result to * ensure we don't try to create a FiniteDuration longer than 2^63 - 1 nanoseconds. From d2acde95d06f7bf29138c3261df72199c2dfc927 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:47:59 +0200 Subject: [PATCH 41/45] Start work on retry tests --- tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala diff --git a/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala b/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala new file mode 100644 index 0000000000..c0d5c6b664 --- /dev/null +++ b/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala @@ -0,0 +1,3 @@ +package cats.effect.std + + From 7232eba624c6791295792531fd3c77b5f7c159b2 Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:54:50 +0200 Subject: [PATCH 42/45] Add retry syntax --- .../scala/cats/effect/std/syntax/AllSyntax.scala | 2 +- .../cats/effect/std/syntax/RetrySyntax.scala | 15 +++++++++++++++ .../scala/cats/effect/std/syntax/package.scala | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 std/shared/src/main/scala/cats/effect/std/syntax/RetrySyntax.scala diff --git a/std/shared/src/main/scala/cats/effect/std/syntax/AllSyntax.scala b/std/shared/src/main/scala/cats/effect/std/syntax/AllSyntax.scala index 2636bab87f..026e36f570 100644 --- a/std/shared/src/main/scala/cats/effect/std/syntax/AllSyntax.scala +++ b/std/shared/src/main/scala/cats/effect/std/syntax/AllSyntax.scala @@ -17,4 +17,4 @@ package cats.effect package std.syntax -trait AllSyntax extends BackpressureSyntax with SupervisorSyntax +trait AllSyntax extends BackpressureSyntax with SupervisorSyntax with RetrySyntax diff --git a/std/shared/src/main/scala/cats/effect/std/syntax/RetrySyntax.scala b/std/shared/src/main/scala/cats/effect/std/syntax/RetrySyntax.scala new file mode 100644 index 0000000000..a9315c5b92 --- /dev/null +++ b/std/shared/src/main/scala/cats/effect/std/syntax/RetrySyntax.scala @@ -0,0 +1,15 @@ +package cats.effect.std.syntax + +import cats.effect.kernel.Temporal +import cats.effect.std.Retry + +trait RetrySyntax { + implicit def retryOps[F[_], A](wrapped: F[A]): RetryOps[F, A] = + new RetryOps(wrapped) +} + +final class RetryOps[F[_], A] private[syntax] (private[syntax] val wrapped: F[A]) + extends AnyVal { + def retry(r: Retry[F])(implicit F: Temporal[F]): F[A] = + Retry.retry(r, wrapped) +} diff --git a/std/shared/src/main/scala/cats/effect/std/syntax/package.scala b/std/shared/src/main/scala/cats/effect/std/syntax/package.scala index 8016cca34e..75193ada35 100644 --- a/std/shared/src/main/scala/cats/effect/std/syntax/package.scala +++ b/std/shared/src/main/scala/cats/effect/std/syntax/package.scala @@ -22,4 +22,6 @@ package object syntax { object supervisor extends SupervisorSyntax + object retry extends RetrySyntax + } From 39bdd0e8782a2a8f0c6b696cb073311ba2e6869f Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:55:27 +0200 Subject: [PATCH 43/45] Add missing syntax package for backpressure syntax --- std/shared/src/main/scala/cats/effect/std/syntax/package.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/std/shared/src/main/scala/cats/effect/std/syntax/package.scala b/std/shared/src/main/scala/cats/effect/std/syntax/package.scala index 75193ada35..a495739430 100644 --- a/std/shared/src/main/scala/cats/effect/std/syntax/package.scala +++ b/std/shared/src/main/scala/cats/effect/std/syntax/package.scala @@ -24,4 +24,6 @@ package object syntax { object retry extends RetrySyntax + object backpressure extends BackpressureSyntax + } From c5df4e67a7b0751a6d9bf02610af1b6b2eac09fa Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 10:57:39 +0200 Subject: [PATCH 44/45] Add retry syntax for concrete IO --- core/shared/src/main/scala/cats/effect/IO.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/shared/src/main/scala/cats/effect/IO.scala b/core/shared/src/main/scala/cats/effect/IO.scala index 63571da90e..dbbad17e63 100644 --- a/core/shared/src/main/scala/cats/effect/IO.scala +++ b/core/shared/src/main/scala/cats/effect/IO.scala @@ -36,7 +36,7 @@ import cats.{ } import cats.data.Ior import cats.effect.instances.spawn -import cats.effect.std.{Console, Env, UUIDGen} +import cats.effect.std.{Console, Env, UUIDGen, Retry} import cats.effect.tracing.{Tracing, TracingEvent} import cats.syntax.all._ @@ -484,6 +484,10 @@ sealed abstract class IO[+A] private () extends IOPlatform[A] { def recoverWith[B >: A](pf: PartialFunction[Throwable, IO[B]]): IO[B] = handleErrorWith(e => pf.applyOrElse(e, IO.raiseError)) + + def retry(r: Retry[IO]): IO[A] = + Retry.retry(r, this) + def ifM[B](ifTrue: => IO[B], ifFalse: => IO[B])(implicit ev: A <:< Boolean): IO[B] = flatMap(a => if (ev(a)) ifTrue else ifFalse) From c76444e7824e304500788248b139df6f6dd2f79e Mon Sep 17 00:00:00 2001 From: Fabio Labella Date: Sun, 11 Sep 2022 11:04:07 +0200 Subject: [PATCH 45/45] Start porting tests --- .../scala/cats/effect/std/RetrySpec.scala | 970 +++++++++++++++++- 1 file changed, 969 insertions(+), 1 deletion(-) diff --git a/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala b/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala index c0d5c6b664..784cd36a44 100644 --- a/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala +++ b/tests/shared/src/test/scala/cats/effect/std/RetrySpec.scala @@ -1,3 +1,971 @@ -package cats.effect.std +// package cats.effect.std +// import cats.{Id, catsInstancesForId} +// import org.scalatest.flatspec.AnyFlatSpec +// import retry.syntax.all._ +// import scala.collection.mutable.ArrayBuffer +// import scala.concurrent.duration._ + +// class SyntaxSpec extends AnyFlatSpec { +// type StringOr[A] = Either[String, A] + +// behavior of "retryingOnFailures" + +// it should "retry until the action succeeds" in new TestContext { +// val policy: RetryPolicy[Id] = +// RetryPolicies.constantDelay[Id](1.second) +// def onFailure: (String, RetryDetails) => Id[Unit] = onError +// def wasSuccessful(res: String): Id[Boolean] = res.toInt > 3 +// val sleeps = ArrayBuffer.empty[FiniteDuration] + +// implicit val dummySleep: Sleep[Id] = +// (delay: FiniteDuration) => sleeps.append(delay) + +// def action: Id[String] = { +// attempts = attempts + 1 +// attempts.toString +// } + +// val finalResult: Id[String] = +// action.retryingOnFailures(wasSuccessful, policy, onFailure) + +// assert(finalResult == "4") +// assert(attempts == 4) +// assert(errors.toList == List("1", "2", "3")) +// assert(delays.toList == List(1.second, 1.second, 1.second)) +// assert(sleeps.toList == delays.toList) +// assert(!gaveUp) +// } + +// it should "retry until the policy chooses to give up" in new TestContext { +// val policy: RetryPolicy[Id] = RetryPolicies.limitRetries[Id](2) +// implicit val dummySleep: Sleep[Id] = _ => () + +// def action: Id[String] = { +// attempts = attempts + 1 +// attempts.toString +// } + +// val finalResult: Id[String] = +// action.retryingOnFailures(_.toInt > 3, policy, onError) + +// assert(finalResult == "3") +// assert(attempts == 3) +// assert(errors.toList == List("1", "2", "3")) +// assert(delays.toList == List(Duration.Zero, Duration.Zero)) +// assert(gaveUp) +// } + +// behavior of "retryingOnSomeErrors" + +// it should "retry until the action succeeds" in new TestContext { +// implicit val sleepForEither: Sleep[StringOr] = _ => Right(()) + +// val policy: RetryPolicy[StringOr] = +// RetryPolicies.constantDelay[StringOr](1.second) + +// def action: StringOr[String] = { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Right("yay") +// } + +// val finalResult: StringOr[String] = +// action.retryingOnSomeErrors( +// s => Right(s == "one more time"), +// policy, +// (err, rd) => onError(err, rd) +// ) + +// assert(finalResult == Right("yay")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert(!gaveUp) +// } + +// it should "retry only if the error is worth retrying" in new TestContext { +// implicit val sleepForEither: Sleep[StringOr] = _ => Right(()) + +// val policy: RetryPolicy[StringOr] = +// RetryPolicies.constantDelay[StringOr](1.second) + +// def action: StringOr[Nothing] = { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Left("nope") +// } + +// val finalResult = +// action.retryingOnSomeErrors( +// s => Right(s == "one more time"), +// policy, +// (err, rd) => onError(err, rd) +// ) + +// assert(finalResult == Left("nope")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert( +// !gaveUp +// ) // false because onError is only called when the error is worth retrying +// } + +// it should "retry until the policy chooses to give up" in new TestContext { +// implicit val sleepForEither: Sleep[StringOr] = _ => Right(()) + +// val policy: RetryPolicy[StringOr] = +// RetryPolicies.limitRetries[StringOr](2) + +// def action: StringOr[Nothing] = { +// attempts = attempts + 1 + +// Left("one more time") +// } + +// val finalResult: StringOr[Nothing] = +// action.retryingOnSomeErrors( +// s => Right(s == "one more time"), +// policy, +// (err, rd) => onError(err, rd) +// ) + +// assert(finalResult == Left("one more time")) +// assert(attempts == 3) +// assert( +// errors.toList == List("one more time", "one more time", "one more time") +// ) +// assert(gaveUp) +// } + +// behavior of "retryingOnAllErrors" + +// it should "retry until the action succeeds" in new TestContext { +// implicit val sleepForEither: Sleep[StringOr] = _ => Right(()) + +// val policy: RetryPolicy[StringOr] = +// RetryPolicies.constantDelay[StringOr](1.second) + +// def action: StringOr[String] = { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Right("yay") +// } + +// val finalResult: StringOr[String] = +// action.retryingOnAllErrors(policy, (err, rd) => onError(err, rd)) + +// assert(finalResult == Right("yay")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert(!gaveUp) +// } + +// it should "retry until the policy chooses to give up" in new TestContext { +// implicit val sleepForEither: Sleep[StringOr] = _ => Right(()) + +// val policy: RetryPolicy[StringOr] = +// RetryPolicies.limitRetries[StringOr](2) + +// def action: StringOr[Nothing] = { +// attempts = attempts + 1 +// Left("one more time") +// } + +// val finalResult = +// action.retryingOnAllErrors(policy, (err, rd) => onError(err, rd)) + +// assert(finalResult == Left("one more time")) +// assert(attempts == 3) +// assert( +// errors.toList == List("one more time", "one more time", "one more time") +// ) +// assert(gaveUp) +// } + +// private class TestContext { +// var attempts = 0 +// val errors = ArrayBuffer.empty[String] +// val delays = ArrayBuffer.empty[FiniteDuration] +// var gaveUp = false + +// def onError(error: String, details: RetryDetails): Either[String, Unit] = { +// errors.append(error) +// details match { +// case RetryDetails.WillDelayAndRetry(delay, _, _) => delays.append(delay) +// case RetryDetails.GivingUp(_, _) => gaveUp = true +// } +// Right(()) +// } +// } +// } + +// import org.scalatest.flatspec.AnyFlatSpec + +// class FibonacciSpec extends AnyFlatSpec { +// it should "calculate the Fibonacci sequence" in { +// assert(Fibonacci.fibonacci(0) == 0) +// assert(Fibonacci.fibonacci(1) == 1) +// assert(Fibonacci.fibonacci(2) == 1) +// assert(Fibonacci.fibonacci(3) == 2) +// assert(Fibonacci.fibonacci(4) == 3) +// assert(Fibonacci.fibonacci(5) == 5) +// assert(Fibonacci.fibonacci(6) == 8) +// assert(Fibonacci.fibonacci(7) == 13) +// assert(Fibonacci.fibonacci(75) == 2111485077978050L) +// } +// } + +// import cats.{Id, catsInstancesForId} +// import org.scalatest.flatspec.AnyFlatSpec + +// import scala.collection.mutable.ArrayBuffer +// import scala.concurrent.duration._ + +// class PackageObjectSpec extends AnyFlatSpec { +// type StringOr[A] = Either[String, A] + +// implicit val sleepForEither: Sleep[StringOr] = _ => Right(()) + +// behavior of "retryingOnFailures" + +// it should "retry until the action succeeds" in new TestContext { +// val policy = RetryPolicies.constantDelay[Id](1.second) + +// val sleeps = ArrayBuffer.empty[FiniteDuration] + +// implicit val dummySleep: Sleep[Id] = +// (delay: FiniteDuration) => sleeps.append(delay) + +// val finalResult = retryingOnFailures[String][Id]( +// policy, +// _.toInt > 3, +// onError +// ) { +// attempts = attempts + 1 +// attempts.toString +// } + +// assert(finalResult == "4") +// assert(attempts == 4) +// assert(errors.toList == List("1", "2", "3")) +// assert(delays.toList == List(1.second, 1.second, 1.second)) +// assert(sleeps.toList == delays.toList) +// assert(!gaveUp) +// } + +// it should "retry until the policy chooses to give up" in new TestContext { +// val policy = RetryPolicies.limitRetries[Id](2) + +// implicit val dummySleep: Sleep[Id] = _ => () + +// val finalResult = retryingOnFailures[String][Id]( +// policy, +// _.toInt > 3, +// onError +// ) { +// attempts = attempts + 1 +// attempts.toString +// } + +// assert(finalResult == "3") +// assert(attempts == 3) +// assert(errors.toList == List("1", "2", "3")) +// assert(delays.toList == List(Duration.Zero, Duration.Zero)) +// assert(gaveUp) +// } + +// it should "retry in a stack-safe way" in new TestContext { +// val policy = RetryPolicies.limitRetries[Id](10000) + +// implicit val dummySleep: Sleep[Id] = _ => () + +// val finalResult = retryingOnFailures[String][Id]( +// policy, +// _.toInt > 20000, +// onError +// ) { +// attempts = attempts + 1 +// attempts.toString +// } + +// assert(finalResult == "10001") +// assert(attempts == 10001) +// assert(gaveUp) +// } + +// behavior of "retryingOnSomeErrors" + +// it should "retry until the action succeeds" in new TestContext { +// val policy = RetryPolicies.constantDelay[StringOr](1.second) + +// val finalResult = retryingOnSomeErrors( +// policy, +// (s: String) => Right(s == "one more time"), +// onError +// ) { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Right("yay") +// } + +// assert(finalResult == Right("yay")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert(!gaveUp) +// } + +// it should "retry only if the error is worth retrying" in new TestContext { +// val policy = RetryPolicies.constantDelay[StringOr](1.second) + +// val finalResult = retryingOnSomeErrors( +// policy, +// (s: String) => Right(s == "one more time"), +// onError +// ) { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Left("nope") +// } + +// assert(finalResult == Left("nope")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert( +// !gaveUp +// ) // false because onError is only called when the error is worth retrying +// } + +// it should "retry until the policy chooses to give up" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](2) + +// val finalResult = retryingOnSomeErrors( +// policy, +// (s: String) => Right(s == "one more time"), +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 3) +// assert( +// errors.toList == List("one more time", "one more time", "one more time") +// ) +// assert(gaveUp) +// } + +// it should "retry in a stack-safe way" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](10000) + +// val finalResult = retryingOnSomeErrors( +// policy, +// (s: String) => Right(s == "one more time"), +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 10001) +// assert(gaveUp) +// } + +// behavior of "retryingOnAllErrors" + +// it should "retry until the action succeeds" in new TestContext { +// val policy = RetryPolicies.constantDelay[StringOr](1.second) + +// val finalResult = retryingOnAllErrors( +// policy, +// onError +// ) { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Right("yay") +// } + +// assert(finalResult == Right("yay")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert(!gaveUp) +// } + +// it should "retry until the policy chooses to give up" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](2) + +// val finalResult = retryingOnAllErrors( +// policy, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 3) +// assert( +// errors.toList == List("one more time", "one more time", "one more time") +// ) +// assert(gaveUp) +// } + +// it should "retry in a stack-safe way" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](10000) + +// val finalResult = retryingOnAllErrors( +// policy, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 10001) +// assert(gaveUp) +// } + +// behavior of "retryingOnFailuresAndSomeErrors" + +// it should "retry until the action succeeds" in new TestContext { +// val policy = RetryPolicies.constantDelay[StringOr](1.second) + +// val finalResult = retryingOnFailuresAndSomeErrors[String]( +// policy, +// s => Right(s == "yay"), +// (s: String) => Right(s == "one more time"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Right("yay") +// } + +// assert(finalResult == Right("yay")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert(!gaveUp) +// } + +// it should "retry only if the error is worth retrying" in new TestContext { +// val policy = RetryPolicies.constantDelay[StringOr](1.second) + +// val finalResult = retryingOnFailuresAndSomeErrors[String]( +// policy, +// s => Right(s == "will never happen"), +// (s: String) => Right(s == "one more time"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Left("nope") +// } + +// assert(finalResult == Left("nope")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert( +// !gaveUp +// ) // false because onError is only called when the error is worth retrying +// } + +// it should "retry until the policy chooses to give up due to errors" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](2) + +// val finalResult = retryingOnFailuresAndSomeErrors[String]( +// policy, +// s => Right(s == "will never happen"), +// (s: String) => Right(s == "one more time"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 3) +// assert( +// errors.toList == List("one more time", "one more time", "one more time") +// ) +// assert(gaveUp) +// } + +// it should "retry until the policy chooses to give up due to failures" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](2) + +// val finalResult = retryingOnFailuresAndSomeErrors[String]( +// policy, +// s => Right(s == "yay"), +// (s: String) => Right(s == "one more time"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Right("boo") +// } + +// assert(finalResult == Right("boo")) +// assert(attempts == 3) +// assert(errors.toList == List("boo", "boo", "boo")) +// assert(gaveUp) +// } + +// it should "retry in a stack-safe way" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](10000) + +// val finalResult = retryingOnFailuresAndSomeErrors[String]( +// policy, +// s => Right(s == "yay"), +// (s: String) => Right(s == "one more time"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 10001) +// assert(gaveUp) +// } + +// it should "should fail fast if isWorthRetrying's effect fails" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](10000) + +// val finalResult = retryingOnFailuresAndSomeErrors[String]( +// policy, +// s => Right(s == "yay, but it doesn't matter"), +// (_: String) => Left("isWorthRetrying failed"): StringOr[Boolean], +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("isWorthRetrying failed")) +// assert(attempts == 1) +// assert(!gaveUp) +// } + +// behavior of "retryingOnFailuresAndAllErrors" + +// it should "retry until the action succeeds" in new TestContext { +// val policy = RetryPolicies.constantDelay[StringOr](1.second) + +// val finalResult = retryingOnFailuresAndAllErrors[String]( +// policy, +// s => Right(s == "yay"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// if (attempts < 3) +// Left("one more time") +// else +// Right("yay") +// } + +// assert(finalResult == Right("yay")) +// assert(attempts == 3) +// assert(errors.toList == List("one more time", "one more time")) +// assert(!gaveUp) +// } + +// it should "retry until the policy chooses to give up due to errors" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](2) + +// val finalResult = retryingOnFailuresAndAllErrors[String]( +// policy, +// s => Right(s == "will never happen"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 3) +// assert( +// errors.toList == List("one more time", "one more time", "one more time") +// ) +// assert(gaveUp) +// } + +// it should "retry until the policy chooses to give up due to failures" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](2) + +// val finalResult = retryingOnFailuresAndAllErrors[String]( +// policy, +// s => Right(s == "yay"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Right("boo") +// } + +// assert(finalResult == Right("boo")) +// assert(attempts == 3) +// assert(errors.toList == List("boo", "boo", "boo")) +// assert(gaveUp) +// } + +// it should "retry in a stack-safe way" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](10000) + +// val finalResult = retryingOnFailuresAndAllErrors[String]( +// policy, +// s => Right(s == "will never happen"), +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Left("one more time") +// } + +// assert(finalResult == Left("one more time")) +// assert(attempts == 10001) +// assert(gaveUp) +// } + +// it should "should fail fast if wasSuccessful's effect fails" in new TestContext { +// val policy = RetryPolicies.limitRetries[StringOr](10000) + +// val finalResult = retryingOnFailuresAndAllErrors[String]( +// policy, +// _ => Left("an error was raised!"): StringOr[Boolean], +// onError, +// onError +// ) { +// attempts = attempts + 1 +// Right("one more time") +// } + +// assert(finalResult == Left("an error was raised!")) +// assert(attempts == 1) +// assert(!gaveUp) +// } + +// private class TestContext { +// var attempts = 0 +// val errors = ArrayBuffer.empty[String] +// val delays = ArrayBuffer.empty[FiniteDuration] +// var gaveUp = false + +// def onError(error: String, details: RetryDetails): Either[String, Unit] = { +// errors.append(error) +// details match { +// case RetryDetails.WillDelayAndRetry(delay, _, _) => delays.append(delay) +// case RetryDetails.GivingUp(_, _) => gaveUp = true +// } +// Right(()) +// } +// } +// } + +// import java.util.concurrent.TimeUnit + +// import retry.RetryPolicies._ +// import cats.{Id, catsInstancesForId} +// import org.scalacheck.{Arbitrary, Gen} +// import org.scalatest.flatspec.AnyFlatSpec +// import org.scalatestplus.scalacheck.Checkers +// import retry.PolicyDecision.{DelayAndRetry, GiveUp} + +// import scala.concurrent.duration._ + +// class RetryPoliciesSpec extends AnyFlatSpec with Checkers { +// override implicit val generatorDrivenConfig: PropertyCheckConfiguration = +// PropertyCheckConfiguration(minSuccessful = 100) + +// implicit val arbRetryStatus: Arbitrary[RetryStatus] = Arbitrary { +// for { +// a <- Gen.choose(0, 1000) +// b <- Gen.choose(0, 1000) +// c <- Gen.option(Gen.choose(b, 10000)) +// } yield RetryStatus( +// a, +// FiniteDuration(b, TimeUnit.MILLISECONDS), +// c.map(FiniteDuration(_, TimeUnit.MILLISECONDS)) +// ) +// } + +// val genFiniteDuration: Gen[FiniteDuration] = +// Gen.posNum[Long].map(FiniteDuration(_, TimeUnit.NANOSECONDS)) + +// case class LabelledRetryPolicy(policy: RetryPolicy[Id], description: String) { +// override def toString: String = description +// } + +// implicit val arbRetryPolicy: Arbitrary[LabelledRetryPolicy] = Arbitrary { +// Gen.oneOf( +// Gen.const(LabelledRetryPolicy(alwaysGiveUp[Id], "alwaysGiveUp")), +// genFiniteDuration.map(delay => +// LabelledRetryPolicy( +// constantDelay[Id](delay), +// s"constantDelay($delay)" +// ) +// ), +// genFiniteDuration.map(baseDelay => +// LabelledRetryPolicy( +// exponentialBackoff[Id](baseDelay), +// s"exponentialBackoff($baseDelay)" +// ) +// ), +// Gen +// .posNum[Int] +// .map(maxRetries => +// LabelledRetryPolicy( +// limitRetries(maxRetries), +// s"limitRetries($maxRetries)" +// ) +// ), +// genFiniteDuration.map(baseDelay => +// LabelledRetryPolicy( +// fibonacciBackoff[Id](baseDelay), +// s"fibonacciBackoff($baseDelay)" +// ) +// ), +// genFiniteDuration.map(baseDelay => +// LabelledRetryPolicy( +// fullJitter[Id](baseDelay), +// s"fullJitter($baseDelay)" +// ) +// ) +// ) +// } + +// behavior of "constantDelay" + +// it should "always retry with the same delay" in check { +// (status: RetryStatus) => +// constantDelay[Id](1.second) +// .decideNextRetry(status) == PolicyDecision.DelayAndRetry(1.second) +// } + +// behavior of "exponentialBackoff" + +// it should "start with the base delay and double the delay after each iteration" in { +// val policy = exponentialBackoff[Id](100.milliseconds) +// val arbitraryCumulativeDelay = 999.milliseconds +// val arbitraryPreviousDelay = Some(999.milliseconds) + +// def test(retriesSoFar: Int, expectedDelay: FiniteDuration) = { +// val status = RetryStatus( +// retriesSoFar, +// arbitraryCumulativeDelay, +// arbitraryPreviousDelay +// ) +// val verdict = policy.decideNextRetry(status) +// assert(verdict == PolicyDecision.DelayAndRetry(expectedDelay)) +// } + +// test(0, 100.milliseconds) +// test(1, 200.milliseconds) +// test(2, 400.milliseconds) +// test(3, 800.milliseconds) +// } + +// behavior of "fibonacciBackoff" + +// it should "start with the base delay and increase the delay in a Fibonacci-y way" in { +// val policy = fibonacciBackoff[Id](100.milliseconds) +// val arbitraryCumulativeDelay = 999.milliseconds +// val arbitraryPreviousDelay = Some(999.milliseconds) + +// def test(retriesSoFar: Int, expectedDelay: FiniteDuration) = { +// val status = RetryStatus( +// retriesSoFar, +// arbitraryCumulativeDelay, +// arbitraryPreviousDelay +// ) +// val verdict = policy.decideNextRetry(status) +// assert(verdict == PolicyDecision.DelayAndRetry(expectedDelay)) +// } + +// test(0, 100.milliseconds) +// test(1, 100.milliseconds) +// test(2, 200.milliseconds) +// test(3, 300.milliseconds) +// test(4, 500.milliseconds) +// test(5, 800.milliseconds) +// test(6, 1300.milliseconds) +// test(7, 2100.milliseconds) +// } + +// behavior of "fullJitter" + +// it should "implement the AWS Full Jitter backoff algorithm" in { +// val policy = fullJitter[Id](100.milliseconds) +// val arbitraryCumulativeDelay = 999.milliseconds +// val arbitraryPreviousDelay = Some(999.milliseconds) + +// def test(retriesSoFar: Int, expectedMaximumDelay: FiniteDuration): Unit = { +// val status = RetryStatus( +// retriesSoFar, +// arbitraryCumulativeDelay, +// arbitraryPreviousDelay +// ) +// for (_ <- 1 to 1000) { +// val verdict = policy.decideNextRetry(status) +// val delay = verdict.asInstanceOf[PolicyDecision.DelayAndRetry].delay +// assert(delay >= Duration.Zero) +// assert(delay < expectedMaximumDelay) +// } +// } + +// test(0, 100.milliseconds) +// test(1, 200.milliseconds) +// test(2, 400.milliseconds) +// test(3, 800.milliseconds) +// test(4, 1600.milliseconds) +// test(5, 3200.milliseconds) +// } + +// behavior of "all built-in policies" + +// it should "never try to create a FiniteDuration of more than Long.MaxValue nanoseconds" in check { +// (labelledPolicy: LabelledRetryPolicy, status: RetryStatus) => +// labelledPolicy.policy.decideNextRetry(status) match { +// case PolicyDecision.DelayAndRetry(nextDelay) => +// nextDelay.toNanos <= Long.MaxValue +// case PolicyDecision.GiveUp => true +// } +// } + +// behavior of "limitRetries" + +// it should "retry with no delay until the limit is reached" in check { +// (status: RetryStatus) => +// val limit = 500 +// val verdict = +// limitRetries[Id](limit).decideNextRetry(status) +// if (status.retriesSoFar < limit) { +// verdict == PolicyDecision.DelayAndRetry(Duration.Zero) +// } else { +// verdict == PolicyDecision.GiveUp +// } +// } + +// behavior of "capDelay" + +// it should "cap the delay" in { +// check { (status: RetryStatus) => +// capDelay(100.milliseconds, constantDelay[Id](101.milliseconds)) +// .decideNextRetry(status) == DelayAndRetry(100.milliseconds) +// } + +// check { (status: RetryStatus) => +// capDelay(100.milliseconds, constantDelay[Id](99.milliseconds)) +// .decideNextRetry(status) == DelayAndRetry(99.milliseconds) +// } +// } + +// behavior of "limitRetriesByDelay" + +// it should "give up if the underlying policy chooses a delay greater than the threshold" in { +// check { (status: RetryStatus) => +// limitRetriesByDelay(100.milliseconds, constantDelay[Id](101.milliseconds)) +// .decideNextRetry(status) == GiveUp +// } + +// check { (status: RetryStatus) => +// limitRetriesByDelay(100.milliseconds, constantDelay[Id](99.milliseconds)) +// .decideNextRetry(status) == DelayAndRetry(99.milliseconds) +// } +// } + +// behavior of "limitRetriesByCumulativeDelay" + +// it should "give up if cumulativeDelay + underlying policy's next delay >= threshold" in { +// val cumulativeDelay = 400.milliseconds +// val arbitraryRetriesSoFar = 5 +// val arbitraryPreviousDelay = Some(123.milliseconds) +// val status = RetryStatus( +// arbitraryRetriesSoFar, +// cumulativeDelay, +// arbitraryPreviousDelay +// ) + +// val threshold = 500.milliseconds + +// def test( +// underlyingPolicy: RetryPolicy[Id], +// expectedDecision: PolicyDecision +// ) = { +// val policy = limitRetriesByCumulativeDelay(threshold, underlyingPolicy) +// assert(policy.decideNextRetry(status) == expectedDecision) +// } + +// test(constantDelay(98.milliseconds), DelayAndRetry(98.milliseconds)) +// test(constantDelay(99.milliseconds), DelayAndRetry(99.milliseconds)) +// test(constantDelay(100.milliseconds), GiveUp) +// test(constantDelay(101.milliseconds), GiveUp) +// } +// } + +// import cats.{Id, catsInstancesForId} +// import cats.syntax.semigroup._ +// import org.scalatest.flatspec.AnyFlatSpec + +// import scala.concurrent.duration._ + +// class RetryPolicySpec extends AnyFlatSpec { +// behavior of "BoundedSemilattice append" + +// it should "give up if either of the composed policies decides to give up" in { +// val alwaysGiveUp = RetryPolicy.lift[Id](_ => PolicyDecision.GiveUp) +// val alwaysRetry = RetryPolicies.constantDelay[Id](1.second) + +// assert( +// (alwaysGiveUp |+| alwaysRetry) +// .decideNextRetry(RetryStatus.NoRetriesYet) == PolicyDecision.GiveUp +// ) +// assert( +// (alwaysRetry |+| alwaysGiveUp) +// .decideNextRetry(RetryStatus.NoRetriesYet) == PolicyDecision.GiveUp +// ) +// } + +// it should "choose the maximum of the delays if both of the composed policies decides to retry" in { +// val delayOneSecond = +// RetryPolicy.lift[Id](_ => PolicyDecision.DelayAndRetry(1.second)) +// val delayTwoSeconds = +// RetryPolicy.lift[Id](_ => PolicyDecision.DelayAndRetry(2.seconds)) + +// assert( +// (delayOneSecond |+| delayTwoSeconds).decideNextRetry( +// RetryStatus.NoRetriesYet +// ) == PolicyDecision.DelayAndRetry(2.seconds) +// ) +// assert( +// (delayTwoSeconds |+| delayOneSecond).decideNextRetry( +// RetryStatus.NoRetriesYet +// ) == PolicyDecision.DelayAndRetry(2.seconds) +// ) +// } +// }