add retry logic

Author: jsbroks

apps/workspace-engine/oapi/openapi.json

@@ -837,6 +837,9 @@
                },
                "policyId": {
                   "type": "string"
+               },
+               "retry": {
+                  "$ref": "#/components/schemas/RetryRule"
                }
             },
             "required": [
@@ -1301,6 +1304,48 @@
             ],
             "type": "object"
          },
+         "RetryRule": {
+            "properties": {
+               "backoffSeconds": {
+                  "description": "Minimum seconds to wait between retry attempts. If null, retries are allowed immediately after job completion.",
+                  "format": "int32",
+                  "minimum": 0,
+                  "type": "integer"
+               },
+               "backoffStrategy": {
+                  "default": "linear",
+                  "description": "Backoff strategy: \"linear\" uses constant backoffSeconds delay, \"exponential\" doubles the delay with each retry (backoffSeconds * 2^(attempt-1)).",
+                  "enum": [
+                     "linear",
+                     "exponential"
+                  ],
+                  "type": "string"
+               },
+               "maxBackoffSeconds": {
+                  "description": "Maximum backoff time in seconds (cap for exponential backoff). If null, no maximum is enforced.",
+                  "format": "int32",
+                  "minimum": 0,
+                  "type": "integer"
+               },
+               "maxRetries": {
+                  "description": "Maximum number of retries allowed. 0 means no retries (1 attempt total), 3 means up to 4 attempts (1 initial + 3 retries).",
+                  "format": "int32",
+                  "minimum": 0,
+                  "type": "integer"
+               },
+               "retryOnStatuses": {
+                  "description": "Job statuses that count toward the retry limit. If null or empty and maxRetries > 0, defaults to [\"failure\", \"invalidIntegration\"] (smart default: only retry on errors, not on success). If maxRetries = 0, counts all statuses (strict: no retries at all). Example: [\"failure\", \"cancelled\"] will only count failed/cancelled jobs.",
+                  "items": {
+                     "$ref": "#/components/schemas/JobStatus"
+                  },
+                  "type": "array"
+               }
+            },
+            "required": [
+               "maxRetries"
+            ],
+            "type": "object"
+         },
          "RuleEvaluation": {
             "properties": {
                "actionRequired": {

apps/workspace-engine/oapi/spec/schemas/policy.jsonnet

@@ -59,6 +59,43 @@ local openapi = import '../lib/openapi.libsonnet';
       anyApproval: openapi.schemaRef('AnyApprovalRule'),
       environmentProgression: openapi.schemaRef('EnvironmentProgressionRule'),
       gradualRollout: openapi.schemaRef('GradualRolloutRule'),
+      retry: openapi.schemaRef('RetryRule'),
+    },
+  },
+
+  RetryRule: {
+    type: 'object',
+    required: ['maxRetries'],
+    properties: {
+      maxRetries: {
+        type: 'integer',
+        format: 'int32',
+        minimum: 0,
+        description: 'Maximum number of retries allowed. 0 means no retries (1 attempt total), 3 means up to 4 attempts (1 initial + 3 retries).',
+      },
+      retryOnStatuses: {
+        type: 'array',
+        items: openapi.schemaRef('JobStatus'),
+        description: 'Job statuses that count toward the retry limit. If null or empty, defaults to ["failure", "invalidIntegration", "invalidJobAgent"] for maxRetries > 0, or ["failure", "invalidIntegration", "invalidJobAgent", "successful"] for maxRetries = 0. Cancelled and skipped jobs never count by default (allows redeployment after cancellation). Example: ["failure", "cancelled"] will only count failed/cancelled jobs.',
+      },
+      backoffSeconds: {
+        type: 'integer',
+        format: 'int32',
+        minimum: 0,
+        description: 'Minimum seconds to wait between retry attempts. If null, retries are allowed immediately after job completion.',
+      },
+      backoffStrategy: {
+        type: 'string',
+        enum: ['linear', 'exponential'],
+        default: 'linear',
+        description: 'Backoff strategy: "linear" uses constant backoffSeconds delay, "exponential" doubles the delay with each retry (backoffSeconds * 2^(attempt-1)).',
+      },
+      maxBackoffSeconds: {
+        type: 'integer',
+        format: 'int32',
+        minimum: 0,
+        description: 'Maximum backoff time in seconds (cap for exponential backoff). If null, no maximum is enforced.',
+      },
     },
   },
 

apps/workspace-engine/pkg/events/handler/deployment/deployment.go

@@ -266,7 +266,7 @@ func getJobsToRetrigger(ws *workspace.Workspace, deployment *oapi.Deployment) []
 
 // retriggerInvalidJobAgentJobs creates new Pending jobs for all releases that currently have InvalidJobAgent jobs
 // Note: This is an explicit retrigger operation for configuration fixes, so we bypass normal
-// eligibility checks (like skipdeployed). The old InvalidJobAgent job remains for history.
+// eligibility checks (like retry limits). The old InvalidJobAgent job remains for history.
 func retriggerInvalidJobAgentJobs(ctx context.Context, ws *workspace.Workspace, jobsToRetrigger []*oapi.Job) {
 	// Create job factory and dispatcher
 	jobFactory := jobs.NewFactory(ws.Store())

apps/workspace-engine/pkg/oapi/oapi.gen.go

@@ -0,0 +1,56 @@
+package deployment
+
+import (
+	"time"
+)
+
+// EligibilityDecision represents whether a job should be created.
+type EligibilityDecision string
+
+const (
+	// EligibilityAllowed indicates the job should be created immediately
+	EligibilityAllowed EligibilityDecision = "allowed"
+
+	// EligibilityDenied indicates the job should not be created (retry limit exceeded, etc.)
+	EligibilityDenied EligibilityDecision = "denied"
+
+	// EligibilityPending indicates the job creation is delayed (e.g., waiting for backoff)
+	// The system should schedule re-evaluation at NextEvaluationTime
+	EligibilityPending EligibilityDecision = "pending"
+)
+
+// EligibilityResult contains the result of a job eligibility check.
+type EligibilityResult struct {
+	// Decision indicates whether the job should be created
+	Decision EligibilityDecision
+
+	// Reason provides a human-readable explanation for the decision
+	Reason string
+
+	// NextEvaluationTime is when the eligibility should be re-evaluated (for pending decisions)
+	// This is used to schedule future reconciliation
+	NextEvaluationTime *time.Time
+
+	// Details contains additional structured information about the decision
+	Details map[string]interface{}
+}
+
+// IsAllowed returns true if the job can be created immediately.
+func (r *EligibilityResult) IsAllowed() bool {
+	return r.Decision == EligibilityAllowed
+}
+
+// IsDenied returns true if the job should not be created.
+func (r *EligibilityResult) IsDenied() bool {
+	return r.Decision == EligibilityDenied
+}
+
+// IsPending returns true if the job creation is delayed pending some condition.
+func (r *EligibilityResult) IsPending() bool {
+	return r.Decision == EligibilityPending
+}
+
+// ShouldScheduleRetry returns true if there's a future time when this should be re-evaluated.
+func (r *EligibilityResult) ShouldScheduleRetry() bool {
+	return r.IsPending() && r.NextEvaluationTime != nil
+}