fix(scheduler/model-gw): failed pipelines never retried

k8s/helm-charts/seldon-core-v2-setup/templates/_components-deployments.tpl

@@ -526,6 +526,8 @@ spec:
         - --pprof-port=$(PPROF_PORT)
         - --pprof-block-rate=$(PPROF_BLOCK_RATE)
         - --pprof-mutex-rate=$(PPROF_MUTEX_RATE)
+        - --retry-creating-failed-pipelines-tick=$(RETRY_CREATING_FAILED_PIPELINES_TICK)
+        - --retry-deleting-failed-pipelines-tick=$(RETRY_DELETING_FAILED_PIPELINES_TICK)
         command:
         - /bin/scheduler
         env:
@@ -611,6 +613,10 @@ spec:
           value: "0"
         - name: PPROF_MUTEX_RATE
           value: "0"
+        - name: RETRY_CREATING_FAILED_PIPELINES_TICK
+          value: 60s
+        - name: RETRY_DELETING_FAILED_PIPELINES_TICK
+          value: 60s
         image: '{{ .Values.scheduler.image.registry }}/{{ .Values.scheduler.image.repository
           }}:{{ .Values.scheduler.image.tag }}'
         imagePullPolicy: '{{ .Values.scheduler.image.pullPolicy }}'

k8s/helm-charts/seldon-core-v2-setup/templates/_components-statefulsets.tpl

@@ -526,6 +526,8 @@ spec:
         - --pprof-port=$(PPROF_PORT)
         - --pprof-block-rate=$(PPROF_BLOCK_RATE)
         - --pprof-mutex-rate=$(PPROF_MUTEX_RATE)
+        - --retry-creating-failed-pipelines-tick=$(RETRY_CREATING_FAILED_PIPELINES_TICK)
+        - --retry-deleting-failed-pipelines-tick=$(RETRY_DELETING_FAILED_PIPELINES_TICK)
         command:
         - /bin/scheduler
         env:
@@ -611,6 +613,10 @@ spec:
           value: "0"
         - name: PPROF_MUTEX_RATE
           value: "0"
+        - name: RETRY_CREATING_FAILED_PIPELINES_TICK
+          value: 60s
+        - name: RETRY_DELETING_FAILED_PIPELINES_TICK
+          value: 60s
         image: '{{ .Values.scheduler.image.registry }}/{{ .Values.scheduler.image.repository
           }}:{{ .Values.scheduler.image.tag }}'
         imagePullPolicy: '{{ .Values.scheduler.image.pullPolicy }}'

k8s/yaml/components.yaml

@@ -373,6 +373,8 @@ spec:
         - --pprof-port=$(PPROF_PORT)
         - --pprof-block-rate=$(PPROF_BLOCK_RATE)
         - --pprof-mutex-rate=$(PPROF_MUTEX_RATE)
+        - --retry-creating-failed-pipelines-tick=$(RETRY_CREATING_FAILED_PIPELINES_TICK)
+        - --retry-deleting-failed-pipelines-tick=$(RETRY_DELETING_FAILED_PIPELINES_TICK)
         command:
         - /bin/scheduler
         env:
@@ -454,6 +456,10 @@ spec:
           value: "0"
         - name: PPROF_MUTEX_RATE
           value: "0"
+        - name: RETRY_CREATING_FAILED_PIPELINES_TICK
+          value: 60s
+        - name: RETRY_DELETING_FAILED_PIPELINES_TICK
+          value: 60s
         image: 'docker.io/seldonio/seldon-scheduler:latest'
         imagePullPolicy: 'IfNotPresent'
         livenessProbe:

operator/config/seldonconfigs/default.yaml

@@ -347,6 +347,8 @@ spec:
         - --pprof-port=$(PPROF_PORT)
         - --pprof-block-rate=$(PPROF_BLOCK_RATE)
         - --pprof-mutex-rate=$(PPROF_MUTEX_RATE)
+        - --retry-creating-failed-pipelines-tick=$(RETRY_CREATING_FAILED_PIPELINES_TICK)
+        - --retry-deleting-failed-pipelines-tick=$(RETRY_DELETING_FAILED_PIPELINES_TICK)
         command:
         - /bin/scheduler
         env:
@@ -386,6 +388,10 @@ spec:
           value: "0"
         - name: PPROF_MUTEX_RATE
           value: "0"
+        - name: RETRY_CREATING_FAILED_PIPELINES_TICK
+          value: "60s"
+        - name: RETRY_DELETING_FAILED_PIPELINES_TICK
+          value: "60s"
         image: seldonio/seldon-scheduler:latest
         imagePullPolicy: Always
         name: scheduler

scheduler/cmd/scheduler/main.go

@@ -51,37 +51,39 @@ import (
 )
 
 var (
-	envoyPort                    uint
-	agentPort                    uint
-	agentMtlsPort                uint
-	schedulerPort                uint
-	schedulerMtlsPort            uint
-	chainerPort                  uint
-	healthProbePort              uint
-	namespace                    string
-	pipelineGatewayHost          string
-	pipelineGatewayHttpPort      int
-	pipelineGatewayGrpcPort      int
-	logLevel                     string
-	tracingConfigPath            string
-	dbPath                       string
-	nodeID                       string
-	allowPlaintxt                bool // scheduler server
-	autoscalingModelEnabled      bool
-	autoscalingServerEnabled     bool
-	kafkaConfigPath              string
-	scalingConfigPath            string
-	schedulerReadyTimeoutSeconds uint
-	deletedResourceTTLSeconds    uint
-	serverPackingEnabled         bool
-	serverPackingPercentage      float64
-	accessLogPath                string
-	enableAccessLog              bool
-	includeSuccessfulRequests    bool
-	enablePprof                  bool
-	pprofPort                    int
-	pprofMutexRate               int
-	pprofBlockRate               int
+	envoyPort                        uint
+	agentPort                        uint
+	agentMtlsPort                    uint
+	schedulerPort                    uint
+	schedulerMtlsPort                uint
+	chainerPort                      uint
+	healthProbePort                  uint
+	namespace                        string
+	pipelineGatewayHost              string
+	pipelineGatewayHttpPort          int
+	pipelineGatewayGrpcPort          int
+	logLevel                         string
+	tracingConfigPath                string
+	dbPath                           string
+	nodeID                           string
+	allowPlaintxt                    bool // scheduler server
+	autoscalingModelEnabled          bool
+	autoscalingServerEnabled         bool
+	kafkaConfigPath                  string
+	scalingConfigPath                string
+	schedulerReadyTimeoutSeconds     uint
+	deletedResourceTTLSeconds        uint
+	serverPackingEnabled             bool
+	serverPackingPercentage          float64
+	accessLogPath                    string
+	enableAccessLog                  bool
+	includeSuccessfulRequests        bool
+	enablePprof                      bool
+	pprofPort                        int
+	pprofMutexRate                   int
+	pprofBlockRate                   int
+	retryFailedCreatingPipelinesTick time.Duration
+	retryFailedDeletePipelinesTick   time.Duration
 )
 
 const (
@@ -172,6 +174,10 @@ func init() {
 	flag.IntVar(&pprofPort, "pprof-port", 6060, "pprof HTTP server port")
 	flag.IntVar(&pprofBlockRate, "pprof-block-rate", 0, "pprof block rate")
 	flag.IntVar(&pprofMutexRate, "pprof-mutex-rate", 0, "pprof mutex rate")
+
+	// frequency to retry creating/deleting pipelines which failed to create/delete
+	flag.DurationVar(&retryFailedCreatingPipelinesTick, "retry-creating-failed-pipelines-tick", time.Minute, "tick interval for re-attempting to create pipelines which failed to create")
+	flag.DurationVar(&retryFailedDeletePipelinesTick, "retry-deleting-failed-pipelines-tick", time.Minute, "tick interval for re-attempting to delete pipelines which failed to terminate")
 }
 
 func getNamespace() string {
@@ -322,8 +328,11 @@ func main() {
 		logger.WithError(err).Fatal("Failed to start data engine chainer server")
 	}
 	defer cs.Stop()
+
+	ctx, stopPipelinePollers := context.WithCancel(context.Background())
+	defer stopPipelinePollers()
 	go func() {
-		err := cs.StartGrpcServer(chainerPort)
+		err := cs.StartGrpcServer(ctx, retryFailedCreatingPipelinesTick, retryFailedDeletePipelinesTick, chainerPort)
 		if err != nil {
 			log.WithError(err).Fatalf("Chainer server start error")
 		}
@@ -382,7 +391,7 @@ func main() {
 	)
 	defer s.Stop()
 
-	err = s.StartGrpcServers(allowPlaintxt, schedulerPort, schedulerMtlsPort)
+	err = s.StartGrpcServers(ctx, allowPlaintxt, schedulerPort, schedulerMtlsPort, retryFailedCreatingPipelinesTick, retryFailedDeletePipelinesTick)
 	if err != nil {
 		logger.WithError(err).Fatal("Failed to start server gRPC servers")
 	}
@@ -421,6 +430,7 @@ func main() {
 	s.StopSendServerEvents()
 	s.StopSendExperimentEvents()
 	s.StopSendPipelineEvents()
+	stopPipelinePollers()
 	s.StopSendControlPlaneEvents()
 	as.StopAgentStreams()
 

scheduler/go.mod

@@ -148,7 +148,10 @@ require (
 	sigs.k8s.io/yaml v1.5.0 // indirect
 )
 
-tool go.uber.org/mock/mockgen
+tool (
+	go.uber.org/mock/mockgen
+	golang.org/x/tools/cmd/stringer
+)
 
 replace github.com/seldonio/seldon-core/components/tls/v2 => ../components/tls
 

scheduler/pkg/kafka/conflict-resolution/conflict_resolution.go

@@ -146,6 +146,11 @@ func GetPipelineStatus(
 		messageStr += fmt.Sprintf("%d/%d failed ", failedCount, len(streams))
 	}
 
+	failedTerminatingCount := cr.GetCountResourceWithStatus(pipelineName, pipeline.PipelineFailedTerminating)
+	if failedTerminatingCount > 0 {
+		messageStr += fmt.Sprintf("%d/%d failed ", failedTerminatingCount, len(streams))
+	}
+
 	rebalancingCount := cr.GetCountResourceWithStatus(pipelineName, pipeline.PipelineRebalancing)
 	if rebalancingCount > 0 {
 		messageStr += fmt.Sprintf("%d/%d rebalancing ", rebalancingCount, len(streams))
@@ -170,8 +175,8 @@ func GetPipelineStatus(
 	}
 
 	if message.Update.Op == chainer.PipelineUpdateMessage_Delete {
-		if failedCount > 0 {
-			return pipeline.PipelineFailed, messageStr
+		if failedTerminatingCount > 0 {
+			return pipeline.PipelineFailedTerminating, messageStr
 		}
 		if terminatedCount == len(streams) {
 			return pipeline.PipelineTerminated, messageStr

scheduler/pkg/kafka/conflict-resolution/conflict_resolution_test.go

@@ -205,9 +205,9 @@ func TestGetPipelineStatus(t *testing.T) {
 			name: "delete failed",
 			op:   chainer.PipelineUpdateMessage_Delete,
 			statuses: map[string]pipeline.PipelineStatus{
-				"a": pipeline.PipelineFailed,
+				"a": pipeline.PipelineFailedTerminating,
 			},
-			expected: pipeline.PipelineFailed,
+			expected: pipeline.PipelineFailedTerminating,
 		},
 		{
 			name: "rebalance failed",