[release-v0.65.x] adding efficient polling to waitForStepsToFinish

config/config-defaults.yaml

@@ -144,3 +144,11 @@ data:
     #     limits:
     #       memory: "256Mi"
     #       cpu: "500m"
+
+    # default-sidecar-log-polling-interval specifies the polling interval for the Tekton sidecar log results container.
+    # This controls how frequently the sidecar checks for step completion files written by steps in a TaskRun.
+    # Lower values (e.g., "10ms") make the sidecar more responsive but may increase CPU usage; higher values (e.g., "1s")
+    # reduce resource usage but may delay result collection.
+    # This value is used by the sidecar-tekton-log-results container and can be tuned for performance or test scenarios.
+    # Example values: "100ms", "500ms", "1s"
+    default-sidecar-log-polling-interval: "100ms"

docs/additional-configs.md

@@ -243,6 +243,7 @@ The example below customizes the following:
 - the default maximum combinations of `Parameters` in a `Matrix` that can be used to fan out a `PipelineTask`. For
 more information, see [`Matrix`](matrix.md).
 - the default resolver type to `git`.
+- the default polling interval for the sidecar log results container via `default-sidecar-log-polling-interval`.
 
 ```yaml
 apiVersion: v1
@@ -260,8 +261,26 @@ data:
     emptyDir: {}
   default-max-matrix-combinations-count: "1024"
   default-resolver-type: "git"
+  default-sidecar-log-polling-interval: "100ms"
 ```
 
+### `default-sidecar-log-polling-interval`
+
+The `default-sidecar-log-polling-interval` key in the `config-defaults` ConfigMap specifies how frequently the Tekton
+sidecar log results container polls for step completion files written by steps in a TaskRun. Lower values (e.g., `10ms`)
+make the sidecar more responsive but may increase CPU usage; higher values (e.g., `1s`) reduce resource usage but may
+delay result collection. This value is used by the `sidecar-tekton-log-results` container and can be tuned for performance
+or test scenarios.
+
+**Example values:**
+- `100ms` (default)
+- `500ms`
+- `1s`
+- `10ms` (for fast polling in tests)
+
+**Note:** The `default-sidecar-log-polling-interval` setting is only applicable when results are created using the
+[sidecar approach](#enabling-larger-results-using-sidecar-logs).
+
 **Note:** The `_example` key in the provided [config-defaults.yaml](./../config/config-defaults.yaml)
 file lists the keys you can customize along with their default values.
 

examples/v1/pipelineruns/beta/git-resolver.yaml

@@ -25,7 +25,7 @@ spec:
             - name: url
               value: https://github.com/tektoncd/catalog.git
             - name: pathInRepo
-              value: /task/git-clone/0.7/git-clone.yaml
+              value: /task/git-clone/0.10/git-clone.yaml
             - name: revision
               value: main
         params:

examples/v1/pipelineruns/beta/http-resolver-credentials.yaml

@@ -16,20 +16,28 @@ kind: PipelineRun
 metadata:
   generateName: http-resolver-
 spec:
+  workspaces:
+  - name: output
+    emptyDir: {}
   pipelineSpec:
+    workspaces:
+    - name: output
     tasks:
       - name: http-resolver
         taskRef:
           resolver: http
           params:
             - name: url
-              value: https://api.hub.tekton.dev/v1/resource/tekton/task/tkn/0.4/raw
+              value: https://api.hub.tekton.dev/v1/resource/tekton/task/git-clone/0.10/raw
             - name: http-username
               value: git
             - name: http-password-secret
               value: my-secret
             - name: http-password-secret-key
               value: token
         params:
-          - name: ARGS
-            value: ["version"]
+          - name: url
+            value: "https://github.com/kelseyhightower/nocode"
+        workspaces:
+          - name: output
+            workspace: output

examples/v1/pipelineruns/beta/http-resolver.yaml

@@ -4,14 +4,22 @@ kind: PipelineRun
 metadata:
   generateName: http-resolver-
 spec:
+  workspaces:
+  - name: output
+    emptyDir: {}
   pipelineSpec:
+    workspaces:
+    - name: output
     tasks:
       - name: http-resolver
         taskRef:
           resolver: http
           params:
             - name: url
-              value: https://api.hub.tekton.dev/v1/resource/tekton/task/tkn/0.4/raw
+              value: https://api.hub.tekton.dev/v1/resource/tekton/task/git-clone/0.10/raw
         params:
-          - name: ARGS
-            value: ["version"]
+          - name: url
+            value: "https://github.com/kelseyhightower/nocode"
+        workspaces:
+          - name: output
+            workspace: output

examples/v1/pipelineruns/no-ci/git-resolver-custom-apiurl.yaml

@@ -25,7 +25,7 @@ spec:
             - name: url
               value: https://github.com/tektoncd/catalog.git
             - name: pathInRepo
-              value: /task/git-clone/0.7/git-clone.yaml
+              value: /task/git-clone/0.10/git-clone.yaml
             - name: revision
               value: main
             # my-secret-token should be created in the namespace where the

examples/v1/pipelineruns/no-ci/git-resolver-custom-secret.yaml

@@ -25,7 +25,7 @@ spec:
             - name: url
               value: https://github.com/tektoncd/catalog.git
             - name: pathInRepo
-              value: /task/git-clone/0.7/git-clone.yaml
+              value: /task/git-clone/0.10/git-clone.yaml
             - name: revision
               value: main
             # my-secret-token should be created in the namespace where the

examples/v1/pipelineruns/pipelinerun-with-final-tasks.yaml

@@ -57,7 +57,7 @@ spec:
       description: The precise commit SHA that was fetched by this Task
   steps:
     - name: clone
-      image: gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/git-init:latest
+      image: ghcr.io/tektoncd-catalog/git-clone:v1.1.0
       securityContext:
         runAsUser: 0  # This needs root, and git-init is nonroot by default
       script: |
@@ -200,4 +200,4 @@ spec:
           resources:
             requests:
               storage: 1Gi
----
+---

examples/v1/pipelineruns/pipelinerun.yaml

@@ -58,7 +58,7 @@ spec:
     description: The precise commit SHA that was fetched by this Task
   steps:
   - name: clone
-    image: gcr.io/tekton-releases/github.com/tektoncd/pipeline/cmd/git-init:latest
+    image: ghcr.io/tektoncd-catalog/git-clone:v1.1.0
     securityContext:
       runAsUser: 0  # This needs root, and git-init is nonroot by default
     script: |
@@ -223,4 +223,4 @@ spec:
           - ReadWriteOnce
         resources:
           requests:
-            storage: 1Gi
+            storage: 1Gi

examples/v1/taskruns/authenticating-git-commands.yaml

@@ -166,7 +166,7 @@ spec:
         git commit -m "Test commit!"
         git push origin master
     - name: git-clone-and-check
-      image: gcr.io/tekton-releases/dogfooding/alpine-git-nonroot:latest
+      image: ghcr.io/tektoncd/plumbing/alpine-git-nonroot:latest
       # Because this Step runs with a non-root security context, the creds-init
       # credentials will fail to copy into /tekton/home. This happens because
       # our previous step _already_ wrote to /tekton/home and ran as a root

examples/v1/taskruns/beta/authenticating-git-commands.yaml

@@ -161,7 +161,7 @@ spec:
         git commit -m "Test commit!"
         git push origin master
     - name: git-clone-and-check
-      image: gcr.io/tekton-releases/dogfooding/alpine-git-nonroot:latest
+      image: ghcr.io/tektoncd/plumbing/alpine-git-nonroot:latest
       # Because this Step runs with a non-root security context, the creds-init
       # credentials will fail to copy into /tekton/home. This happens because
       # our previous step _already_ wrote to /tekton/home and ran as a root

examples/v1/taskruns/beta/bundles-resolver.yaml

@@ -13,7 +13,7 @@ spec:
     resolver: bundles
     params:
       - name: bundle
-        value: gcr.io/tekton-releases/catalog/upstream/git-clone@sha256:8e2c3fb0f719d6463e950f3e44965aa314e69b800833e29e68ba2616bb82deeb
+        value: ghcr.io/tektoncd/catalog/upstream/tasks/git-clone@sha256:65e61544c5870c8828233406689d812391735fd4100cb444bbd81531cb958bb3 # 0.10 bundle
       - name: name
         value: git-clone
       - name: kind

examples/v1/taskruns/beta/git-resolver.yaml

@@ -22,4 +22,4 @@ spec:
       - name: revision
         value: main
       - name: pathInRepo
-        value: task/git-clone/0.8/git-clone.yaml
+        value: task/git-clone/0.10/git-clone.yaml

examples/v1/taskruns/beta/hub-resolver.yaml

@@ -24,7 +24,7 @@ spec:
       - name: name
         value: git-clone
       - name: version
-        value: "0.6"
+        value: "0.10"
 ---
 apiVersion: tekton.dev/v1
 kind: TaskRun
@@ -48,4 +48,4 @@ spec:
       - name: name
         value: git-clone
       - name: version
-        value: "0.6.0"
+        value: "0.10"

examples/v1/taskruns/entrypoint-resolution.yaml

@@ -8,7 +8,7 @@ spec:
     # Multi-arch image with no command defined. We should look up the command
     # for each platform-specific image and pass it to the Pod, which selects
     # the right command at runtime based on the node's runtime platform.
-    - image: gcr.io/tekton-nightly/github.com/tektoncd/pipeline/cmd/nop
+    - image: ghcr.io/tektoncd/pipeline/nop-8eac7c133edad5df719dc37b36b62482:latest
 
     # Multi-arch image with no command defined, but with args. We'll look
     # up the commands and pass it to the entrypoint binary via env var, then

examples/v1/taskruns/no-ci/docker-creds.yaml

@@ -37,6 +37,6 @@ spec:
   taskSpec:
     steps:
     - name: test
-      image: gcr.io/tekton-releases/dogfooding/skopeo:latest
+      image: ghcr.io/tektoncd/catalog/upstream/tasks/skopeo-copy:latest
       # Test pulling a private builder container.
       script: skopeo copy docker://gcr.io/build-crd-testing/secret-sauce dir:///tmp/

examples/v1/taskruns/no-ci/pull-private-image.yaml

@@ -47,5 +47,5 @@ spec:
     steps:
     - name: pull
       # Private image is just Ubuntu
-      image: gcr.io/tekton-releases/dogfooding/skopeo:latest
+      image: ghcr.io/tektoncd/catalog/upstream/tasks/skopeo-copy:latest
       script: skopeo copy docker://gcr.io/build-crd-testing/secret-sauce dir:///tmp/

internal/sidecarlogresults/sidecarlogresults.go

@@ -26,6 +26,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/tektoncd/pipeline/pkg/apis/config"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
@@ -74,7 +75,7 @@ func encode(w io.Writer, v any) error {
 	return json.NewEncoder(w).Encode(v)
 }
 
-func waitForStepsToFinish(runDir string) error {
+func waitForStepsToFinish(runDir string, sleepInterval time.Duration) error {
 	steps := make(map[string]bool)
 	files, err := os.ReadDir(runDir)
 	if err != nil {
@@ -103,6 +104,9 @@ func waitForStepsToFinish(runDir string) error {
 				return err
 			}
 		}
+		if sleepInterval > 0 {
+			time.Sleep(sleepInterval)
+		}
 	}
 	return nil
 }
@@ -143,7 +147,11 @@ func readResults(resultsDir, resultFile, stepName string, resultType SidecarLogR
 // in their results path and prints them in a structured way to its
 // stdout so that the reconciler can parse those logs.
 func LookForResults(w io.Writer, runDir string, resultsDir string, resultNames []string, stepResultsDir string, stepResults map[string][]string) error {
-	if err := waitForStepsToFinish(runDir); err != nil {
+	interval, err := getSidecarLogPollingInterval()
+	if err != nil {
+		return fmt.Errorf("error getting polling interval: %w", err)
+	}
+	if err := waitForStepsToFinish(runDir, interval); err != nil {
 		return fmt.Errorf("error while waiting for the steps to finish  %w", err)
 	}
 	results := make(chan SidecarLogResult)
@@ -205,7 +213,11 @@ func LookForResults(w io.Writer, runDir string, resultsDir string, resultNames [
 // If the provenance file exists, the function extracts artifact information, formats it into a
 // JSON string, and encodes it for output alongside relevant metadata (step name, artifact type).
 func LookForArtifacts(w io.Writer, names []string, runDir string) error {
-	if err := waitForStepsToFinish(runDir); err != nil {
+	interval, err := getSidecarLogPollingInterval()
+	if err != nil {
+		return fmt.Errorf("error getting polling interval: %w", err)
+	}
+	if err := waitForStepsToFinish(runDir, interval); err != nil {
 		return err
 	}
 
@@ -314,3 +326,18 @@ func extractArtifactsFromFile(filename string) (v1.Artifacts, error) {
 	}
 	return parseArtifacts(b)
 }
+
+// getSidecarLogPollingInterval reads the SIDECAR_LOG_POLLING_INTERVAL environment variable,
+// parses it as a time.Duration, and returns the result. If the variable is not set or is invalid,
+// it defaults to 100ms.
+func getSidecarLogPollingInterval() (time.Duration, error) {
+	intervalStr := os.Getenv("SIDECAR_LOG_POLLING_INTERVAL")
+	if intervalStr == "" {
+		intervalStr = "100ms"
+	}
+	interval, err := time.ParseDuration(intervalStr)
+	if err != nil {
+		return 100 * time.Millisecond, err
+	}
+	return interval, nil
+}