detect if image exists or access is denied

This also adds e2e tests for edge-cases that may have been missed as
well as a test-case that exposes the clusters' internal image registry
(if it exists) and validates the error handling functions against it.

The exposure functions were repackaged from the devex helpers for easy
reuse across both contexts.

Author: cheesesashimi

devex/cmd/mco-builder/local.go

@@ -15,6 +15,7 @@ import (
 	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
+	"github.com/openshift/machine-config-operator/test/helpers"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	aggerrs "k8s.io/apimachinery/pkg/util/errors"
@@ -192,7 +193,7 @@ func buildLocallyAndPushIntoCluster(cs *framework.ClientSet, buildOpts localBuil
 		}
 	}()
 
-	extHostname, err := rollout.ExposeClusterImageRegistry(cs)
+	extHostname, err := helpers.ExposeClusterImageRegistry(context.TODO(), cs)
 	if err != nil {
 		return err
 	}

devex/cmd/mco-builder/revert.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
+	"github.com/openshift/machine-config-operator/test/helpers"
 	"github.com/spf13/cobra"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +35,7 @@ func doRevert(_ *cobra.Command, _ []string) error {
 		return fmt.Errorf("could not revert to original MCO image: %w", err)
 	}
 
-	if err := rollout.UnexposeClusterImageRegistry(cs); err != nil {
+	if err := helpers.UnexposeClusterImageRegistry(context.TODO(), cs); err != nil {
 		return fmt.Errorf("could not unexpose cluster image registry: %w", err)
 	}
 

devex/internal/pkg/rollout/external.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/containers/image/v5/docker"
 	"github.com/docker/distribution/registry/api/errcode"
-	errcodev2 "github.com/docker/distribution/registry/api/v2"
+	v2 "github.com/docker/distribution/registry/api/v2"
 )
 
 // IsTolerableDeleteErr determines if the returned error message during image deletion can be
@@ -24,23 +24,59 @@ func IsTolerableDeleteErr(err error) bool {
 		return false
 	}
 
-	// Any errors related to the actual image registry query are wrapped in an
-	// ErrImage instance. This allows us to easily identify intolerable errors
-	// such as not being able to write the authfile or certs, etc.
-	var errImage *ErrImage
-	if !errors.As(err, &errImage) {
+	if IsImageNotFoundErr(err) {
+		return true
+	}
+
+	if IsAccessDeniedErr(err) {
+		return true
+	}
+
+	return false
+}
+
+// IsImageNotFoundErr determines if the returned error message indicates that
+// the image is not found. This assumes that the image registry returns such an
+// error code, which is not always the case. Some image registries return an
+// unauthorized or forbidden error which this function does not take into
+// account. For that, use IsAccessDeniedErr below as an additional check.
+func IsImageNotFoundErr(err error) bool {
+	if err == nil {
 		return false
 	}
 
-	if isTolerableErrorCode(err) {
+	if !isErrImage(err) {
+		return false
+	}
+
+	if isMaskedHTTP404(err) {
 		return true
 	}
 
-	if isTolerableUnexpectedHTTPStatusError(err) {
+	if isImageNotFoundErrorCode(err) {
 		return true
 	}
 
-	if isMaskedHTTP404(err) {
+	return false
+}
+
+// IsAccessDeniedErr determines if the returned error indicates that the image
+// cannot be accessed or the operation cannot be performed due to permissions
+// issue. Some image registries use this as a proxy for the image not existing.
+func IsAccessDeniedErr(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if !isErrImage(err) {
+		return false
+	}
+
+	if isAccessDeniedErrorCode(err) {
+		return true
+	}
+
+	if isTolerableUnexpectedHTTPStatusError(err) {
 		return true
 	}
 
@@ -59,37 +95,80 @@ func isMaskedHTTP404(err error) bool {
 	return strings.Contains(err.Error(), "Image may not exist or is not stored with a v2 Schema in a v2 registry")
 }
 
-// isTolerableErrorCode checks if the error code from the registry is tolerable for deletion.
-// This includes cases where the manifest is unknown, or authorization is denied.
-func isTolerableErrorCode(err error) bool {
+// isImageNotFoundErrorCode checks if the error is an ErrorCode instance
+// indicating that a given manifest is not found or the repo name is unknown.
+// This also handles the Quay.io edgecase of an image being deleted and Quay
+// returning an HTTP 500 indicating that.
+func isImageNotFoundErrorCode(err error) bool {
 	if err == nil {
 		return false
 	}
 
+	if isManifestUnknownError(err) {
+		return true
+	}
+
+	if isNameUnknownError(err) {
+		return true
+	}
+
 	var errCode errcode.Error
-	if !errors.As(err, &errCode) {
-		return false
+	if errors.As(err, &errCode) {
+		return isQuayErrorCode(errCode)
 	}
 
-	code := errCode.ErrorCode()
+	return false
+}
 
-	if code == errcodev2.ErrorCodeManifestUnknown {
+// Determines if the error is due to the repo name being unknown.
+func isNameUnknownError(err error) bool {
+	var ec errcode.ErrorCoder
+	if errors.As(err, &ec) && ec.ErrorCode() == v2.ErrorCodeNameUnknown {
 		return true
 	}
 
-	// Quay.io returns this code whenever one is not authorized to delete an image.
-	if code == errcode.ErrorCodeUnauthorized {
+	return false
+}
+
+// Adapted from: https://github.com/containers/image/blob/52ee4dff559a09ffa45783c50bcb7b3f7faebb04/docker/docker_client.go#L1109-L1133
+func isManifestUnknownError(err error) bool {
+	// docker/distribution, and as defined in the spec
+	var ec errcode.ErrorCoder
+	if errors.As(err, &ec) && ec.ErrorCode() == v2.ErrorCodeManifestUnknown {
+		return true
+	}
+	// registry.redhat.io as of October 2022
+	var e errcode.Error
+	if errors.As(err, &e) && e.ErrorCode() == errcode.ErrorCodeUnknown && e.Message == "Not Found" {
+		return true
+	}
+	// Harbor v2.10.2
+	if errors.As(err, &e) && e.ErrorCode() == errcode.ErrorCodeUnknown && strings.Contains(strings.ToLower(e.Message), "not found") {
+		return true
+	}
+	// registry.access.redhat.com as of August 2025
+	if errors.As(err, &e) && e.ErrorCode() == v2.ErrorCodeNameUnknown {
 		return true
 	}
 
-	// Docker.io returns this code whenever one is not authorized to delete an image.
-	if code == errcode.ErrorCodeDenied {
+	return false
+}
+
+// isAccessDeniedErrorCode checks if the error is an ErrorCode instance and
+// then checks the known status codes.
+func isAccessDeniedErrorCode(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	var ec errcode.ErrorCoder
+	// Quay.io returns this code whenever one is not authorized to delete an image.
+	if errors.As(err, &ec) && ec.ErrorCode() == errcode.ErrorCodeUnauthorized {
 		return true
 	}
 
-	// Quay.io returns an HTTP 500 if an image was recently deleted and the
-	// garbage collection has not run yet.
-	if isQuayErrorCode(errCode) {
+	// Docker.io returns this code whenever one is not authorized to delete an image.
+	if errors.As(err, &ec) && ec.ErrorCode() == errcode.ErrorCodeDenied {
 		return true
 	}
 
@@ -128,15 +207,16 @@ func isTolerableUnexpectedHTTPStatusError(err error) bool {
 	}
 
 	var unexpectedHTTPErr docker.UnexpectedHTTPStatusError
-	if !errors.As(err, &unexpectedHTTPErr) {
-		return false
+	if errors.As(err, &unexpectedHTTPErr) && unexpectedHTTPErr.StatusCode == http.StatusUnauthorized {
+		return true
 	}
 
-	if unexpectedHTTPErr.StatusCode == http.StatusUnauthorized {
+	if errors.As(err, &unexpectedHTTPErr) && unexpectedHTTPErr.StatusCode == http.StatusForbidden {
 		return true
 	}
 
-	if unexpectedHTTPErr.StatusCode == http.StatusForbidden {
+	var unauthedForCreds docker.ErrUnauthorizedForCredentials
+	if errors.As(err, &unauthedForCreds) {
 		return true
 	}
 
@@ -189,3 +269,17 @@ func (e *ErrImage) Error() string {
 func (e *ErrImage) Unwrap() error {
 	return e.err
 }
+
+// isErrImage determines whether the given error is an instance of the ErrImage
+// type defined above.
+func isErrImage(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Any errors related to the actual image registry query are wrapped in an
+	// ErrImage instance. This allows us to easily identify intolerable errors
+	// such as not being able to write the authfile or certs, etc.
+	var errImage *ErrImage
+	return errors.As(err, &errImage)
+}