OCPBUGS-60157: ensure that images are rebuilt if they do not exist

Makefile

@@ -109,6 +109,18 @@ else
 	GO111MODULE=on go build -o $(GOPATH)/bin/golangci-lint ./vendor/github.com/golangci/golangci-lint/cmd/golangci-lint
 endif
 
+SKOPEO := $(shell command -v skopeo 2> /dev/null)
+install-skopeo:
+ifdef SKOPEO
+	@echo "Found skopeo"
+	skopeo --version
+else
+	@echo "Installing skopeo"
+	./hack/install-skopeo.sh
+endif
+
+# install-skopeo is purposely omitted from this target because it is only
+# needed for a single test target (test-e2e-ocl).
 install-tools: install-golangci-lint install-go-junit-report install-setup-envtest
 
 # Runs golangci-lint
@@ -203,8 +215,9 @@ test-e2e-techpreview: install-go-junit-report
 test-e2e-single-node: install-go-junit-report
 	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 120m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-single-node/ | ./hack/test-with-junit.sh $(@)
 
-test-e2e-ocl: install-go-junit-report
-	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 120m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-ocl/ | ./hack/test-with-junit.sh $(@)
+test-e2e-ocl: install-go-junit-report install-skopeo
+	# Temporarily include /tmp/skopeo/bin in our PATH variable so that the test suite can find skopeo.
+	set -o pipefail; PATH="$(PATH):/tmp/skopeo/bin" go test -tags=$(GOTAGS) -failfast -timeout 190m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-ocl/ | ./hack/test-with-junit.sh $(@)
 
 bootstrap-e2e: install-go-junit-report install-setup-envtest
 	@echo "Setting up KUBEBUILDER_ASSETS"

devex/cmd/mco-builder/local.go

@@ -15,6 +15,7 @@ import (
 	"github.com/openshift/machine-config-operator/devex/internal/pkg/utils"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
+	"github.com/openshift/machine-config-operator/test/helpers"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	aggerrs "k8s.io/apimachinery/pkg/util/errors"
@@ -192,7 +193,7 @@ func buildLocallyAndPushIntoCluster(cs *framework.ClientSet, buildOpts localBuil
 		}
 	}()
 
-	extHostname, err := rollout.ExposeClusterImageRegistry(cs)
+	extHostname, err := helpers.ExposeClusterImageRegistry(context.TODO(), cs)
 	if err != nil {
 		return err
 	}

devex/cmd/mco-builder/revert.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openshift/machine-config-operator/devex/internal/pkg/rollout"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	"github.com/openshift/machine-config-operator/test/framework"
+	"github.com/openshift/machine-config-operator/test/helpers"
 	"github.com/spf13/cobra"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +35,7 @@ func doRevert(_ *cobra.Command, _ []string) error {
 		return fmt.Errorf("could not revert to original MCO image: %w", err)
 	}
 
-	if err := rollout.UnexposeClusterImageRegistry(cs); err != nil {
+	if err := helpers.UnexposeClusterImageRegistry(context.TODO(), cs); err != nil {
 		return fmt.Errorf("could not unexpose cluster image registry: %w", err)
 	}
 

hack/install-skopeo.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -xeuo
+
+# This is an installation script for skopeo. skopeo is needed by the
+# e2e-gcp-op-ocl test suite since it pushes and pulls images around.
+#
+# Using a build-root image for this is the preferred approach (see:
+# https://docs.ci.openshift.org/docs/architecture/ci-operator/#build-root-image)
+# since skopeo can be installed via dnf at build-time. However, that process is
+# a bit more involved. While it would ultimately pay off, it is not in-scope
+# with the bug that is being resolved.
+#
+# Because of limitations within the builder container image and the context it
+# runs in, certain adaptations must be made:
+# - The builder image does not run as a privileged user and is denied privilege
+# escalation. This means that running dnf install -y skopeo cannot be done.
+# - The builder image does not have jq installed. This means we need to use
+# Python for any JSON parsing we need to perform instead.
+# - Because this runs as a non-privileged user, we cannot run the make install
+# step for skopeo. Instead, we need to append /tmp/skopeo/bin to the PATH. This
+# is done in the Makefile and only for the go test invocation.
+
+OPENSHIFT_CI="${OPENSHIFT_CI:-""}"
+
+install_skopeo() {
+  # If we've already built skopeo once, check if it works and then return.
+  if [ -f /tmp/skopeo/bin/skopeo ]; then
+    echo "Prebuilt skopeo found at /tmp/skopeo/bin/skopeo, skipping installation"
+    /tmp/skopeo/bin/skopeo --version
+    return 0
+  fi
+
+  # Get the most recent tagged version of skopeo.
+  skopeo_version="$(curl -s https://api.github.com/repos/containers/skopeo/releases/latest | python3 -c 'import sys, json; print(json.load(sys.stdin)["tag_name"])')"
+
+  echo "Installing skopeo $skopeo_version from source"
+
+  skopeo_clone_dir="/tmp/skopeo"
+
+  mkdir -p "$skopeo_clone_dir"
+
+  # Shallow-clone the skopeo repo to the local repo dir.
+  git clone --branch "$skopeo_version" --depth 1 https://github.com/containers/skopeo.git "$skopeo_clone_dir"
+
+  cd "$skopeo_clone_dir"
+
+  # Build skopeo
+  make bin/skopeo
+}
+
+# Check if we have skopeo installed first.
+if command -v skopeo >/dev/null 2>&1  ; then
+  # If we do, just output its version.
+  skopeo --version
+else
+  # Check if we're running in CI.
+  if [ "$OPENSHIFT_CI" == "true" ]; then
+    # Only when we're in CI should we install skopeo.
+    install_skopeo
+  else
+    # Otherwise, we should exit with a clear error.
+    echo "Missing required binary 'skopeo'"
+    exit 1
+  fi
+fi