install skopeo from source, when missing

Author: cheesesashimi

Makefile

@@ -109,6 +109,18 @@ else
 	GO111MODULE=on go build -o $(GOPATH)/bin/golangci-lint ./vendor/github.com/golangci/golangci-lint/cmd/golangci-lint
 endif
 
+SKOPEO := $(shell command -v skopeo 2> /dev/null)
+install-skopeo:
+ifdef SKOPEO
+	@echo "Found skopeo"
+	skopeo --version
+else
+	@echo "Installing skopeo"
+	./hack/install-skopeo.sh
+endif
+
+# install-skopeo is purposely omitted from this target because it is only
+# needed for a single test target (test-e2e-ocl).
 install-tools: install-golangci-lint install-go-junit-report install-setup-envtest
 
 # Runs golangci-lint
@@ -203,8 +215,9 @@ test-e2e-techpreview: install-go-junit-report
 test-e2e-single-node: install-go-junit-report
 	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 120m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-single-node/ | ./hack/test-with-junit.sh $(@)
 
-test-e2e-ocl: install-go-junit-report
-	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 190m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-ocl/ | ./hack/test-with-junit.sh $(@)
+test-e2e-ocl: install-go-junit-report install-skopeo
+	# Temporarily include /tmp/skopeo/bin in our PATH variable so that the test suite can find skopeo.
+	set -o pipefail; PATH="$(PATH):/tmp/skopeo/bin" go test -tags=$(GOTAGS) -failfast -timeout 190m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-ocl/ | ./hack/test-with-junit.sh $(@)
 
 bootstrap-e2e: install-go-junit-report install-setup-envtest
 	@echo "Setting up KUBEBUILDER_ASSETS"

hack/install-skopeo.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -xeuo
+
+# This is an installation script for skopeo. skopeo is needed by the
+# e2e-gcp-op-ocl test suite since it pushes and pulls images around.
+#
+# Using a build-root image for this is the preferred approach (see:
+# https://docs.ci.openshift.org/docs/architecture/ci-operator/#build-root-image)
+# since skopeo can be installed via dnf at build-time. However, that process is
+# a bit more involved. While it would ultimately pay off, it is not in-scope
+# with the bug that is being resolved.
+#
+# Because of limitations within the builder container image and the context it
+# runs in, certain adaptations must be made:
+# - The builder image does not run as a privileged user and is denied privilege
+# escalation. This means that running dnf install -y skopeo cannot be done.
+# - The builder image does not have jq installed. This means we need to use
+# Python for any JSON parsing we need to perform instead.
+# - Because this runs as a non-privileged user, we cannot run the make install
+# step for skopeo. Instead, we need to append /tmp/skopeo/bin to the PATH. This
+# is done in the Makefile and only for the go test invocation.
+
+OPENSHIFT_CI="${OPENSHIFT_CI:-""}"
+
+install_skopeo() {
+  # If we've already built skopeo once, check if it works and then return.
+  if [ -f /tmp/skopeo/bin/skopeo ]; then
+    echo "Prebuilt skopeo found at /tmp/skopeo/bin/skopeo, skipping installation"
+    /tmp/skopeo/bin/skopeo --version
+    return 0
+  fi
+
+  # Get the most recent tagged version of skopeo.
+  skopeo_version="$(curl -s https://api.github.com/repos/containers/skopeo/releases/latest | python3 -c 'import sys, json; print(json.load(sys.stdin)["tag_name"])')"
+
+  echo "Installing skopeo $skopeo_version from source"
+
+  skopeo_clone_dir="/tmp/skopeo"
+
+  mkdir -p "$skopeo_clone_dir"
+
+  # Shallow-clone the skopeo repo to the local repo dir.
+  git clone --branch "$skopeo_version" --depth 1 https://github.com/containers/skopeo.git "$skopeo_clone_dir"
+
+  cd "$skopeo_clone_dir"
+
+  # Build skopeo
+  make bin/skopeo
+}
+
+# Check if we have skopeo installed first.
+if command -v skopeo >/dev/null 2>&1  ; then
+  # If we do, just output its version.
+  skopeo --version
+else
+  # Check if we're running in CI.
+  if [ "$OPENSHIFT_CI" == "true" ]; then
+    # Only when we're in CI should we install skopeo.
+    install_skopeo
+  else
+    # Otherwise, we should exit with a clear error.
+    echo "Missing required binary 'skopeo'"
+    exit 1
+  fi
+fi

test/e2e-ocl/imagepruner_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,7 +16,9 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 
+	"github.com/containers/image/v5/docker"
 	"github.com/containers/image/v5/types"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/distribution/reference"
@@ -141,23 +144,52 @@ func TestImagePrunerOnCluster(t *testing.T) {
 	certsDir := filepath.Join(t.TempDir())
 	require.NoError(t, os.WriteFile(filepath.Join(certsDir, externalRegistryHostname+".crt"), ingressCert.Data["tls.crt"], 0o644))
 
-	// Wait for the route to finish setting up. Not sure if there is an object we can poll for this instead.
-	time.Sleep(time.Second * 5)
+	// Wait for the route to finish setting up. We can determine that the route
+	// setup is complete when we get an image not found error when inspecting a
+	// nonexistent image.
+	err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
+		imgPruner := imagepruner.NewImageInspectorDeleter()
+		sysCtx := &types.SystemContext{DockerCertPath: certsDir, AuthFilePath: secretPath}
+
+		_, _, err = imgPruner.ImageInspect(ctx, sysCtx, pullspec)
+
+		// If we get an image not found error, that means the route is set up
+		// because we were able to authenticate to the image registry and make a
+		// query for a nonexistent image.
+		if imagepruner.IsImageNotFoundErr(err) {
+			return true, nil
+		}
+
+		// If this is an HTTP 503 error, that means the route has not finished
+		// being set up, so we need to try again.
+		var unexpectedHTTPError docker.UnexpectedHTTPStatusError
+		if errors.As(err, &unexpectedHTTPError) && unexpectedHTTPError.StatusCode == http.StatusServiceUnavailable {
+			return false, nil
+		}
+
+		// We were unable to identify this error, so return it.
+		return false, fmt.Errorf("unknown registry error when polling: %w", err)
+	})
+
+	require.NoError(t, err, unwrapAll(err))
 
 	// Now we can run our test cases. All test cases use the
 	// ImageInspectorDeleter directly since we need to have a bit more control
 	// over the SystemContext given that we're running out-of-cluster.
 	t.Run("Inspect without creds", func(t *testing.T) {
 		t.Parallel()
+
 		imgPruner := imagepruner.NewImageInspectorDeleter()
 		sysCtx := &types.SystemContext{DockerCertPath: certsDir}
+
 		_, _, err = imgPruner.ImageInspect(ctx, sysCtx, pullspec)
 		assert.Error(t, err)
 		assert.True(t, imagepruner.IsAccessDeniedErr(err), "expected access denied err: %s", unwrapAll(err))
 	})
 
 	t.Run("Inspect nonexistent image digest with creds", func(t *testing.T) {
 		t.Parallel()
+
 		imgPruner := imagepruner.NewImageInspectorDeleter()
 		sysCtx := &types.SystemContext{DockerCertPath: certsDir, AuthFilePath: secretPath}
 
@@ -172,6 +204,7 @@ func TestImagePrunerOnCluster(t *testing.T) {
 
 	t.Run("Inspect nonexistent image tag with creds", func(t *testing.T) {
 		t.Parallel()
+
 		imgPruner := imagepruner.NewImageInspectorDeleter()
 		sysCtx := &types.SystemContext{DockerCertPath: certsDir, AuthFilePath: secretPath}
 
@@ -185,6 +218,7 @@ func TestImagePrunerOnCluster(t *testing.T) {
 
 	t.Run("Inspect nonexistent image repo with creds", func(t *testing.T) {
 		t.Parallel()
+
 		imgPruner := imagepruner.NewImageInspectorDeleter()
 		sysCtx := &types.SystemContext{DockerCertPath: certsDir, AuthFilePath: secretPath}
 
@@ -198,9 +232,12 @@ func TestImagePrunerOnCluster(t *testing.T) {
 
 	t.Run("Push image and inspect", func(t *testing.T) {
 		t.Parallel()
+
 		require.NoError(t, createAndPushScratchImage(ctx, t, pullspec, secretPath, certsDir))
+
 		imgPruner := imagepruner.NewImageInspectorDeleter()
 		sysCtx := &types.SystemContext{DockerCertPath: certsDir, AuthFilePath: secretPath}
+
 		_, _, err := imgPruner.ImageInspect(ctx, sysCtx, pullspec)
 		assert.NoError(t, err)
 
@@ -779,6 +816,21 @@ func canTestOnInClusterRegistry(ctx context.Context, kubeconfig string) (bool, e
 	return false, nil
 }
 
+// Skopeo requires that a policy.json file be present. Usually, this file is
+// placed in /etc/containers/policy.json when Skopeo is installed. Because we
+// must install skopeo from source in CI, this file is missing. So what we do
+// in this scenario is write our own policy.json file to a temp directory
+// instead. The temp directory is managed by the Go test suite and will be
+// removed after the test is finished.
+func writePolicyFile(t *testing.T) (string, error) {
+	policyPath := filepath.Join(t.TempDir(), "policy.json")
+
+	// Compacted contents of https://github.com/containers/skopeo/blob/main/default-policy.json
+	policyJSONBytes := []byte(`{"default":[{"type":"insecureAcceptAnything"}],"transports":{"docker-daemon":{"":[{"type":"insecureAcceptAnything"}]}}}`)
+
+	return policyPath, os.WriteFile(policyPath, policyJSONBytes, 0o755)
+}
+
 // Creates an empty scratch image and pushes it to the given pullspec using the
 // provided secret path. Accepts an optional certsDir parameter which is
 // particularly useful for pushing internal image registries which have
@@ -792,9 +844,14 @@ func createAndPushScratchImage(ctx context.Context, t *testing.T, pullspec, secr
 		return err
 	}
 
-	cmd := exec.Command("skopeo", "copy", "--dest-authfile", secretPath, "tarball://"+srcImage, "docker://"+pullspec)
+	policyPath, err := writePolicyFile(t)
+	if err != nil {
+		return fmt.Errorf("could not write policy.json file: %w", err)
+	}
+
+	cmd := exec.Command("skopeo", "--policy", policyPath, "copy", "--dest-authfile", secretPath, "tarball://"+srcImage, "docker://"+pullspec)
 	if certsDir != "" {
-		cmd = exec.Command("skopeo", "copy", "--dest-cert-dir", certsDir, "--dest-authfile", secretPath, "tarball://"+srcImage, "docker://"+pullspec)
+		cmd = exec.Command("skopeo", "--policy", policyPath, "copy", "--dest-cert-dir", certsDir, "--dest-authfile", secretPath, "tarball://"+srcImage, "docker://"+pullspec)
 	}
 
 	t.Logf("Copying %s to %s using skopeo", srcImage, pullspec)