add repo agnosticism to e2e/integration tests

Signed-off-by: Humair Khan <[email protected]>

Author: HumairAK

.github/workflows/integration-tests-v1.yml

@@ -73,7 +73,7 @@ jobs:
         id: integration-tests
         if: ${{ steps.forward-mysql-port.outcome == 'success' }}
         working-directory: ./backend/test/integration
-        run: go test -v ./... -args -runIntegrationTests=true -namespace=kubeflow
+        run: go test -v ./... -args -runIntegrationTests=true -namespace=kubeflow -repoName=${{ github.repository }} -branchName=${{ github.event.repository.default_branch }}
         env:
           PULL_NUMBER: ${{ github.event.pull_request.number }}
         continue-on-error: true

.github/workflows/legacy-v2-api-integration-tests.yml

@@ -83,7 +83,7 @@ jobs:
         id: tests
         if: ${{ steps.forward-mlmd-port.outcome == 'success' }}
         working-directory: ./backend/test/v2/integration
-        run: go test -v ./... -args -runIntegrationTests=true -namespace=kubeflow -tlsEnabled=${{ matrix.pod_to_pod_tls_enabled }} -caCertPath=${{ env.CA_CERT_PATH }}
+        run: go test -v ./... -args -runIntegrationTests=true -namespace=kubeflow -repoName=${{ github.repository }} -branchName=${{ github.event.repository.default_branch }}
         env:
           PULL_NUMBER: ${{ github.event.pull_request.number }}
           PIPELINE_STORE: ${{ matrix.pipeline_store }}

backend/src/apiserver/server/pipeline_upload_server_test.go

@@ -47,24 +47,24 @@ const (
 func TestUploadPipeline(t *testing.T) {
 	// TODO(v2): when we add a field to distinguish between v1 and v2 template, verify it's in the response
 	tt := []struct {
-		name        string
-		spec        []byte
+		name       string
+		spec       []byte
 		apiVersion string
 	}{{
-		name:        "upload argo workflow YAML",
-		spec:        []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"),
+		name:       "upload argo workflow YAML",
+		spec:       []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"),
 		apiVersion: "v1beta1",
 	}, {
-		name:        "upload argo workflow YAML",
-		spec:        []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"),
+		name:       "upload argo workflow YAML",
+		spec:       []byte("apiVersion: argoproj.io/v1alpha1\nkind: Workflow"),
 		apiVersion: "v2beta1",
 	}, {
-		name:        "upload pipeline v2 job in proto yaml",
-		spec:        []byte(v2SpecHelloWorld),
+		name:       "upload pipeline v2 job in proto yaml",
+		spec:       []byte(v2SpecHelloWorld),
 		apiVersion: "v1beta1",
 	}, {
-		name:        "upload pipeline v2 job in proto yaml",
-		spec:        []byte(v2SpecHelloWorld),
+		name:       "upload pipeline v2 job in proto yaml",
+		spec:       []byte(v2SpecHelloWorld),
 		apiVersion: "v2beta1",
 	}}
 	for _, test := range tt {

backend/src/common/client/api_server/v2/pipeline_client.go

@@ -17,6 +17,7 @@ package api_server_v2
 import (
 	"crypto/tls"
 	"fmt"
+
 	"github.com/go-openapi/runtime"
 	"github.com/go-openapi/strfmt"
 	apiclient "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_client"

backend/test/integration/pipeline_api_test.go

@@ -15,7 +15,6 @@
 package integration
 
 import (
-	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -24,10 +23,11 @@ import (
 	model "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_model"
 	uploadParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
 	pipelinetemplate "github.com/kubeflow/pipelines/backend/src/apiserver/template"
-	api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
+	"github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 	"github.com/kubeflow/pipelines/backend/test"
 	"github.com/kubeflow/pipelines/backend/test/config"
+	"github.com/kubeflow/pipelines/backend/test/testutil"
 
 	"github.com/golang/glog"
 	"github.com/stretchr/testify/assert"
@@ -45,6 +45,8 @@ import (
 type PipelineApiTest struct {
 	suite.Suite
 	namespace            string
+	repoName             string
+	branchName           string
 	resourceNamespace    string
 	pipelineClient       *api_server.PipelineClient
 	pipelineUploadClient *api_server.PipelineUploadClient
@@ -64,6 +66,8 @@ func (s *PipelineApiTest) SetupTest() {
 		}
 	}
 	s.namespace = *config.Namespace
+	s.repoName = *config.REPO_NAME
+	s.branchName = *config.BRANCH_NAME
 
 	var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
 	var newPipelineClient func() (*api_server.PipelineClient, error)
@@ -123,10 +127,11 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	assert.Contains(t, err.Error(), "Failed to upload pipeline")
 
 	/* ---------- Import pipeline YAML by URL ---------- */
+	pipelineURL := testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "backend/test/v2/resources/sequential.yaml")
 	time.Sleep(1 * time.Second)
 	sequentialPipeline, err := s.pipelineClient.Create(&params.PipelineServiceCreatePipelineV1Params{
 		Pipeline: &model.APIPipeline{Name: "sequential", URL: &model.APIURL{
-			PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential.yaml",
+			PipelineURL: pipelineURL,
 		}},
 	})
 	require.Nil(t, err)
@@ -140,11 +145,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	assert.Equal(t, "zip-arguments-parameters", argumentUploadPipeline.Name)
 
 	/* ---------- Import pipeline tarball by URL ---------- */
-	pipelineURL := "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip"
-
-	if pullNumber := os.Getenv("PULL_NUMBER"); pullNumber != "" {
-		pipelineURL = fmt.Sprintf("https://raw.githubusercontent.com/kubeflow/pipelines/pull/%s/head/backend/test/v2/resources/arguments.pipeline.zip", pullNumber)
-	}
+	pipelineURL = testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "backend/test/v2/resources/arguments.pipeline.zip")
 
 	time.Sleep(1 * time.Second)
 	argumentUrlPipeline, err := s.pipelineClient.Create(&params.PipelineServiceCreatePipelineV1Params{

backend/test/integration/pipeline_version_api_test.go

@@ -15,7 +15,6 @@
 package integration
 
 import (
-	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -24,10 +23,11 @@ import (
 	"github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_model"
 	uploadParams "github.com/kubeflow/pipelines/backend/api/v1beta1/go_http_client/pipeline_upload_client/pipeline_upload_service"
 	pipelinetemplate "github.com/kubeflow/pipelines/backend/src/apiserver/template"
-	api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
+	"github.com/kubeflow/pipelines/backend/src/common/client/api_server/v1"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 	"github.com/kubeflow/pipelines/backend/test"
 	"github.com/kubeflow/pipelines/backend/test/config"
+	"github.com/kubeflow/pipelines/backend/test/testutil"
 
 	"github.com/golang/glog"
 	"github.com/stretchr/testify/assert"
@@ -43,6 +43,8 @@ import (
 type PipelineVersionApiTest struct {
 	suite.Suite
 	namespace            string
+	repoName             string
+	branchName           string
 	pipelineClient       *api_server.PipelineClient
 	pipelineUploadClient *api_server.PipelineUploadClient
 }
@@ -84,6 +86,9 @@ func (s *PipelineVersionApiTest) SetupTest() {
 		}
 	}
 
+	s.repoName = *config.REPO_NAME
+	s.branchName = *config.BRANCH_NAME
+
 	var err error
 	s.pipelineUploadClient, err = newPipelineUploadClient()
 	if err != nil {
@@ -143,12 +148,13 @@ func (s *PipelineVersionApiTest) TestArgoSpec() {
 	assert.Contains(t, err.Error(), "Failed to upload pipeline version")
 
 	/* ---------- Import pipeline version YAML by URL ---------- */
+	pipelineURL := testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "backend/test/v2/resources/sequential.yaml")
 	time.Sleep(1 * time.Second)
 	sequentialPipelineVersion, err := s.pipelineClient.CreatePipelineVersion(&params.PipelineServiceCreatePipelineVersionV1Params{
 		Version: &pipeline_model.APIPipelineVersion{
 			Name: "sequential",
 			PackageURL: &pipeline_model.APIURL{
-				PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential.yaml",
+				PipelineURL: pipelineURL,
 			},
 			ResourceReferences: []*pipeline_model.APIResourceReference{
 				{
@@ -173,11 +179,7 @@ func (s *PipelineVersionApiTest) TestArgoSpec() {
 
 	/* ---------- Import pipeline tarball by URL ---------- */
 	time.Sleep(1 * time.Second)
-	pipelineURL := "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/resources/arguments.pipeline.zip"
-
-	if pullNumber := os.Getenv("PULL_NUMBER"); pullNumber != "" {
-		pipelineURL = fmt.Sprintf("https://raw.githubusercontent.com/kubeflow/pipelines/pull/%s/head/backend/test/resources/arguments.pipeline.zip", pullNumber)
-	}
+	pipelineURL = testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "backend/test/resources/arguments.pipeline.zip")
 
 	argumentUrlPipelineVersion, err := s.pipelineClient.CreatePipelineVersion(&params.PipelineServiceCreatePipelineVersionV1Params{
 		Version: &pipeline_model.APIPipelineVersion{

backend/test/integration/upgrade_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 	"github.com/kubeflow/pipelines/backend/test"
 	"github.com/kubeflow/pipelines/backend/test/config"
+	"github.com/kubeflow/pipelines/backend/test/testutil"
 
 	"github.com/go-openapi/strfmt"
 	"github.com/golang/glog"
@@ -50,6 +51,8 @@ import (
 type UpgradeTests struct {
 	suite.Suite
 	namespace            string
+	repoName             string
+	branchName           string
 	resourceNamespace    string
 	experimentClient     *api_server.ExperimentClient
 	pipelineClient       *api_server.PipelineClient
@@ -101,6 +104,7 @@ func (s *UpgradeTests) SetupSuite() {
 		}
 	}
 	s.namespace = *config.Namespace
+	s.repoName = *config.REPO_NAME
 
 	var newExperimentClient func() (*api_server.ExperimentClient, error)
 	var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
@@ -273,11 +277,7 @@ func (s *UpgradeTests) PreparePipelines() {
 	assert.Equal(t, "arguments-parameters.yaml", argumentYAMLPipeline.Name)
 
 	/* ---------- Import pipeline YAML by URL ---------- */
-	pipelineURL := "https://github.com/kubeflow/pipelines/raw/refs/heads/master/test_data/sdk_compiled_pipelines/valid/sequential_v1.yaml"
-
-	if pullNumber := os.Getenv("PULL_NUMBER"); pullNumber != "" {
-		pipelineURL = fmt.Sprintf("https://raw.githubusercontent.com/kubeflow/pipelines/pull/%s/head/test_data/sdk_compiled_pipelines/valid/sequential_v1.yaml", pullNumber)
-	}
+	pipelineURL := testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "test_data/sdk_compiled_pipelines/valid/sequential_v1.yaml")
 	time.Sleep(1 * time.Second)
 	sequentialPipeline, err := s.pipelineClient.Create(&pipelineParams.PipelineServiceCreatePipelineV1Params{
 		Pipeline: &pipeline_model.APIPipeline{Name: "sequential", URL: &pipeline_model.APIURL{
@@ -295,12 +295,7 @@ func (s *UpgradeTests) PreparePipelines() {
 	assert.Equal(t, "zip-arguments-parameters", argumentUploadPipeline.Name)
 
 	/* ---------- Import pipeline tarball by URL ---------- */
-	pipelineURL = "https://github.com/kubeflow/pipelines/raw/refs/heads/master/test_data/sdk_compiled_pipelines/valid/arguments.pipeline.zip"
-
-	if pullNumber := os.Getenv("PULL_NUMBER"); pullNumber != "" {
-		pipelineURL = fmt.Sprintf("https://raw.githubusercontent.com/kubeflow/pipelines/pull/%s/head/test_data/sdk_compiled_pipelines/valid/arguments.pipeline.zip", pullNumber)
-	}
-
+	pipelineURL = testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "test_data/sdk_compiled_pipelines/valid/arguments.pipeline.zip")
 	time.Sleep(1 * time.Second)
 	argumentUrlPipeline, err := s.pipelineClient.Create(&pipelineParams.PipelineServiceCreatePipelineV1Params{
 		Pipeline: &pipeline_model.APIPipeline{

backend/test/testutil/test_utils.go

@@ -118,3 +118,12 @@ func ReplaceSDKInPipelineSpec(pipelineFilePath string, cacheDisabled bool, defau
 
 	return []byte(modifiedPipelineSpec)
 }
+
+func GetRepoBranchURLRAW(repoName, branch, path string) string {
+	url := fmt.Sprintf("https://github.com/%s/raw/refs/heads/%s/%s", repoName, branch, path)
+
+	if pullNumber := os.Getenv("PULL_NUMBER"); pullNumber != "" {
+		url = fmt.Sprintf("https://raw.githubusercontent.com/%s/pull/%s/head/%s", repoName, pullNumber, path)
+	}
+	return url
+}

backend/test/v2/integration/pipeline_api_test.go

@@ -16,8 +16,6 @@ package integration
 
 import (
 	"crypto/tls"
-	"fmt"
-	"os"
 	"testing"
 	"time"
 
@@ -27,6 +25,7 @@ import (
 	api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 	"github.com/kubeflow/pipelines/backend/test/config"
+	"github.com/kubeflow/pipelines/backend/test/testutil"
 	test "github.com/kubeflow/pipelines/backend/test/v2"
 
 	"github.com/golang/glog"
@@ -44,6 +43,8 @@ import (
 type PipelineApiTest struct {
 	suite.Suite
 	namespace            string
+	repoName             string
+	branchName           string
 	resourceNamespace    string
 	pipelineClient       *api_server.PipelineClient
 	pipelineUploadClient api_server.PipelineUploadInterface
@@ -63,6 +64,8 @@ func (s *PipelineApiTest) SetupTest() {
 		}
 	}
 	s.namespace = *config.Namespace
+	s.repoName = *config.REPO_NAME
+	s.branchName = *config.BRANCH_NAME
 
 	var newPipelineClient func() (*api_server.PipelineClient, error)
 	var tlsCfg *tls.Config
@@ -148,6 +151,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	assert.Contains(t, err.Error(), "Failed to upload pipeline")
 
 	/* ---------- Import pipeline YAML by URL ---------- */
+	pipelineURL := testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "backend/test/v2/resources/sequential-v2.yaml")
 	time.Sleep(1 * time.Second)
 	sequentialPipeline, err := s.pipelineClient.CreatePipelineAndVersion(&params.PipelineServiceCreatePipelineAndVersionParams{
 		Body: &model.V2beta1CreatePipelineAndVersionRequest{
@@ -158,7 +162,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 			},
 			PipelineVersion: &model.V2beta1PipelineVersion{
 				PackageURL: &model.V2beta1URL{
-					PipelineURL: "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential-v2.yaml",
+					PipelineURL: pipelineURL,
 				},
 			},
 		},
@@ -175,7 +179,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	assert.Equal(t, "sequential", sequentialPipelineVersions[0].DisplayName)
 	assert.Equal(t, "sequential pipeline", sequentialPipelineVersions[0].Description)
 	assert.Equal(t, sequentialPipeline.PipelineID, sequentialPipelineVersions[0].PipelineID)
-	assert.Equal(t, "https://raw.githubusercontent.com/kubeflow/pipelines/refs/heads/master/backend/test/v2/resources/sequential-v2.yaml", string(sequentialPipelineVersions[0].PackageURL.PipelineURL))
+	assert.Equal(t, pipelineURL, sequentialPipelineVersions[0].PackageURL.PipelineURL)
 
 	/* ---------- Upload pipelines zip ---------- */
 	time.Sleep(1 * time.Second)
@@ -185,12 +189,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	assert.Equal(t, "zip-arguments-parameters", argumentUploadPipeline.DisplayName)
 
 	/* ---------- Import pipeline tarball by URL ---------- */
-	pipelineURL := "https://github.com/kubeflow/pipelines/raw/refs/heads/master/backend/test/v2/resources/arguments.pipeline.zip"
-
-	if pullNumber := os.Getenv("PULL_NUMBER"); pullNumber != "" {
-		pipelineURL = fmt.Sprintf("https://raw.githubusercontent.com/kubeflow/pipelines/pull/%s/head/backend/test/v2/resources/arguments.pipeline.zip", pullNumber)
-	}
-
+	pipelineURL = testutil.GetRepoBranchURLRAW(s.repoName, s.branchName, "backend/test/v2/resources/arguments.pipeline.zip")
 	time.Sleep(1 * time.Second)
 	argumentUrlPipeline, err := s.pipelineClient.Create(&params.PipelineServiceCreatePipelineParams{
 		Pipeline: &model.V2beta1Pipeline{DisplayName: "arguments.pipeline.zip", Name: "arguments-pipeline-zip"},
@@ -212,7 +211,7 @@ func (s *PipelineApiTest) TestPipelineAPI() {
 	assert.Equal(t, "argumenturl-v1", argumentUrlPipelineVersion.DisplayName)
 	assert.Equal(t, "1st version of argument url pipeline", argumentUrlPipelineVersion.Description)
 	assert.Equal(t, argumentUrlPipeline.PipelineID, argumentUrlPipelineVersion.PipelineID)
-	assert.Equal(t, pipelineURL, string(argumentUrlPipelineVersion.PackageURL.PipelineURL))
+	assert.Equal(t, pipelineURL, argumentUrlPipelineVersion.PackageURL.PipelineURL)
 
 	/* ---------- Verify list pipeline works ---------- */
 	pipelines, totalSize, _, err := s.pipelineClient.List(&params.PipelineServiceListPipelinesParams{})