Merge pull request #6 from xmudrii/bot-changes

Implement required patches for kcp

Author: kcp-ci-bot

artifacts/scripts/construct.sh

@@ -37,8 +37,8 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
-if [ ! $# -eq 15 ]; then
-    echo "usage: $0 repo src_branch dst_branch dependent_k8s.io_repos required_packages kubernetes_remote subdirectories source_repo_org source_repo_name base_package is_library recursive_delete_pattern skip_tags last_published_upstream_hash git_default_branch"
+if [ ! $# -eq 17 ]; then
+    echo "usage: $0 repo src_branch dst_branch dependent_k8s.io_repos required_packages kubernetes_remote subdirectories source_repo_org source_repo_name base_package is_library recursive_delete_pattern skip_tags skip_non_semver_tags semver_tags_base last_published_upstream_hash git_default_branch"
     exit 1
 fi
 
@@ -72,12 +72,16 @@ IS_LIBRARY="${2}"
 RECURSIVE_DELETE_PATTERN="${3}"
 # Skip syncing tags
 SKIP_TAGS="${4}"
+# Skip tags that do not start with 'v'
+SKIP_NON_SEMVER_TAGS="${5}"
+# Major version to use for building the tag (e.g. v1, v2...)
+SEMVER_TAGS_BASE="${6}"
 # last published upstream hash of this branch
-LAST_PUBLISHED_UPSTREAM_HASH="${5}"
+LAST_PUBLISHED_UPSTREAM_HASH="${7}"
 # name of the main branch. master for k8s.io/kubernetes
-GIT_DEFAULT_BRANCH="${6}"
+GIT_DEFAULT_BRANCH="${8}"
 
-readonly REPO SRC_BRANCH DST_BRANCH DEPS REQUIRED SOURCE_REMOTE SOURCE_REPO_ORG SUBDIRS SOURCE_REPO_NAME BASE_PACKAGE IS_LIBRARY RECURSIVE_DELETE_PATTERN SKIP_TAGS LAST_PUBLISHED_UPSTREAM_HASH GIT_DEFAULT_BRANCH
+readonly REPO SRC_BRANCH DST_BRANCH DEPS REQUIRED SOURCE_REMOTE SOURCE_REPO_ORG SUBDIRS SOURCE_REPO_NAME BASE_PACKAGE IS_LIBRARY RECURSIVE_DELETE_PATTERN SKIP_TAGS SKIP_NON_SEMVER_TAGS SEMVER_TAGS_BASE LAST_PUBLISHED_UPSTREAM_HASH GIT_DEFAULT_BRANCH
 
 SCRIPT_DIR=$(dirname "${BASH_SOURCE}")
 source "${SCRIPT_DIR}"/util.sh
@@ -177,7 +181,9 @@ if [ -z "${SKIP_TAGS}" ]; then
                --push-script ${PUSH_SCRIPT} \
                --dependencies "${DEPS}" \
                --mapping-output-file "../tag-${REPO}-{{.Tag}}-mapping" \
-               --publish-v0-semver \
+               --publish-semver-tags \
+               --skip-non-semver-tags="${SKIP_NON_SEMVER_TAGS}" \
+               --semver-tags-base "${SEMVER_TAGS_BASE}" \
                -alsologtostderr \
                "${EXTRA_ARGS[@]-}"
     if [ "${LAST_HEAD}" != "$(git rev-parse ${LAST_BRANCH})" ]; then

artifacts/scripts/util.sh

@@ -171,7 +171,7 @@ sync_repo() {
     local f_mainline_commits=""
     if [ "${new_branch}" = "true" ] && [ "${src_branch}" = "${git_default_branch}" ]; then
         # new master branch
-        filter-branch "${commit_msg_tag}" "${subdirectories}" "${recursive_delete_pattern}" ${src_branch} filtered-branch
+        filter-branch "${commit_msg_tag}" "${subdirectories}" "${recursive_delete_pattern}" "upstream/${src_branch}" filtered-branch
 
         # find commits on the main line (will mostly be merges, but could be non-merges if filter-branch dropped
         # the corresponding fast-forward merge and left the feature branch commits)

cmd/publishing-bot/config/rules.go

@@ -73,6 +73,8 @@ type BranchRule struct {
 type RepositoryRule struct {
 	DestinationRepository string       `yaml:"destination"`
 	Branches              []BranchRule `yaml:"branches"`
+	// the value to use as vX in vX.Y.Z published at the destination repo
+	DestinationTagBase string `yaml:"destination-tag-base,omitempty"`
 	// SmokeTest applies to all branches
 	SmokeTest string `yaml:"smoke-test,omitempty"` // a multiline bash script
 	Library   bool   `yaml:"library,omitempty"`
@@ -81,10 +83,13 @@ type RepositoryRule struct {
 }
 
 type RepositoryRules struct {
-	SkippedSourceBranches []string         `yaml:"skip-source-branches,omitempty"`
-	SkipGomod             bool             `yaml:"skip-gomod,omitempty"`
-	SkipTags              bool             `yaml:"skip-tags,omitempty"`
-	Rules                 []RepositoryRule `yaml:"rules"`
+	SkippedSourceBranches []string `yaml:"skip-source-branches,omitempty"`
+	SkipGomod             bool     `yaml:"skip-gomod,omitempty"`
+	SkipTags              bool     `yaml:"skip-tags,omitempty"`
+	// this skips tags in the source repo that are not valid semver tags
+	// e.g. v1.2.3 is valid, but <prefix>/v1.2.3 is not valid.
+	SkipNonSemverTags bool             `yaml:"skip-non-semver-tags,omitempty"`
+	Rules             []RepositoryRule `yaml:"rules"`
 
 	// ls-files patterns like: */BUILD *.ext pkg/foo.go Makefile
 	RecursiveDeletePatterns []string `yaml:"recursive-delete-patterns"`

cmd/publishing-bot/publisher.go

@@ -292,6 +292,12 @@ func (p *PublisherMunger) construct() error {
 				p.plog.Infof("synchronizing tags is disabled")
 			}
 
+			skipNonSemverTags := "false"
+			if p.reposRules.SkipNonSemverTags {
+				skipNonSemverTags = "true"
+				p.plog.Infof("synchronizing non-semver tags is disabled")
+			}
+
 			// get old published hash to eventually skip cherry picking
 			var lastPublishedUpstreamHash string
 			bs, err := os.ReadFile(path.Join(p.baseRepoPath, publishedFileName(repoRule.DestinationRepository, branchRule.Name)))
@@ -318,6 +324,8 @@ func (p *PublisherMunger) construct() error {
 				strconv.FormatBool(repoRule.Library),
 				strings.Join(p.reposRules.RecursiveDeletePatterns, " "),
 				skipTags,
+				skipNonSemverTags,
+				repoRule.DestinationTagBase,
 				lastPublishedUpstreamHash,
 				p.config.GitDefaultBranch,
 			)

cmd/sync-tags/main.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 	"text/template"
 	"time"
@@ -78,7 +79,10 @@ func main() {
 	dependencies := flag.String("dependencies", "", "comma-separated list of repo:branch pairs of dependencies")
 	skipFetch := flag.Bool("skip-fetch", false, "skip fetching tags")
 	mappingOutputFile := flag.String("mapping-output-file", "", "a file name to write the source->dest hash mapping to ({{.Tag}} is substituted with the tag name, {{.Branch}} with the local branch name)")
-	publishSemverTags := flag.Bool("publish-v0-semver", false, "publish v0.x.y tag at destination repo for v1.x.y tag at the source repo")
+	publishV0Semver := flag.Bool("publish-v0-semver", false, "publish v0.x.y tag at destination repo for v1.x.y tag at the source repo")
+	publishSemverTags := flag.Bool("publish-semver-tags", false, "publish vX.Y.Z tag at destination repo for vX.Y.Z tag at the source repo")
+	skipNonSemverTags := flag.Bool("skip-non-semver-tags", false, "skip non-semver tags at the source repo")
+	semverTagsBase := flag.String("semver-tags-base", "v0", "the value to use as vX in vX.Y.Z published at the destination repo")
 
 	flag.Usage = Usage
 	flag.Parse()
@@ -91,6 +95,10 @@ func main() {
 		glog.Fatalf("source-branch cannot be empty")
 	}
 
+	if *publishV0Semver && *publishSemverTags {
+		glog.Fatalf("only one of publish-v0-semver and publish-semver-tags can be true")
+	}
+
 	var dependentRepos []string
 	if *dependencies != "" {
 		for _, pair := range strings.Split(*dependencies, ",") {
@@ -188,23 +196,37 @@ func main() {
 	// create or update tags from srcTagCommits as local tags with the given prefix
 	createdTags := []string{}
 	for name, kh := range srcTagCommits {
+		if *skipNonSemverTags {
+			if _, semverErr := semver.Parse(strings.TrimPrefix(name, "v")); semverErr != nil {
+				continue
+			}
+		}
+
 		bName := name
 		if *prefix != "" {
 			bName = *prefix + name[1:] // remove the v
 		}
 
 		var (
-			semverTag        = ""
-			publishSemverTag = false
+			semverTag          = ""
+			publishSemverTag   = false
+			versionPrefixRegex = regexp.MustCompile(`^v\d+\.`)
 		)
 		// if we are publishing semver tags
-		if *publishSemverTags {
+		if *publishV0Semver {
 			// and this is a valid v1... semver tag
 			if _, semverErr := semver.Parse(name[1:]); semverErr == nil && strings.HasPrefix(name, "v1.") {
 				publishSemverTag = true
 				semverTag = "v0." + strings.TrimPrefix(name, "v1.") // replace v1.x.y with v0.x.y
 			}
 		}
+		if *publishSemverTags {
+			// and this is a valid semver tag
+			if _, semverErr := semver.Parse(strings.TrimPrefix(name, "v")); semverErr == nil {
+				publishSemverTag = true
+				semverTag = *semverTagsBase + "." + versionPrefixRegex.ReplaceAllString(name, "")
+			}
+		}
 
 		// ignore non-annotated tags
 		tag, err := r.TagObject(kh)
@@ -291,7 +313,7 @@ func main() {
 				fmt.Printf("Writing source->dest hash mapping to %q\n", fname)
 				f, err := os.Create(fname)
 				if err != nil {
-					glog.Fatal(f)
+					glog.Fatal(err)
 				}
 				if err := writeKubeCommitMapping(f, sourceCommitsToDstCommits, srcFirstParents); err != nil {
 					glog.Fatal(err)

pkg/golang/install.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/golang/glog"
@@ -95,7 +96,7 @@ func installGoVersion(v, pth string) error {
 	}
 	defer os.RemoveAll(tmpPath)
 
-	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("curl -SLf https://storage.googleapis.com/golang/go%s.linux-amd64.tar.gz | tar -xz --strip 1 -C %s", v, tmpPath))
+	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("curl -SLf https://storage.googleapis.com/golang/go%s.%s-%s.tar.gz | tar -xz --strip 1 -C %s", v, runtime.GOOS, runtime.GOARCH, tmpPath))
 	cmd.Dir = tmpPath
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr