From afb8562d9a7f6d8ff793a2966e571d4b68fa6038 Mon Sep 17 00:00:00 2001 From: Pavel Efarinov Date: Fri, 18 Jul 2025 16:04:03 +0300 Subject: [PATCH 1/4] ci: add branch sync to prestables Refactored automerge in rightlib to process all branches Refs: #20359, #21069 --- .github/actions/rightlib_sync/action.yml | 17 +- .github/workflows/automerge_pr.yaml | 27 ++- .github/workflows/rightlib_sync.yml | 22 ++- ydb/ci/rightlib/automerge.py | 156 +++++++++++++++ ydb/ci/rightlib/create_sync_pr.py | 149 +++++++++++++++ ydb/ci/rightlib/sync-rightlib.py | 234 ----------------------- 6 files changed, 359 insertions(+), 246 deletions(-) create mode 100755 ydb/ci/rightlib/automerge.py create mode 100755 ydb/ci/rightlib/create_sync_pr.py delete mode 100755 ydb/ci/rightlib/sync-rightlib.py diff --git a/.github/actions/rightlib_sync/action.yml b/.github/actions/rightlib_sync/action.yml index cf31b8b15afb..0af5bfabe417 100644 --- a/.github/actions/rightlib_sync/action.yml +++ b/.github/actions/rightlib_sync/action.yml @@ -1,9 +1,15 @@ name: rightlib sync description: Automatically sync rightlib branch into main inputs: - command: + base_branch: required: true - description: "create-pr or check-pr" + description: "Branch to merge into" + head_branch: + required: true + description: "Branch to merge into base" + label: + required: true + description: "Label to mark your PR with" repository: required: true description: "token for access GitHub" @@ -21,8 +27,9 @@ runs: - name: configure shell: bash run: | - git config --global user.email "alex@ydb.tech" - git config --global user.name "Alexander Smirnov" + git config --global user.name YDBot + git config --global user.email ydbot@ydb.tech + git config --local github.token ${{ inputs.gh_personal_access_token }} - name: run-command shell: bash @@ -31,5 +38,5 @@ runs: TOKEN: ${{ inputs.gh_personal_access_token }} run: | cd ./ydb/ci/rightlib - ./sync-rightlib.py "${{ inputs.command }}" + ./create_sync_pr.py --base-branch="${{ inputs.base_branch }}" --head-branch="${{ inputs.head_branch }}" --process-label="${{ inputs.label }}" diff --git a/.github/workflows/automerge_pr.yaml b/.github/workflows/automerge_pr.yaml index fe6401d1d227..fd0f012e679f 100644 --- a/.github/workflows/automerge_pr.yaml +++ b/.github/workflows/automerge_pr.yaml @@ -6,6 +6,8 @@ on: concurrency: group: ${{ github.workflow }} cancel-in-progress: true +env: + GH_TOKEN: ${{ secrets.YDBOT_TOKEN }} jobs: check-pr: runs-on: ubuntu-latest @@ -16,8 +18,23 @@ jobs: sparse-checkout: | .github ydb/ci/ - - uses: ./.github/actions/rightlib_sync - with: - command: check-pr - repository: ${{ github.repository }} - gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + - name: install packages + shell: bash + run: | + pip install PyGithub==2.5.0 + + - name: configure + shell: bash + run: | + git config user.name YDBot + git config user.email ydbot@ydb.tech + git config --local github.token ${{ env.GH_TOKEN }} + + - name: run-command + shell: bash + env: + REPO: ${{ github.repository }} + TOKEN: ${{ env.GH_TOKEN }} + run: | + cd ./ydb/ci/rightlib + ./automerge.py diff --git a/.github/workflows/rightlib_sync.yml b/.github/workflows/rightlib_sync.yml index b7e89627a5e9..d5190892b24f 100644 --- a/.github/workflows/rightlib_sync.yml +++ b/.github/workflows/rightlib_sync.yml @@ -6,18 +6,36 @@ on: concurrency: group: ${{ github.workflow }} cancel-in-progress: true +env: + GH_TOKEN: ${{ secrets.YDBOT_TOKEN }} jobs: create-pr: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - base_branch: prestable-25-2 + head_branch: stable-25-1 + label: sync-prestable + - base_branch: prestable-25-3 + head_branch: main + label: sync-prestable + - base_branch: main + head_branch: rightlib + label: rightlib steps: - name: checkout uses: actions/checkout@v4 with: + fetch-depth: 0 sparse-checkout: | .github ydb/ci/ - uses: ./.github/actions/rightlib_sync with: - command: create-pr + base_branch: ${{ matrix.base_branch }} + head_branch: ${{ matrix.head_branch }} + label: ${{ matrix.label }} repository: ${{ github.repository }} - gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + gh_personal_access_token: ${{ env.GH_TOKEN }} diff --git a/ydb/ci/rightlib/automerge.py b/ydb/ci/rightlib/automerge.py new file mode 100755 index 000000000000..0358f834195d --- /dev/null +++ b/ydb/ci/rightlib/automerge.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +import os +import datetime +import logging +import subprocess +import argparse +from typing import Optional +from github import Github +from github.PullRequest import PullRequest + +automerge_pr_label = "automerge" +pr_label_fail = "automerge-blocked" +check_name = "checks_integrated" +failed_comment_mark = "" + + +class PrAutomerger: + def __init__(self, repo, token): + self.repo_name = repo + self.token = token + self.gh = Github(login_or_token=self.token) + self.repo = self.gh.get_repo(self.repo_name) + self.dtm = datetime.datetime.now().strftime("%y%m%d-%H%M") + self.logger = logging.getLogger("sync") + self.workflow_url = None + self.detect_env() + + def detect_env(self): + if "GITHUB_RUN_ID" in os.environ: + self.workflow_url = ( + f"{os.environ['GITHUB_SERVER_URL']}/{self.repo_name}/actions/runs/{os.environ['GITHUB_RUN_ID']}" + ) + + def get_latest_open_prs(self) -> Optional[PullRequest]: + query = f"label:{automerge_pr_label} repo:{self.repo_name} is:pr state:open sort:created-desc" + result = self.gh.search_issues(query).get_page(0) + if result: + return result + return None + + def get_commit_check_status(self, sha): + checks = self.repo.get_commit(sha).get_combined_status().statuses + + for c in checks: + if c.context == check_name: + return c + return None + + def check_opened_pr(self, pr: PullRequest): + pr_labels = [l.name for l in pr.labels] + + self.logger.info("check opened pr %r (labels %s)", pr, pr_labels) + + if pr_label_fail in pr_labels: + self.logger.info("pr has %s label, exit", pr_label_fail) + return + + check = self.get_commit_check_status(pr.head.sha) + + if check is None: + self.logger.info("no %r checks found", check_name) + return + + self.logger.info("check result %s", check) + + if check.state == "failure": + self.logger.info("check failed") + self.add_failed_comment(pr, f"Check `{check_name}` failed.") + self.add_pr_failed_label(pr) + return + + elif check.state == "success": + self.logger.info("check success, going to merge") + self.merge_pr(pr) + else: + self.logger.info("wait for success") + + def add_pr_failed_label(self, pr: PullRequest): + pr.add_to_labels(pr_label_fail) + + def git_merge_pr(self, pr: PullRequest): + self.git_run("clone", f"https://{self.token}@github.com/{self.repo_name}.git", "merge-repo") + os.chdir("merge-repo") + self.git_run("fetch", "origin", f"pull/{pr.number}/head:PR") + self.git_run("checkout", pr.base.ref) + + commit_msg = f"Merge pull request #{pr.number} from {pr.head.user.login}/{pr.head.ref}" + try: + self.git_run("merge", "PR", "-m", commit_msg) + except subprocess.CalledProcessError: + self.add_failed_comment(pr, "Unable to merge PR.") + self.add_pr_failed_label(pr) + return False + + try: + self.git_run("push") + except subprocess.CalledProcessError: + self.add_failed_comment(pr, "Unable to push merged revision.") + self.add_pr_failed_label(pr) + return False + + def merge_pr(self, pr: PullRequest): + self.logger.info("start merge %s into %s", pr, pr.base.ref) + if not self.git_merge_pr(pr): + self.logger.info("unable to merge PR") + return + self.logger.info("deleting ref %r", pr.head.ref) + self.repo.get_git_ref(f"heads/{pr.head.ref}").delete() + body = f"The PR was successfully merged into {pr.base.ref} using workflow" + pr.create_issue_comment(body=body) + + def add_failed_comment(self, pr: PullRequest, text: str): + text += f" All future check are suspended, please remove the `{pr_label_fail}` label to enable checks." + if self.workflow_url: + text += f" Sync workflow logs can be found [here]({self.workflow_url})." + pr.create_issue_comment(f"{failed_comment_mark}\n{text}") + + def git_run(self, *args): + args = ["git"] + list(args) + + self.logger.info("run: %r", args) + try: + output = subprocess.check_output(args).decode() + except subprocess.CalledProcessError as e: + self.logger.error(e.output.decode()) + raise + else: + self.logger.info("output:\n%s", output) + return output + + def git_revparse_head(self): + return self.git_run("rev-parse", "HEAD").strip() + + def cmd_check_pr(self): + prs = self.get_latest_open_prs() + + if prs is not None: + for pr in prs: + self.check_opened_pr(pr.as_pull_request()) + else: + self.logger.info("No open PRs found") + + +def main(): + log_fmt = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + logging.basicConfig(format=log_fmt, level=logging.DEBUG) + repo = os.environ["REPO"] + token = os.environ["TOKEN"] + + syncer = PrAutomerger(repo, token) + + syncer.cmd_check_pr() + + +if __name__ == "__main__": + main() diff --git a/ydb/ci/rightlib/create_sync_pr.py b/ydb/ci/rightlib/create_sync_pr.py new file mode 100755 index 000000000000..acf5c2bcb595 --- /dev/null +++ b/ydb/ci/rightlib/create_sync_pr.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +import os +import datetime +import logging +import subprocess +import argparse +from typing import Optional +from github import Github +from github.PullRequest import PullRequest +import automerge + + +class PrSyncCreator: + rightlib_sha_file = "ydb/ci/rightlib.txt" + check_name = "checks_integrated" + failed_comment_mark = "" + + def __init__(self, repo, base_branch, head_branch, token, pr_label, pr_label_failed): + self.repo_name = repo + self.base_branch = base_branch + self.head_branch = head_branch + self.token = token + self.pr_label = pr_label + self.pr_label_fail = pr_label_failed + self.gh = Github(login_or_token=self.token) + self.repo = self.gh.get_repo(self.repo_name) + self.dtm = datetime.datetime.now().strftime("%y%m%d-%H%M") + self.logger = logging.getLogger("sync") + self.workflow_url = None + self.detect_env() + + def detect_env(self): + if "GITHUB_RUN_ID" in os.environ: + self.workflow_url = ( + f"{os.environ['GITHUB_SERVER_URL']}/{self.repo_name}/actions/runs/{os.environ['GITHUB_RUN_ID']}" + ) + + def rightlib_latest_repo_sha(self): + return self.repo.get_branch(self.head_branch).commit.sha + + def is_commit_present_on_branch(self, sha, branch): + try: + command = ["git", "merge-base", "--is-ancestor", sha, 'origin/{}'.format(branch)] + + result = subprocess.run( + command, + capture_output=True, + text=True + ) + + if result.returncode == 0: + return True + elif result.returncode == 1: + return False + else: + self.logger.warning(f"Command '{' '.join(command)}' finished with error:") + self.logger.warning(f"Exit code: {result.returncode}") + self.logger.warning(f"Stderr: {result.stderr.strip()}") + return None + except Exception as e: + self.logger.error(f"Exception occured while git merge-base: {e}") + return None + + def get_latest_open_pr(self) -> Optional[PullRequest]: + query = f"label:{self.pr_label} repo:{self.repo_name} base:{self.base_branch} label:{automerge.automerge_pr_label} is:pr state:open sort:created-desc" + result = self.gh.search_issues(query).get_page(0) + if result: + return result[0].as_pull_request() + return None + + def git_run(self, *args): + args = ["git"] + list(args) + + self.logger.info("run: %r", args) + try: + output = subprocess.check_output(args).decode() + except subprocess.CalledProcessError as e: + self.logger.error(e.output.decode()) + raise + else: + self.logger.info("output:\n%s", output) + return output + + def git_revparse_head(self): + return self.git_run("rev-parse", "HEAD").strip() + + def create_new_pr(self): + dev_branch_name = f"merge-{self.head_branch}-{self.dtm}" + commit_msg = f"Sync branches {self.dtm}" + pr_title = f"Sync branches {self.dtm}: {self.head_branch} to {self.base_branch}" + + self.git_run("clone", f"https://{self.token}@github.com/{self.repo_name}.git", "ydb-new-pr") + os.chdir("ydb-new-pr") + self.git_run("checkout", self.head_branch) + rightlib_sha = self.git_revparse_head() + + self.logger.info(f"{rightlib_sha=}") + + self.git_run("checkout", self.base_branch) + self.git_run("checkout", "-b", dev_branch_name) + + self.git_run("merge", self.head_branch, "-m", commit_msg) + self.git_run("push", "--set-upstream", "origin", dev_branch_name) + + if self.workflow_url: + pr_body = f"PR was created by rightlib sync workflow [run]({self.workflow_url})" + else: + pr_body = "PR was created by rightlib sync script" + + pr = self.repo.create_pull( + self.base_branch, dev_branch_name, title=pr_title, body=pr_body, maintainer_can_modify=True + ) + pr.add_to_labels(self.pr_label) + pr.add_to_labels(automerge.automerge_pr_label) + + def cmd_create_pr(self): + pr = self.get_latest_open_pr() + + if not pr: + cur_sha = self.rightlib_latest_repo_sha() + self.logger.info("cur_sha=%s", cur_sha) + + if self.is_commit_present_on_branch(cur_sha, self.base_branch) is False: + self.create_new_pr() + else: + self.logger.info("Skipping create-pr because base branch is up-to-date") + else: + self.logger.info("Skipping create-pr because an open PR was found") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-branch", help="Branch to merge into") + parser.add_argument("--head-branch", help="Branch to be merged") + parser.add_argument("--process-label", help="Label to filter PRs") + args = parser.parse_args() + + log_fmt = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + logging.basicConfig(format=log_fmt, level=logging.DEBUG) + repo = os.environ["REPO"] + token = os.environ["TOKEN"] + + syncer = PrSyncCreator(repo, args.base_branch, args.head_branch, token, args.process_label, f'{args.process_label}-fail') + + syncer.cmd_create_pr() + + +if __name__ == "__main__": + main() diff --git a/ydb/ci/rightlib/sync-rightlib.py b/ydb/ci/rightlib/sync-rightlib.py deleted file mode 100755 index b89d7e57560a..000000000000 --- a/ydb/ci/rightlib/sync-rightlib.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python3 -import os -import datetime -import logging -import subprocess -import argparse -from typing import Optional -from github import Github -from github.PullRequest import PullRequest - - -class RightlibSync: - pr_label_rightlib = "rightlib" - pr_label_fail = "rightlib-fail" - rightlib_sha_file = "ydb/ci/rightlib.txt" - check_name = "checks_integrated" - failed_comment_mark = "" - rightlib_check_status_name = "rightlib-merge" - - def __init__(self, repo, base_branch, head_branch, token): - self.repo_name = repo - self.base_branch = base_branch - self.head_branch = head_branch - self.token = token - self.gh = Github(login_or_token=self.token) - self.repo = self.gh.get_repo(self.repo_name) - self.dtm = datetime.datetime.now().strftime("%y%m%d-%H%M") - self.logger = logging.getLogger("sync") - self.workflow_url = None - self.detect_env() - - def detect_env(self): - if "GITHUB_RUN_ID" in os.environ: - self.workflow_url = ( - f"{os.environ['GITHUB_SERVER_URL']}/{self.repo_name}/actions/runs/{os.environ['GITHUB_RUN_ID']}" - ) - - def rightlib_latest_repo_sha(self): - return self.repo.get_branch(self.head_branch).commit.sha - - def rightlib_sha_file_contents(self, ref): - return self.repo.get_contents(self.rightlib_sha_file, ref=ref).decoded_content.decode().strip() - - def rightlib_latest_sync_commit(self): - return self.rightlib_sha_file_contents(ref=self.base_branch) - - def get_latest_open_pr(self) -> Optional[PullRequest]: - query = f"label:{self.pr_label_rightlib} repo:{self.repo_name} base:{self.base_branch} is:pr state:open sort:created-desc" - result = self.gh.search_issues(query).get_page(0) - if result: - return result[0].as_pull_request() - return None - - def get_commit_check_status(self, sha): - checks = self.repo.get_commit(sha).get_combined_status().statuses - - for c in checks: - if c.context == self.check_name: - return c - return None - - def check_opened_pr(self, pr: PullRequest): - pr_labels = [l.name for l in pr.labels] - - self.logger.info("check opened pr %r (labels %s)", pr, pr_labels) - - if self.pr_label_fail in pr_labels: - self.logger.info("pr has %s label, exit", self.pr_label_fail) - return - - check = self.get_commit_check_status(pr.head.sha) - - if check is None: - self.logger.info("no %r checks found", self.check_name) - return - - self.logger.info("check result %s", check) - - if check.state == "failure": - self.logger.info("check failed") - self.add_failed_comment(pr, f"Check `{self.check_name}` failed.") - self.add_pr_failed_label(pr) - return - - elif check.state == "success": - self.logger.info("check success, going to merge") - self.merge_pr(pr) - else: - self.logger.info("wait for success") - - def add_pr_failed_label(self, pr: PullRequest): - pr.add_to_labels(self.pr_label_fail) - - def git_merge_pr(self, pr: PullRequest): - self.git_run("clone", f"https://{self.token}@github.com/{self.repo_name}.git", "merge-repo") - os.chdir("merge-repo") - self.git_run("fetch", "origin", f"pull/{pr.number}/head:PR") - self.git_run("checkout", self.base_branch) - - commit_msg = f"Merge pull request #{pr.number} from {pr.head.user.login}/{pr.head.ref}" - try: - self.git_run("merge", "PR", "-m", commit_msg) - except subprocess.CalledProcessError: - self.add_failed_comment(pr, "Unable to merge PR.") - self.add_pr_failed_label(pr) - return False - - try: - self.git_run("push") - except subprocess.CalledProcessError: - self.add_failed_comment(pr, "Unable to push merged revision.") - self.add_pr_failed_label(pr) - return False - - def merge_pr(self, pr: PullRequest): - self.logger.info("start merge %s into main", pr) - if not self.git_merge_pr(pr): - self.logger.info("unable to merge PR") - return - self.logger.info("deleting ref %r", pr.head.ref) - self.repo.get_git_ref(f"heads/{pr.head.ref}").delete() - body = f"The PR was successfully merged into main using workflow" - pr.create_issue_comment(body=body) - - def add_failed_comment(self, pr: PullRequest, text: str): - text += f" All future check are suspended, please remove the `{self.pr_label_fail}` label to enable checks." - if self.workflow_url: - text += f" Rightlib sync workflow logs can be found [here]({self.workflow_url})." - pr.create_issue_comment(f"{self.failed_comment_mark}\n{text}") - - def git_run(self, *args): - args = ["git"] + list(args) - - self.logger.info("run: %r", args) - try: - output = subprocess.check_output(args).decode() - except subprocess.CalledProcessError as e: - self.logger.error(e.output.decode()) - raise - else: - self.logger.info("output:\n%s", output) - return output - - def git_revparse_head(self): - return self.git_run("rev-parse", "HEAD").strip() - - def create_new_pr(self): - dev_branch_name = f"merge-libs-{self.dtm}" - commit_msg = f"Import libraries {self.dtm}" - pr_title = f"Library import {self.dtm}" - - self.git_run("clone", f"https://{self.token}@github.com/{self.repo_name}.git", "ydb-new-pr") - os.chdir("ydb-new-pr") - self.git_run("checkout", self.head_branch) - rightlib_sha = self.git_revparse_head() - - self.logger.info(f"{rightlib_sha=}") - - self.git_run("checkout", self.base_branch) - self.git_run(f"checkout", "-b", dev_branch_name) - - prev_sha = self.git_revparse_head() - - self.git_run("merge", self.head_branch, "--no-edit") - - cur_sha = self.git_revparse_head() - - if prev_sha == cur_sha: - logging.info("Merge did not bring any changes, exiting") - return - - with open(self.rightlib_sha_file, "w") as fp: - fp.write(f"{rightlib_sha}\n") - - self.git_run("add", ".") - self.git_run("commit", "-m", commit_msg) - self.git_run("push", "--set-upstream", "origin", dev_branch_name) - - if self.workflow_url: - pr_body = f"PR was created by rightlib sync workflow [run]({self.workflow_url})" - else: - pr_body = f"PR was created by rightlib sync script" - - pr = self.repo.create_pull( - self.base_branch, dev_branch_name, title=pr_title, body=pr_body, maintainer_can_modify=True - ) - pr.add_to_labels(self.pr_label_rightlib) - - def cmd_check_pr(self): - pr = self.get_latest_open_pr() - - if pr: - self.check_opened_pr(pr) - else: - self.logger.info("No open PR found") - - def cmd_create_pr(self): - pr = self.get_latest_open_pr() - - if not pr: - cur_sha = self.rightlib_latest_repo_sha() - latest_sha = self.rightlib_latest_sync_commit() - self.logger.info("cur_sha=%s", cur_sha) - self.logger.info("latest_sha=%s", latest_sha) - - if cur_sha != latest_sha: - self.create_new_pr() - else: - self.logger.info("Skipping create-pr because an open PR was found") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("create-pr", "check-pr")) - args = parser.parse_args() - - log_fmt = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" - logging.basicConfig(format=log_fmt, level=logging.DEBUG) - repo = os.environ["REPO"] - token = os.environ["TOKEN"] - - syncer = RightlibSync(repo, "main", "rightlib", token) - - if args.command == "create-pr": - syncer.cmd_create_pr() - elif args.command == "check-pr": - syncer.cmd_check_pr() - else: - print(f"Unknown command {args.command!r}") - raise SystemExit(1) - - -if __name__ == "__main__": - main() From 3718a65865ad27d6d971ce3765f0c518b84059d7 Mon Sep 17 00:00:00 2001 From: Pavel Efarinov Date: Mon, 21 Jul 2025 16:56:45 +0300 Subject: [PATCH 2/4] ci: move stable branches to file Moved stable branches for nightly and regression --- .github/config/stable_branches.json | 20 +++++++++++++++++++ .github/workflows/automerge_pr.yaml | 4 ++-- .github/workflows/nightly_build.yml | 12 ++++++++--- .github/workflows/regression_run.yml | 4 +++- .github/workflows/regression_run_large.yml | 4 +++- .../workflows/regression_run_small_medium.yml | 4 +++- .github/workflows/regression_run_stress.yml | 4 +++- .../workflows/regression_whitelist_run.yml | 4 +++- .github/workflows/run_tests.yml | 20 +++++++++++++++++++ ydb/ci/rightlib/automerge.py | 1 + 10 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 .github/config/stable_branches.json diff --git a/.github/config/stable_branches.json b/.github/config/stable_branches.json new file mode 100644 index 000000000000..1a483089ade1 --- /dev/null +++ b/.github/config/stable_branches.json @@ -0,0 +1,20 @@ +{ + "stables_for_nightly": [ + "main", + "stable-25-1", + "stable-24-4-hotfix", + "stable-25-1-1", + "stable-25-1-analytics", + "stable-25-1-2", + "stable-25-1-3", + "prestable-25-2", + "prestable-25-3" + ], + "stables_for_tests": [ + "main", + "stable-25-1", + "stable-25-1-analytics", + "prestable-25-2", + "prestable-25-3" + ] +} \ No newline at end of file diff --git a/.github/workflows/automerge_pr.yaml b/.github/workflows/automerge_pr.yaml index fd0f012e679f..b2b81dd6d7dd 100644 --- a/.github/workflows/automerge_pr.yaml +++ b/.github/workflows/automerge_pr.yaml @@ -26,8 +26,8 @@ jobs: - name: configure shell: bash run: | - git config user.name YDBot - git config user.email ydbot@ydb.tech + git config --global user.name YDBot + git config --global user.email ydbot@ydb.tech git config --local github.token ${{ env.GH_TOKEN }} - name: run-command diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index babca453c278..43354c8645da 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -18,10 +18,16 @@ jobs: outputs: branches: ${{ steps.set-branches.outputs.branches }} steps: - - id: set-branches - run: | + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/config/stable_branches.json + - name: Read branches from config + id: set-branches + run: | if [[ "${{ github.event_name }}" == "schedule" || ("${{ inputs.use_default_branches }}" == "true" && "${{ github.ref_type }}" != "tag") ]]; then - echo "branches=['main', 'stable-25-1', 'stable-24-4', 'stable-25-1-1', 'stable-25-1-analytics', 'stable-25-1-2', 'stable-25-1-3', 'prestable-25-2', 'prestable-25-3']" >> $GITHUB_OUTPUT + echo "branches=$(jq -c '.stables_for_nightly' .github/config/stable_branches.json)" >> $GITHUB_OUTPUT else echo "branches=['${{ github.ref_name }}']" >> $GITHUB_OUTPUT fi diff --git a/.github/workflows/regression_run.yml b/.github/workflows/regression_run.yml index 60b90c544353..d3663f335c1c 100644 --- a/.github/workflows/regression_run.yml +++ b/.github/workflows/regression_run.yml @@ -23,5 +23,7 @@ jobs: build_preset: ["relwithdebinfo", "release-asan", "release-tsan", "release-msan"] with: test_targets: ydb/ - branches: ${{ (inputs.use_default_branches == true || github.event_name == 'schedule') && '["main", "stable-25-1", "stable-25-1-analytics", "prestable-25-2", "prestable-25-3"]' || github.ref_name }} + branches: '' + branches_config_path: '.github/config/stable_branches.json' + branches_config_set: '.stables_for_tests' build_preset: ${{ matrix.build_preset }} \ No newline at end of file diff --git a/.github/workflows/regression_run_large.yml b/.github/workflows/regression_run_large.yml index c721ef2f5aed..561e3b0786b4 100644 --- a/.github/workflows/regression_run_large.yml +++ b/.github/workflows/regression_run_large.yml @@ -23,5 +23,7 @@ jobs: with: test_targets: ydb/ test_size: large - branches: ${{ (inputs.use_default_branches == true || github.event_name == 'schedule') && '["main", "stable-25-1", "stable-25-1-analytics", "prestable-25-2", "prestable-25-3"]' || github.ref_name }} + branches: '' + branches_config_path: '.github/config/stable_branches.json' + branches_config_set: '.stables_for_tests' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/regression_run_small_medium.yml b/.github/workflows/regression_run_small_medium.yml index e4b982d4e53f..925b0be23025 100644 --- a/.github/workflows/regression_run_small_medium.yml +++ b/.github/workflows/regression_run_small_medium.yml @@ -23,5 +23,7 @@ jobs: with: test_targets: ydb/ test_size: small,medium - branches: ${{ (inputs.use_default_branches == true || github.event_name == 'schedule') && '["main", "stable-25-1", "stable-25-1-analytics", "prestable-25-2", "prestable-25-3"]' || github.ref_name }} + branches: '' + branches_config_path: '.github/config/stable_branches.json' + branches_config_set: '.stables_for_tests' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/regression_run_stress.yml b/.github/workflows/regression_run_stress.yml index ba2cacf9bbc8..b76af3ef8e56 100644 --- a/.github/workflows/regression_run_stress.yml +++ b/.github/workflows/regression_run_stress.yml @@ -22,5 +22,7 @@ jobs: build_preset: ["relwithdebinfo", "release-asan", "release-tsan", "release-msan"] with: test_targets: ydb/tests/stress/ - branches: ${{ (inputs.use_default_branches == true || github.event_name == 'schedule') && '["main", "stable-25-1", "stable-25-1-analytics", "prestable-25-2", "prestable-25-3"]' || github.ref_name }} + branches: '' + branches_config_path: '.github/config/stable_branches.json' + branches_config_set: '.stables_for_tests' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/regression_whitelist_run.yml b/.github/workflows/regression_whitelist_run.yml index caad02eaacf6..27014eb17192 100644 --- a/.github/workflows/regression_whitelist_run.yml +++ b/.github/workflows/regression_whitelist_run.yml @@ -22,5 +22,7 @@ jobs: build_preset: ["relwithdebinfo", "release-asan", "release-tsan", "release-msan"] with: test_targets: ydb/tests/sql/ ydb/tests/stress ydb/tests/functional/tpc ydb/tests/functional/benchmarks_init - branches: ${{ (inputs.use_default_branches == true || github.event_name == 'schedule') && '["main", "stable-25-1", "stable-25-1-analytics", "prestable-25-2", "prestable-25-3"]' || github.ref_name }} + branches: '' + branches_config_path: '.github/config/stable_branches.json' + branches_config_set: '.stables_for_tests' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ad52813d5d42..752a32cae4bb 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -32,6 +32,16 @@ on: required: false type: string default: '["main"]' + branches_config_path: + description: 'Path to JSON file with branches to test' + required: false + type: string + default: '' + branches_config_set: + description: 'Property in file with branches to use' + required: false + type: string + default: '' workflow_dispatch: inputs: @@ -75,10 +85,18 @@ jobs: outputs: branch_array: ${{ steps.set-branches.outputs.branch_array }} steps: + - name: Checkout ${{ matrix.branch }} + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/config/stable_branches.json + - name: Set branches id: set-branches env: CALLED_BRANCHES: '${{ inputs.branches }}' + BRANCHES_CONFIG_PATH: '${{ inputs.branches_config_path }}' + BRANCHES_CONFIG_SET: '${{ inputs.branches_config_set }}' run: | # Проверяем, был ли передан параметр branches из вызывающего workflow if [[ -n "$CALLED_BRANCHES" ]]; then @@ -91,6 +109,8 @@ jobs: # Если это одна ветка, создаем JSON-массив с одним элементом echo "branch_array=[\"$CALLED_BRANCHES\"]" >> $GITHUB_OUTPUT fi + elif [[ -n "$BRANCHES_CONFIG_PATH" ]]; then + echo "branch_array=$(jq -c "$BRANCHES_CONFIG_SET" "$BRANCHES_CONFIG_PATH")" >> $GITHUB_OUTPUT else # Если ветки не переданы, значит это прямой запуск workflow_dispatch echo "No branches specified, using current branch: ${{ github.ref_name }}" diff --git a/ydb/ci/rightlib/automerge.py b/ydb/ci/rightlib/automerge.py index 0358f834195d..32249dcc7b7a 100755 --- a/ydb/ci/rightlib/automerge.py +++ b/ydb/ci/rightlib/automerge.py @@ -94,6 +94,7 @@ def git_merge_pr(self, pr: PullRequest): try: self.git_run("push") + return True except subprocess.CalledProcessError: self.add_failed_comment(pr, "Unable to push merged revision.") self.add_pr_failed_label(pr) From 75fa7e5b60a9a8821db260f86ef560945778a461 Mon Sep 17 00:00:00 2001 From: Pavel Efarinov Date: Wed, 23 Jul 2025 15:06:00 +0300 Subject: [PATCH 3/4] ci: separate branches config into multiple files --- .github/config/stable_branches.json | 20 ------------------- .github/config/stable_nightly_branches.json | 11 ++++++++++ .github/config/stable_tests_branches.json | 7 +++++++ .github/workflows/nightly_build.yml | 2 +- .github/workflows/regression_run.yml | 3 +-- .github/workflows/regression_run_large.yml | 3 +-- .../workflows/regression_run_small_medium.yml | 3 +-- .github/workflows/regression_run_stress.yml | 3 +-- .../workflows/regression_whitelist_run.yml | 3 +-- .github/workflows/run_tests.yml | 8 +------- 10 files changed, 25 insertions(+), 38 deletions(-) delete mode 100644 .github/config/stable_branches.json create mode 100644 .github/config/stable_nightly_branches.json create mode 100644 .github/config/stable_tests_branches.json diff --git a/.github/config/stable_branches.json b/.github/config/stable_branches.json deleted file mode 100644 index 1a483089ade1..000000000000 --- a/.github/config/stable_branches.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "stables_for_nightly": [ - "main", - "stable-25-1", - "stable-24-4-hotfix", - "stable-25-1-1", - "stable-25-1-analytics", - "stable-25-1-2", - "stable-25-1-3", - "prestable-25-2", - "prestable-25-3" - ], - "stables_for_tests": [ - "main", - "stable-25-1", - "stable-25-1-analytics", - "prestable-25-2", - "prestable-25-3" - ] -} \ No newline at end of file diff --git a/.github/config/stable_nightly_branches.json b/.github/config/stable_nightly_branches.json new file mode 100644 index 000000000000..54c3104e426a --- /dev/null +++ b/.github/config/stable_nightly_branches.json @@ -0,0 +1,11 @@ +[ + "main", + "stable-25-1", + "stable-24-4-hotfix", + "stable-25-1-1", + "stable-25-1-analytics", + "stable-25-1-2", + "stable-25-1-3", + "prestable-25-2", + "prestable-25-3" +] \ No newline at end of file diff --git a/.github/config/stable_tests_branches.json b/.github/config/stable_tests_branches.json new file mode 100644 index 000000000000..c0fd962bfb76 --- /dev/null +++ b/.github/config/stable_tests_branches.json @@ -0,0 +1,7 @@ +[ + "main", + "stable-25-1", + "stable-25-1-analytics", + "prestable-25-2", + "prestable-25-3" +] \ No newline at end of file diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 43354c8645da..ffae2651f7b2 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -27,7 +27,7 @@ jobs: id: set-branches run: | if [[ "${{ github.event_name }}" == "schedule" || ("${{ inputs.use_default_branches }}" == "true" && "${{ github.ref_type }}" != "tag") ]]; then - echo "branches=$(jq -c '.stables_for_nightly' .github/config/stable_branches.json)" >> $GITHUB_OUTPUT + echo "branches=$(jq -c '.' .github/config/stable_nightly_branches.json)" >> $GITHUB_OUTPUT else echo "branches=['${{ github.ref_name }}']" >> $GITHUB_OUTPUT fi diff --git a/.github/workflows/regression_run.yml b/.github/workflows/regression_run.yml index d3663f335c1c..615f8cefeb8f 100644 --- a/.github/workflows/regression_run.yml +++ b/.github/workflows/regression_run.yml @@ -24,6 +24,5 @@ jobs: with: test_targets: ydb/ branches: '' - branches_config_path: '.github/config/stable_branches.json' - branches_config_set: '.stables_for_tests' + branches_config_path: '.github/config/stable_tests_branches.json' build_preset: ${{ matrix.build_preset }} \ No newline at end of file diff --git a/.github/workflows/regression_run_large.yml b/.github/workflows/regression_run_large.yml index 561e3b0786b4..a8e1d64f2bdb 100644 --- a/.github/workflows/regression_run_large.yml +++ b/.github/workflows/regression_run_large.yml @@ -24,6 +24,5 @@ jobs: test_targets: ydb/ test_size: large branches: '' - branches_config_path: '.github/config/stable_branches.json' - branches_config_set: '.stables_for_tests' + branches_config_path: '.github/config/stable_tests_branches.json' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/regression_run_small_medium.yml b/.github/workflows/regression_run_small_medium.yml index 925b0be23025..5d59715f162a 100644 --- a/.github/workflows/regression_run_small_medium.yml +++ b/.github/workflows/regression_run_small_medium.yml @@ -24,6 +24,5 @@ jobs: test_targets: ydb/ test_size: small,medium branches: '' - branches_config_path: '.github/config/stable_branches.json' - branches_config_set: '.stables_for_tests' + branches_config_path: '.github/config/stable_tests_branches.json' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/regression_run_stress.yml b/.github/workflows/regression_run_stress.yml index b76af3ef8e56..011272feba67 100644 --- a/.github/workflows/regression_run_stress.yml +++ b/.github/workflows/regression_run_stress.yml @@ -23,6 +23,5 @@ jobs: with: test_targets: ydb/tests/stress/ branches: '' - branches_config_path: '.github/config/stable_branches.json' - branches_config_set: '.stables_for_tests' + branches_config_path: '.github/config/stable_tests_branches.json' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/regression_whitelist_run.yml b/.github/workflows/regression_whitelist_run.yml index 27014eb17192..80f210b2c740 100644 --- a/.github/workflows/regression_whitelist_run.yml +++ b/.github/workflows/regression_whitelist_run.yml @@ -23,6 +23,5 @@ jobs: with: test_targets: ydb/tests/sql/ ydb/tests/stress ydb/tests/functional/tpc ydb/tests/functional/benchmarks_init branches: '' - branches_config_path: '.github/config/stable_branches.json' - branches_config_set: '.stables_for_tests' + branches_config_path: '.github/config/stable_tests_branches.json' build_preset: ${{ matrix.build_preset }} diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 752a32cae4bb..4dfbd0407685 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -37,11 +37,6 @@ on: required: false type: string default: '' - branches_config_set: - description: 'Property in file with branches to use' - required: false - type: string - default: '' workflow_dispatch: inputs: @@ -96,7 +91,6 @@ jobs: env: CALLED_BRANCHES: '${{ inputs.branches }}' BRANCHES_CONFIG_PATH: '${{ inputs.branches_config_path }}' - BRANCHES_CONFIG_SET: '${{ inputs.branches_config_set }}' run: | # Проверяем, был ли передан параметр branches из вызывающего workflow if [[ -n "$CALLED_BRANCHES" ]]; then @@ -110,7 +104,7 @@ jobs: echo "branch_array=[\"$CALLED_BRANCHES\"]" >> $GITHUB_OUTPUT fi elif [[ -n "$BRANCHES_CONFIG_PATH" ]]; then - echo "branch_array=$(jq -c "$BRANCHES_CONFIG_SET" "$BRANCHES_CONFIG_PATH")" >> $GITHUB_OUTPUT + echo "branch_array=$(jq -c "." "$BRANCHES_CONFIG_PATH")" >> $GITHUB_OUTPUT else # Если ветки не переданы, значит это прямой запуск workflow_dispatch echo "No branches specified, using current branch: ${{ github.ref_name }}" From 0ce0f76ca39e91f44fc7a96991f43801ae4c74fe Mon Sep 17 00:00:00 2001 From: Pavel Efarinov Date: Wed, 23 Jul 2025 15:06:41 +0300 Subject: [PATCH 4/4] ci: add separate workflow for prestable sync Refs: #20359, #21069 --- .github/workflows/prestable_sync.yml | 38 +++++++++++++++++++++ .github/workflows/rebase_prestable.yml | 47 -------------------------- .github/workflows/rightlib_sync.yml | 19 ++--------- 3 files changed, 41 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/prestable_sync.yml delete mode 100644 .github/workflows/rebase_prestable.yml diff --git a/.github/workflows/prestable_sync.yml b/.github/workflows/prestable_sync.yml new file mode 100644 index 000000000000..aa5e12363a3a --- /dev/null +++ b/.github/workflows/prestable_sync.yml @@ -0,0 +1,38 @@ +name: Sync prestables +on: + schedule: + - cron: "30 0 * * *" # At 00:30 -- for PR creation + workflow_dispatch: +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true +env: + GH_TOKEN: ${{ secrets.YDBOT_TOKEN }} +jobs: + create-pr: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - base_branch: prestable-25-2 + head_branch: stable-25-1 + label: sync-prestable + - base_branch: prestable-25-3 + head_branch: main + label: sync-prestable + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + sparse-checkout: | + .github + ydb/ci/ + - uses: ./.github/actions/rightlib_sync + with: + base_branch: ${{ matrix.base_branch }} + head_branch: ${{ matrix.head_branch }} + label: ${{ matrix.label }} + repository: ${{ github.repository }} + gh_personal_access_token: ${{ env.GH_TOKEN }} diff --git a/.github/workflows/rebase_prestable.yml b/.github/workflows/rebase_prestable.yml deleted file mode 100644 index 643eac39158f..000000000000 --- a/.github/workflows/rebase_prestable.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Sync from stable - -on: - schedule: - - cron: '0 5 * * *' # At 05:00 every day - workflow_dispatch: - -permissions: - contents: write -env: - GH_TOKEN: ${{ secrets.YDBOT_TOKEN }} - -jobs: - sync-branches: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - prestable_branch: prestable-25-2 - stable_branch: stable-25-1 - - prestable_branch: prestable-25-3 - stable_branch: main - - steps: - - name: Verify prestable branch - if: ${{ ! startsWith(matrix.prestable_branch, 'pre') }} - run: | - echo "Dangerous prestable branch name, expected prefix 'pre'" - exit 1 - - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Git user - run: | - git config user.name YDBot - git config user.email ydbot@ydb.tech - git config --local github.token ${{ env.GH_TOKEN }} - - - name: Sync ${{ matrix.stable_branch }} to ${{ matrix.prestable_branch }} - run: | - git checkout ${{ matrix.prestable_branch }} - git merge --no-ff origin/${{ matrix.stable_branch }} - git push origin ${{ matrix.prestable_branch }} \ No newline at end of file diff --git a/.github/workflows/rightlib_sync.yml b/.github/workflows/rightlib_sync.yml index d5190892b24f..15ea49e42915 100644 --- a/.github/workflows/rightlib_sync.yml +++ b/.github/workflows/rightlib_sync.yml @@ -11,19 +11,6 @@ env: jobs: create-pr: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - base_branch: prestable-25-2 - head_branch: stable-25-1 - label: sync-prestable - - base_branch: prestable-25-3 - head_branch: main - label: sync-prestable - - base_branch: main - head_branch: rightlib - label: rightlib steps: - name: checkout uses: actions/checkout@v4 @@ -34,8 +21,8 @@ jobs: ydb/ci/ - uses: ./.github/actions/rightlib_sync with: - base_branch: ${{ matrix.base_branch }} - head_branch: ${{ matrix.head_branch }} - label: ${{ matrix.label }} + base_branch: main + head_branch: rightlib + label: rightlib repository: ${{ github.repository }} gh_personal_access_token: ${{ env.GH_TOKEN }}