From 30d4bd556084587b17b2419469b2efffb3bc62c1 Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Thu, 7 Aug 2025 13:40:29 -0700 Subject: [PATCH 1/2] Update buildkite script to look for new logstash releases location (#358) (cherry picked from commit febb20786b9fe4bc7cdf145687ea1a1c5ed01a91) # Conflicts: # .buildkite/scripts/build-pipeline/generate-steps.py # .buildkite/scripts/e2e-pipeline/generate-steps.py # .buildkite/scripts/pull-request-pipeline/generate-steps.py # .buildkite/scripts/run_tests.sh --- .../scripts/build-pipeline/generate-steps.py | 23 ++++++++++++++++--- .../scripts/e2e-pipeline/generate-steps.py | 18 ++++++++++++++- .../pull-request-pipeline/generate-steps.py | 18 +++++++++++---- .buildkite/scripts/run_tests.sh | 19 ++++++++++++++- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index 3c166410..d4fc8edf 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -6,7 +6,7 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -54,13 +54,15 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque structure = { "agents": { "provider": "gcp", - "machineType": "n1-standard-4", - "image": "family/core-ubuntu-2204" + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204" }, "steps": []} steps = [] response = call_url_with_retry(RELEASES_URL) +<<<<<<< HEAD versions_json = response.json() # there are situations to manually run CIs with PR change, @@ -81,6 +83,21 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque if releases[release_version].startswith(target_branch): steps += generate_unit_and_integration_test_steps(releases[release_version], "false") break +======= + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) + + # Use BUILDKITE_SOURCE to figure out PR merge or schedule. + # If PR merge, no need to run builds on all branches, target branch will be good + # - webhook when PR gets merged + # - schedule when daily schedule starts + # - ui when manually kicking job from BK UI + # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH + bk_source = os.getenv("BUILDKITE_SOURCE") + bk_branch = os.getenv("BUILDKITE_BRANCH") + steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_yaml) +>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) # steps for snapshot version snapshots = versions_json["snapshots"] diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py index 33fe1f80..9a78a326 100644 --- a/.buildkite/scripts/e2e-pipeline/generate-steps.py +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -6,7 +6,7 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh" @@ -45,6 +45,7 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: steps = [] response = call_url_with_retry(RELEASES_URL) +<<<<<<< HEAD versions_json = response.json() # there are situations to manually run CIs with PR change, @@ -65,6 +66,21 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: if releases[release_version].startswith(target_branch): steps.append(generate_test_step(releases[release_version], target_branch, "false")) break +======= + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) + + # Use BUILDKITE_SOURCE to figure out PR merge or schedule. + # If PR merge, no need to run builds on all branches, target branch will be good + # - webhook when PR gets merged + # - schedule when daily schedule starts + # - ui when manually kicking job from BK UI + # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH + bk_source = os.getenv("BUILDKITE_SOURCE") + bk_branch = os.getenv("BUILDKITE_BRANCH") + steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_yaml) +>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) # steps for snapshot version snapshots = versions_json["snapshots"] diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index de431089..5cf013e4 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -6,8 +6,14 @@ from ruamel.yaml import YAML +<<<<<<< HEAD RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml" +======= +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" +TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \ + "-request-test-matrix.yml" +>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -60,14 +66,16 @@ def make_matrix_version_key(branch: str) -> str: structure = { "agents": { "provider": "gcp", - "machineType": "n1-standard-4", - "image": "family/core-ubuntu-2204" + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204" }, "steps": []} steps = [] response = call_url_with_retry(RELEASES_URL) - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) matrix_map = call_url_with_retry(TEST_MATRIX_URL) matrix_map_yaml = YAML().load(matrix_map.text) @@ -87,13 +95,13 @@ def make_matrix_version_key(branch: str) -> str: print(f"matrix_releases: {matrix_releases}") print(f"matrix_snapshots: {matrix_snapshots}") for matrix_release in matrix_releases: - full_stack_version: typing.final = versions_json["releases"].get(matrix_release) + full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release) # noop, if they are declared in the matrix but not in the release if full_stack_version is not None: steps += generate_unit_and_integration_test_steps(full_stack_version, "false") for matrix_snapshot in matrix_snapshots: - full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot) + full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot) # noop, if they are declared in the matrix but not in the snapshot if full_stack_version is not None: steps += generate_unit_and_integration_test_steps(full_stack_version, "true") diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index b462caf4..446a6a00 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1 +1,18 @@ -mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh \ No newline at end of file +<<<<<<< HEAD +mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh +======= +#!/usr/bin/env bash + +export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21" +export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH" +eval "$(rbenv init -)" + +if [ -z "$TARGET_BRANCH" ]; then + echo "Target branch is not specified, using default branch: main or BK defined" +else + echo "Changing the branch for ${TARGET_BRANCH}" + git checkout "$TARGET_BRANCH" +fi + +mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh +>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) From 17d64b7c76a6f6ab21d559853465e5a657fc5613 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 7 Aug 2025 15:13:47 -0700 Subject: [PATCH 2/2] fix merge conflicts --- .../scripts/build-pipeline/generate-steps.py | 28 ++++--------------- .../scripts/e2e-pipeline/generate-steps.py | 28 ++++--------------- .../pull-request-pipeline/generate-steps.py | 5 ---- .buildkite/scripts/run_tests.sh | 4 --- 4 files changed, 12 insertions(+), 53 deletions(-) diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index d4fc8edf..145f6f0e 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -62,8 +62,8 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque steps = [] response = call_url_with_retry(RELEASES_URL) -<<<<<<< HEAD - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) # there are situations to manually run CIs with PR change, # set MANUAL_TARGET_BRANCH with upstream target branch and run @@ -71,36 +71,20 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH") print(f"Running with target_branch: {target_branch}") if target_branch == '8.x': - full_stack_version: typing.final = versions_json["snapshots"]["8.future"] + full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"] steps += generate_unit_and_integration_test_steps(full_stack_version, "true") elif target_branch == 'main': - full_stack_version: typing.final = versions_json["snapshots"][target_branch] + full_stack_version: typing.final = versions_yaml["snapshots"][target_branch] steps += generate_unit_and_integration_test_steps(full_stack_version, "true") else: # generate steps for the version if released - releases = versions_json["releases"] + releases = versions_yaml["releases"] for release_version in releases: if releases[release_version].startswith(target_branch): steps += generate_unit_and_integration_test_steps(releases[release_version], "false") break -======= - yaml = YAML(typ='safe') - versions_yaml: typing.final = yaml.load(response.text) - - # Use BUILDKITE_SOURCE to figure out PR merge or schedule. - # If PR merge, no need to run builds on all branches, target branch will be good - # - webhook when PR gets merged - # - schedule when daily schedule starts - # - ui when manually kicking job from BK UI - # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH - bk_source = os.getenv("BUILDKITE_SOURCE") - bk_branch = os.getenv("BUILDKITE_BRANCH") - steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \ - else generate_steps_for_main_branch(versions_yaml) ->>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) - # steps for snapshot version - snapshots = versions_json["snapshots"] + snapshots = versions_yaml["snapshots"] for snapshot_version in snapshots: if snapshots[snapshot_version].startswith(target_branch): steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false") diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py index 9a78a326..57d2b314 100644 --- a/.buildkite/scripts/e2e-pipeline/generate-steps.py +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -45,8 +45,8 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: steps = [] response = call_url_with_retry(RELEASES_URL) -<<<<<<< HEAD - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) # there are situations to manually run CIs with PR change, # set MANUAL_TARGET_BRANCH with upstream target branch and run @@ -54,36 +54,20 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH") print(f"Running with target_branch: {target_branch}") if target_branch == '8.x': - full_stack_version: typing.final = versions_json["snapshots"]["8.future"] + full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"] steps.append(generate_test_step(full_stack_version, target_branch, "true")) elif target_branch == 'main': - full_stack_version: typing.final = versions_json["snapshots"][target_branch] + full_stack_version: typing.final = versions_yaml["snapshots"][target_branch] steps.append(generate_test_step(full_stack_version, target_branch, "true")) else: # generate steps for the version if released - releases = versions_json["releases"] + releases = versions_yaml["releases"] for release_version in releases: if releases[release_version].startswith(target_branch): steps.append(generate_test_step(releases[release_version], target_branch, "false")) break -======= - yaml = YAML(typ='safe') - versions_yaml: typing.final = yaml.load(response.text) - - # Use BUILDKITE_SOURCE to figure out PR merge or schedule. - # If PR merge, no need to run builds on all branches, target branch will be good - # - webhook when PR gets merged - # - schedule when daily schedule starts - # - ui when manually kicking job from BK UI - # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH - bk_source = os.getenv("BUILDKITE_SOURCE") - bk_branch = os.getenv("BUILDKITE_BRANCH") - steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \ - else generate_steps_for_main_branch(versions_yaml) ->>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) - # steps for snapshot version - snapshots = versions_json["snapshots"] + snapshots = versions_yaml["snapshots"] for snapshot_version in snapshots: if snapshots[snapshot_version].startswith(target_branch): steps.append(generate_test_step(snapshots[snapshot_version], target_branch, "false")) diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index 5cf013e4..f29d9384 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -6,14 +6,9 @@ from ruamel.yaml import YAML -<<<<<<< HEAD -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" -TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml" -======= RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \ "-request-test-matrix.yml" ->>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358)) TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index 446a6a00..3208a200 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1,6 +1,3 @@ -<<<<<<< HEAD -mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh -======= #!/usr/bin/env bash export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21" @@ -15,4 +12,3 @@ else fi mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh ->>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))