diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 853a189cc9..fbfcaa5c41 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,7 +2,7 @@ * @Percona-Lab/build-engineers /cloud/gcp-functions/ @egegunes @inelpandzic -/cloud/ @hors @tplavcic @nmarukovich @ptankov @jvpasinatto @eleo007 +/cloud/ @hors @tplavcic @nmarukovich @jvpasinatto @eleo007 /pmm/ @talhabinrizwan @ademidoff @puneet0191 /IaC/pmm.cd/ @talhabinrizwan @ademidoff /pxb/ @mchawla16 @eleo007 @mohitj1988 @kaushikpuneet07 @panchal-yash @Percona-Lab/build-engineers @@ -14,3 +14,4 @@ /pdmdb/ @olexandr-havryliak @sandraromanchenko @Percona-Lab/build-engineers /pdps/ @eleo007 @kaushikpuneet07 @Percona-Lab/build-engineers /pdpxc/ @eleo007 @kaushikpuneet07 @Percona-Lab/build-engineers +/pml/ @olexandr-havryliak @sandraromanchenko @keithquinnpercona @Percona-Lab/build-engineers diff --git a/IaC/cloud.cd/init.groovy.d/htz.cloud.groovy b/IaC/cloud.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..6f3e7d9aad --- /dev/null +++ b/IaC/cloud.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,185 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "cloud-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['fedora42-x64'] = '232895138' // 232895138 system fedora-42 Fedora 42 x86 - 5 GB Thu Apr 24 10:00:32 EEST 2025 - +imageMap['fedora42-aarch64'] = '232895264' // 232895264 system fedora-42 Fedora 42 arm - 5 GB Thu Apr 24 10:01:01 EEST 2025 - +imageMap['launcher-x64'] = imageMap['fedora42-x64'] + +execMap = [:] +execMap['fedora'] = 1 +execMap['fedora42-x64-nbg1'] = execMap['fedora'] +execMap['fedora42-x64-hel1'] = execMap['fedora'] +execMap['fedora42-x64-fsn1'] = execMap['fedora'] +execMap['fedora42-aarch64-nbg1'] = execMap['fedora'] +execMap['fedora42-aarch64-hel1'] = execMap['fedora'] +execMap['fedora42-aarch64-fsn1'] = execMap['fedora'] +execMap['fedora42-x64-nbg1-min'] = execMap['fedora'] +execMap['fedora42-x64-hel1-min'] = execMap['fedora'] +execMap['fedora42-x64-fsn1-min'] = execMap['fedora'] +execMap['fedora42-aarch64-nbg1-min'] = execMap['fedora'] +execMap['fedora42-aarch64-hel1-min'] = execMap['fedora'] +execMap['fedora42-aarch64-fsn1-min'] = execMap['fedora'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 7 +bootDeadlineMap['fedora42-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['fedora42-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['fedora42'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['fedora42-x64-nbg1'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-x64-hel1'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-x64-fsn1'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-aarch64-nbg1'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-aarch64-hel1'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-aarch64-fsn1'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-x64-nbg1-min'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-x64-hel1-min'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-x64-fsn1-min'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-aarch64-nbg1-min'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-aarch64-hel1-min'] = jvmOptsMap['fedora42'] +jvmOptsMap['fedora42-aarch64-fsn1-min'] = jvmOptsMap['fedora42'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['fedora42'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['fedora42'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['fedora42'] + +labelMap = [:] +labelMap['fedora42-x64-min'] = 'docker-x64-min docker-fedora42-x64-min fedora42-x64-min' +labelMap['fedora42-aarch64-min'] = 'docker-aarch64-min docker-fedora42-aarch64-min fedora42-aarch64-min' +labelMap['fedora42-x64'] = 'docker-x64 docker-fedora42-x64 fedora42-x64' +labelMap['fedora42-aarch64'] = 'docker-aarch64 docker-fedora42-aarch64 fedora42-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['cloud.cd.percona.com'] = '11334955' // cloud.cd.percona.com + +initMap = [:] +initMap['fedora-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + until sudo dnf update -y; do + sleep 1 + echo "try again" + done + until sudo dnf install -y java-21-openjdk-headless ca-certificates curl gnupg unzip git dnf-plugins-core cronie bc npm make; do + sleep 1 + echo "try again" + done + sudo dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo + until sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do + sleep 1 + echo "try again" + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's/nofile=1024:4096/nofile=900000:900000/; s/DAEMON_MAXFILES=.*/DAEMON_MAXFILES=990000/' /etc/sysconfig/docker + echo 'DOCKER_STORAGE_OPTIONS="--data-root=/mnt/docker"' | sudo tee -a /etc/sysconfig/docker-storage + sudo sed -i.bak -e 's|^ExecStart=.*|ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000|' /usr/lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl status docker || sudo systemctl start docker + echo "* * * * * root /usr/sbin/route add default gw 10.30.236.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['fedora42-x64-nbg1'] = initMap['fedora-docker'] +initMap['fedora42-x64-hel1'] = initMap['fedora-docker'] +initMap['fedora42-x64-fsn1'] = initMap['fedora-docker'] +initMap['fedora42-aarch64-nbg1'] = initMap['fedora-docker'] +initMap['fedora42-aarch64-hel1'] = initMap['fedora-docker'] +initMap['fedora42-aarch64-fsn1'] = initMap['fedora-docker'] +initMap['fedora42-x64-nbg1-min'] = initMap['fedora-docker'] +initMap['fedora42-x64-hel1-min'] = initMap['fedora-docker'] +initMap['fedora42-x64-fsn1-min'] = initMap['fedora-docker'] +initMap['fedora42-aarch64-nbg1-min'] = initMap['fedora-docker'] +initMap['fedora42-aarch64-hel1-min'] = initMap['fedora-docker'] +initMap['fedora42-aarch64-fsn1-min'] = initMap['fedora-docker'] +initMap['launcher-x64-nbg1'] = initMap['fedora-docker'] +initMap['launcher-x64-hel1'] = initMap['fedora-docker'] +initMap['launcher-x64-fsn1'] = initMap['fedora-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("fedora42-x64-nbg1-min", labelMap['fedora42-x64-min'], imageMap['fedora42-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("fedora42-aarch64-nbg1-min", labelMap['fedora42-aarch64-min'], imageMap['fedora42-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("fedora42-x64-hel1-min", labelMap['fedora42-x64-min'], imageMap['fedora42-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("fedora42-aarch64-hel1-min", labelMap['fedora42-aarch64-min'], imageMap['fedora42-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("fedora42-x64-fsn1-min", labelMap['fedora42-x64-min'], imageMap['fedora42-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("fedora42-aarch64-fsn1-min", labelMap['fedora42-aarch64-min'], imageMap['fedora42-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("fedora42-x64-nbg1", labelMap['fedora42-x64'], imageMap['fedora42-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("fedora42-aarch64-nbg1", labelMap['fedora42-aarch64'], imageMap['fedora42-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("fedora42-x64-hel1", labelMap['fedora42-x64'], imageMap['fedora42-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("fedora42-aarch64-hel1", labelMap['fedora42-aarch64'], imageMap['fedora42-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("fedora42-x64-fsn1", labelMap['fedora42-x64'], imageMap['fedora42-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("fedora42-aarch64-fsn1", labelMap['fedora42-aarch64'], imageMap['fedora42-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['cloud.cd.percona.com'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/pg.cd/init.groovy.d/htz.cloud.groovy b/IaC/pg.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..8abe489a66 --- /dev/null +++ b/IaC/pg.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pg-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/pmm.cd/init.groovy.d/htz.cloud.groovy b/IaC/pmm.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..f35e877921 --- /dev/null +++ b/IaC/pmm.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pmm-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps3.cd/init.groovy.d/htz.cloud.groovy b/IaC/ps3.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..dad45e88b5 --- /dev/null +++ b/IaC/ps3.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "ps3-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps57.cd/init.groovy.d/htz.cloud.groovy b/IaC/ps57.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..f627ebfa80 --- /dev/null +++ b/IaC/ps57.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "ps57-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps80.cd/init.groovy.d/htz.cloud.groovy b/IaC/ps80.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..f1a7e5a442 --- /dev/null +++ b/IaC/ps80.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "ps80-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/ps80.cd/init.groovy.d/matrix.groovy b/IaC/ps80.cd/init.groovy.d/matrix.groovy index 33309cfece..a82e3424ce 100644 --- a/IaC/ps80.cd/init.groovy.d/matrix.groovy +++ b/IaC/ps80.cd/init.groovy.d/matrix.groovy @@ -149,6 +149,7 @@ authz_strategy_config = [ 'percona*QA-ps': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*qa-distro': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*doc': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], + 'JNKPercona': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], 'percona*external-contractors-ps': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], ] ] diff --git a/IaC/psmdb.cd/init.groovy.d/htz.cloud.groovy b/IaC/psmdb.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..27d0473da5 --- /dev/null +++ b/IaC/psmdb.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "psmdb-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/psmdb.cd/init.groovy.d/matrix.groovy b/IaC/psmdb.cd/init.groovy.d/matrix.groovy index 0723f5e6d4..8ad93fc13d 100644 --- a/IaC/psmdb.cd/init.groovy.d/matrix.groovy +++ b/IaC/psmdb.cd/init.groovy.d/matrix.groovy @@ -148,6 +148,7 @@ authz_strategy_config = [ 'percona*dev-psmdb': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*qa-distro': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*doc': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], + 'JNKPercona': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], ] ] diff --git a/IaC/pxb.cd/init.groovy.d/htz.cloud.groovy b/IaC/pxb.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..e30059faf9 --- /dev/null +++ b/IaC/pxb.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pxb-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/pxc.cd/init.groovy.d/htz.cloud.groovy b/IaC/pxc.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..3042e0900b --- /dev/null +++ b/IaC/pxc.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "pxc-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/IaC/pxc.cd/init.groovy.d/matrix.groovy b/IaC/pxc.cd/init.groovy.d/matrix.groovy index 33309cfece..a82e3424ce 100644 --- a/IaC/pxc.cd/init.groovy.d/matrix.groovy +++ b/IaC/pxc.cd/init.groovy.d/matrix.groovy @@ -149,6 +149,7 @@ authz_strategy_config = [ 'percona*QA-ps': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*qa-distro': ['Overall Read','Agent Configure','Agent Delete','Agent Create','Agent Disconnect','Agent Connect','Agent Build','Agent Provision','Run Delete','Run Update','Run Replay','Job Create','Job Delete','Job Configure','Job Read','Job Discover','Job Build','Job Workspace','Job Cancel','Job Move','View Create','View Delete','View Configure','View Read'], 'percona*doc': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], + 'JNKPercona': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], 'percona*external-contractors-ps': ['Overall Read','Agent Build','Agent Connect','Agent Provision','Job Discover','Job Read','Job Build','Job Cancel','Job Workspace','View Read'], ] ] diff --git a/IaC/rel.cd/init.groovy.d/htz.cloud.groovy b/IaC/rel.cd/init.groovy.d/htz.cloud.groovy new file mode 100644 index 0000000000..4aa9bc864b --- /dev/null +++ b/IaC/rel.cd/init.groovy.d/htz.cloud.groovy @@ -0,0 +1,190 @@ +import cloud.dnation.jenkins.plugins.hetzner.* +import cloud.dnation.jenkins.plugins.hetzner.launcher.* +import hudson.model.* +import jenkins.model.Jenkins +import java.util.logging.Logger + +def cloudName = "rel-htz" + +imageMap = [:] // ID TYPE NAME DESCRIPTION ARCHITECTURE IMAGE SIZE DISK SIZE CREATED DEPRECATED +imageMap['deb12-x64'] = '114690387' // 114690387 system debian-12 Debian 12 x86 - 5 GB Tue Jun 13 09:00:02 EEST 2023 - +imageMap['deb12-aarch64'] = '114690389' // 114690389 system debian-12 Debian 12 arm - 5 GB Tue Jun 13 09:00:03 EEST 2023 - +imageMap['launcher-x64'] = imageMap['deb12-x64'] + +execMap = [:] +execMap['deb'] = 1 +execMap['deb12-x64-nbg1'] = execMap['deb'] +execMap['deb12-x64-hel1'] = execMap['deb'] +execMap['deb12-x64-fsn1'] = execMap['deb'] +execMap['deb12-aarch64-nbg1'] = execMap['deb'] +execMap['deb12-aarch64-hel1'] = execMap['deb'] +execMap['deb12-aarch64-fsn1'] = execMap['deb'] +execMap['deb12-x64-nbg1-min'] = execMap['deb'] +execMap['deb12-x64-hel1-min'] = execMap['deb'] +execMap['deb12-x64-fsn1-min'] = execMap['deb'] +execMap['deb12-aarch64-nbg1-min'] = execMap['deb'] +execMap['deb12-aarch64-hel1-min'] = execMap['deb'] +execMap['deb12-aarch64-fsn1-min'] = execMap['deb'] +execMap['launcher-x64-nbg1'] = 30 +execMap['launcher-x64-hel1'] = 30 +execMap['launcher-x64-fsn1'] = 30 + +bootDeadlineMap =[:] +bootDeadlineMap['default'] = 3 +bootDeadlineMap['deb12-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-x64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-nbg1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-hel1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['deb12-aarch64-fsn1-min'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-nbg1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-hel1'] = bootDeadlineMap['default'] +bootDeadlineMap['launcher-x64-fsn1'] = bootDeadlineMap['default'] + +jvmOptsMap = [:] +jvmOptsMap['deb12'] = '-Xmx512m -Xms512m --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED' +jvmOptsMap['deb12-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-x64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-nbg1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-hel1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['deb12-aarch64-fsn1-min'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-nbg1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-hel1'] = jvmOptsMap['deb12'] +jvmOptsMap['launcher-x64-fsn1'] = jvmOptsMap['deb12'] + +labelMap = [:] +labelMap['deb12-x64-min'] = 'docker-x64-min docker-deb12-x64-min deb12-x64-min' +labelMap['deb12-aarch64-min'] = 'docker-aarch64-min docker-deb12-aarch64-min deb12-aarch64-min' +labelMap['deb12-x64'] = 'docker-x64 docker-deb12-x64 deb12-x64' +labelMap['deb12-aarch64'] = 'docker-aarch64 docker-deb12-aarch64 deb12-aarch64' +labelMap['launcher-x64'] = 'launcher-x64' + +networkMap = [:] +networkMap['percona-vpc-eu'] = '10442325' // percona-vpc-eu + +initMap = [:] +initMap['deb-docker'] = '''#!/bin/bash -x + set -o xtrace + ( sudo systemctl stop sshd; sleep 300; sudo systemctl start sshd ) & + sudo fallocate -l 32G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + + export DEBIAN_FRONTEND=noninteractive + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install openjdk-17-jre-headless apt-transport-https ca-certificates curl gnupg lsb-release unzip git; do + sleep 1 + echo try again + done + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + until sudo apt-get update; do + sleep 1 + echo try again + done + until sudo apt-get -y install docker-ce docker-ce-cli containerd.io; do + sleep 1 + echo try again + done + if ! $(aws --version | grep -q 'aws-cli/2'); then + find /tmp -maxdepth 1 -name "*aws*" | xargs sudo rm -rf + until curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.22.35.zip" -o "/tmp/awscliv2.zip"; do + sleep 1 + echo try again + done + unzip -o /tmp/awscliv2.zip -d /tmp + cd /tmp/aws && sudo ./install + fi + sudo install -o $(id -u -n) -g $(id -g -n) -d /mnt/jenkins + sudo sysctl net.ipv4.tcp_fin_timeout=15 + sudo sysctl net.ipv4.tcp_tw_reuse=1 + sudo sysctl net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl net.ipv6.conf.default.disable_ipv6=1 + sudo sysctl -w fs.inotify.max_user_watches=10000000 || true + sudo sysctl -w fs.aio-max-nr=1048576 || true + sudo sysctl -w fs.file-max=6815744 || true + echo "* soft core unlimited" | sudo tee -a /etc/security/limits.conf + sudo sed -i.bak -e 's^ExecStart=.*^ExecStart=/usr/bin/dockerd --data-root=/mnt/docker --default-ulimit nofile=900000:900000^' /lib/systemd/system/docker.service + sudo systemctl daemon-reload + sudo install -o root -g root -d /mnt/docker + sudo usermod -aG docker $(id -u -n) + sudo mkdir -p /etc/docker + echo '{"experimental": true, "ipv6": true, "fixed-cidr-v6": "fd3c:a8b0:18eb:5c06::/64"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "* * * * * root /usr/sbin/route add default gw 10.177.1.1 eth0" | sudo tee /etc/cron.d/fix-default-route + sudo systemctl start sshd +''' +initMap['deb12-x64-nbg1'] = initMap['deb-docker'] +initMap['deb12-x64-hel1'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1'] = initMap['deb-docker'] +initMap['deb12-x64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-x64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-x64-fsn1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-nbg1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-hel1-min'] = initMap['deb-docker'] +initMap['deb12-aarch64-fsn1-min'] = initMap['deb-docker'] +initMap['launcher-x64-nbg1'] = initMap['deb-docker'] +initMap['launcher-x64-hel1'] = initMap['deb-docker'] +initMap['launcher-x64-fsn1'] = initMap['deb-docker'] + +def templates = [ + /* new HetznerServerTemplate("ubuntu20-cx21", "java", "name=ubuntu20-docker", "fsn1", "cx21"), */ + // tmplName tmplLabels tmplImage region server type + new HetznerServerTemplate("deb12-x64-nbg1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "nbg1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-nbg1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "nbg1", "cax31"), + new HetznerServerTemplate("deb12-x64-hel1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "hel1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-hel1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "hel1", "cax31"), + new HetznerServerTemplate("deb12-x64-fsn1-min", labelMap['deb12-x64-min'], imageMap['deb12-x64'], "fsn1", "cpx41"), + new HetznerServerTemplate("deb12-aarch64-fsn1-min", labelMap['deb12-aarch64-min'], imageMap['deb12-aarch64'], "fsn1", "cax31"), + new HetznerServerTemplate("deb12-x64-nbg1", labelMap['deb12-x64'], imageMap['deb12-x64'], "nbg1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-nbg1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "nbg1", "cax41"), + new HetznerServerTemplate("deb12-x64-hel1", labelMap['deb12-x64'], imageMap['deb12-x64'], "hel1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-hel1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "hel1", "cax41"), + new HetznerServerTemplate("deb12-x64-fsn1", labelMap['deb12-x64'], imageMap['deb12-x64'], "fsn1", "cpx51"), + new HetznerServerTemplate("deb12-aarch64-fsn1", labelMap['deb12-aarch64'], imageMap['deb12-aarch64'], "fsn1", "cax41"), + new HetznerServerTemplate("launcher-x64-nbg1", labelMap['launcher-x64'], imageMap['launcher-x64'], "nbg1", "cpx21"), + new HetznerServerTemplate("launcher-x64-hel1", labelMap['launcher-x64'], imageMap['launcher-x64'], "hel1", "cpx21"), + new HetznerServerTemplate("launcher-x64-fsn1", labelMap['launcher-x64'], imageMap['launcher-x64'], "fsn1", "cpx21") +] + +templates.each { it -> + def sshConnector = new SshConnectorAsRoot("htz.cd.key") + sshConnector.setConnectionMethod(new PublicAddressOnly()) // Replace with the desired method + it.setConnector(sshConnector) + def tmplName = it.name + it.setNumExecutors(execMap[tmplName]) + it.bootDeadline = bootDeadlineMap[tmplName] + it.remoteFs = "/mnt/jenkins/" + it.jvmOpts = jvmOptsMap[tmplName] + it.network = networkMap['percona-vpc-eu'] + it.userData = initMap[tmplName] + } + +// public HetznerCloud(String name, String credentialsId, String instanceCapStr, List serverTemplates) +def cloud = new HetznerCloud(cloudName, "htz.cd.token", "100", templates) + +def jenkins = Jenkins.get() + +jenkins.clouds.remove(jenkins.clouds.getByName(cloudName)) +jenkins.clouds.add(cloud) +jenkins.save() diff --git a/cloud/jenkins/pgo_operator_eks_latest.groovy b/cloud/jenkins/pgo_operator_eks_latest.groovy index c25e55a1f9..ffee85708a 100644 --- a/cloud/jenkins/pgo_operator_eks_latest.groovy +++ b/cloud/jenkins/pgo_operator_eks_latest.groovy @@ -31,7 +31,9 @@ void prepareNode() { } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pgo_operator_eks_version.groovy b/cloud/jenkins/pgo_operator_eks_version.groovy index 6a3105ebce..76c419901e 100644 --- a/cloud/jenkins/pgo_operator_eks_version.groovy +++ b/cloud/jenkins/pgo_operator_eks_version.groovy @@ -31,7 +31,9 @@ void prepareNode() { } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pgo_operator_gke_latest.groovy b/cloud/jenkins/pgo_operator_gke_latest.groovy index 4b3b7845ad..455be73104 100644 --- a/cloud/jenkins/pgo_operator_gke_latest.groovy +++ b/cloud/jenkins/pgo_operator_gke_latest.groovy @@ -48,7 +48,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d '.' -f 1,2", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d '.' -f 1,2", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pgo_operator_gke_version.groovy b/cloud/jenkins/pgo_operator_gke_version.groovy index d23429bdf4..89b5758979 100644 --- a/cloud/jenkins/pgo_operator_gke_version.groovy +++ b/cloud/jenkins/pgo_operator_gke_version.groovy @@ -48,7 +48,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d '.' -f 1,2", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d '.' -f 1,2", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_eks_latest.groovy b/cloud/jenkins/ps_operator_eks_latest.groovy index 983edc7008..617d44a856 100644 --- a/cloud/jenkins/ps_operator_eks_latest.groovy +++ b/cloud/jenkins/ps_operator_eks_latest.groovy @@ -29,7 +29,9 @@ void prepareNode() { void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_eks_version.groovy b/cloud/jenkins/ps_operator_eks_version.groovy index 057c80e912..86cd5de840 100644 --- a/cloud/jenkins/ps_operator_eks_version.groovy +++ b/cloud/jenkins/ps_operator_eks_version.groovy @@ -29,7 +29,9 @@ void prepareNode() { void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_gke_latest.groovy b/cloud/jenkins/ps_operator_gke_latest.groovy index 72aa948a14..7722188f3b 100644 --- a/cloud/jenkins/ps_operator_gke_latest.groovy +++ b/cloud/jenkins/ps_operator_gke_latest.groovy @@ -46,7 +46,7 @@ EOF void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/ps_operator_gke_version.groovy b/cloud/jenkins/ps_operator_gke_version.groovy index c81701fa04..2ac3eeacde 100644 --- a/cloud/jenkins/ps_operator_gke_version.groovy +++ b/cloud/jenkins/ps_operator_gke_version.groovy @@ -46,7 +46,7 @@ EOF void prepareSources() { if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_eks_latest.groovy b/cloud/jenkins/psmdb_operator_eks_latest.groovy index 0c8052706c..f546efec2b 100644 --- a/cloud/jenkins/psmdb_operator_eks_latest.groovy +++ b/cloud/jenkins/psmdb_operator_eks_latest.groovy @@ -70,7 +70,9 @@ void prepareNode() { """ if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_eks_version.groovy b/cloud/jenkins/psmdb_operator_eks_version.groovy index 850598dcdd..b36191c1ab 100644 --- a/cloud/jenkins/psmdb_operator_eks_version.groovy +++ b/cloud/jenkins/psmdb_operator_eks_version.groovy @@ -70,7 +70,9 @@ void prepareNode() { """ if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_gke_latest.groovy b/cloud/jenkins/psmdb_operator_gke_latest.groovy index 199b2e0645..e83a07d8bf 100644 --- a/cloud/jenkins/psmdb_operator_gke_latest.groovy +++ b/cloud/jenkins/psmdb_operator_gke_latest.groovy @@ -90,7 +90,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/psmdb_operator_gke_version.groovy b/cloud/jenkins/psmdb_operator_gke_version.groovy index 01fbd469c2..5d03f3070d 100644 --- a/cloud/jenkins/psmdb_operator_gke_version.groovy +++ b/cloud/jenkins/psmdb_operator_gke_version.groovy @@ -90,7 +90,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", , returnStdout: true).trim() + USED_PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", , returnStdout: true).trim() } else { USED_PLATFORM_VER="$PLATFORM_VER" } diff --git a/cloud/jenkins/pxco-eks.groovy b/cloud/jenkins/pxco-eks.groovy index 12e60d534a..b0de107790 100644 --- a/cloud/jenkins/pxco-eks.groovy +++ b/cloud/jenkins/pxco-eks.groovy @@ -63,7 +63,9 @@ void prepareNode() { """ if ("$PLATFORM_VER" == "latest") { - PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", returnStdout: true).trim() + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + USED_PLATFORM_VER = sh(script: "aws eks describe-addon-versions --query 'addons[].addonVersions[].compatibilities[].clusterVersion' --output json | jq -r 'flatten | unique | sort | reverse | .[0]'", , returnStdout: true).trim() + } } if ("$IMAGE_PXC") { diff --git a/cloud/jenkins/pxco-gke.groovy b/cloud/jenkins/pxco-gke.groovy index 4c5a84d2ae..09422860dc 100644 --- a/cloud/jenkins/pxco-gke.groovy +++ b/cloud/jenkins/pxco-gke.groovy @@ -83,7 +83,7 @@ EOF } if ("$PLATFORM_VER" == "latest") { - PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.defaultVersion)' | cut -d- -f1", returnStdout: true).trim() + PLATFORM_VER = sh(script: "gcloud container get-server-config --region=$region --flatten=channels --filter='channels.channel=RAPID' --format='value(channels.validVersions)' | cut -d- -f1", returnStdout: true).trim() } if ("$IMAGE_PXC") { diff --git a/orchestrator/jenkins/orchestrator-packaging.groovy b/orchestrator/jenkins/orchestrator-packaging.groovy index 8b32efd635..aa33a20d4b 100644 --- a/orchestrator/jenkins/orchestrator-packaging.groovy +++ b/orchestrator/jenkins/orchestrator-packaging.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -30,9 +30,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/orchestrator.git', description: 'URL for orchestrator repository', @@ -54,11 +58,11 @@ pipeline { description: 'General version of the product', name: 'VERSION') string( - defaultValue: '14', + defaultValue: '18', description: 'RPM release value (e.g. custom build -> 1.1custom124)', name: 'RPM_RELEASE') string( - defaultValue: '14', + defaultValue: '18', description: 'DEB release value (e.g. custom build -> 1.1custom124)', name: 'DEB_RELEASE') choice( @@ -89,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build orchestrator generic source packages') { parallel { stage('Build orchestrator generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build orchestrator generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -127,184 +131,236 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -319,7 +375,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild("tools", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "tools", COMPONENT) } } diff --git a/orchestrator/jenkins/orchestrator-packaging.yml b/orchestrator/jenkins/orchestrator-packaging.yml index 91a009371d..50aef7d69a 100644 --- a/orchestrator/jenkins/orchestrator-packaging.yml +++ b/orchestrator/jenkins/orchestrator-packaging.yml @@ -1,5 +1,5 @@ - job: - name: orchestrator-RELEASE + name: hetzner-orchestrator-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: orchestrator/jenkins/orchestrator-packaging.groovy diff --git a/pbm/hetzner-pbm-functional-tests-full.groovy b/pbm/hetzner-pbm-functional-tests-full.groovy new file mode 100644 index 0000000000..321d5d1ce9 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests-full.groovy @@ -0,0 +1,115 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'launcher-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch or commit') + string(name: 'GO_VER', defaultValue: 'bookworm', description: 'GOLANG docker image for building PBM from sources') + choice(name: 'instance', choices: ['docker-x64','docker-aarch64'], description: 'Instance type for running tests') + string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') + string(name: 'PYTEST_PARAMS', defaultValue: '', description: 'Extra args passed to pytest') + booleanParam(name: 'ADD_JENKINS_MARKED_TESTS', defaultValue: false, description: 'Include tests with jenkins marker') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PBM_BRANCH}" + } + } + } + stage ('Run tests') { + matrix { + agent { + label "${params.instance}" + } + axes { + axis { + name 'TEST' + values 'logical', 'physical', 'incremental', 'external', 'load' + } + axis { + name 'PSMDB' + values '6.0', '7.0', '8.0' + } + } + stages { + stage ('Run tests') { + steps { + withCredentials([string(credentialsId: 'olexandr_zephyr_token', variable: 'ZEPHYR_TOKEN'), + string(credentialsId: 'KMS_ID', variable: 'KMS_ID'), + file(credentialsId: 'PBM-AWS-S3', variable: 'PBM_AWS_S3_YML'), + file(credentialsId: 'PBM-GCS-S3', variable: 'PBM_GCS_S3_YML'), + file(credentialsId: 'PBM-GCS-HMAC-S3', variable: 'PBM_GCS_HMAC_S3_YML'), + file(credentialsId: 'PBM-AZURE', variable: 'PBM_AZURE_YML')]) { + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + if [ ! -f "/usr/local/bin/docker-compose" ] ; then + if [ ${params.instance} = "docker-aarch64" ]; then + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-aarch64 -o /usr/local/bin/docker-compose + else + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + fi + sudo chmod +x /usr/local/bin/docker-compose + fi + """ + git poll: false, branch: params.TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + sh """ + cd pbm-functional/pytest + cp $PBM_AWS_S3_YML ./conf/pbm/aws.yaml + cp $PBM_GCS_S3_YML ./conf/pbm/gcs.yaml + cp $PBM_GCS_HMAC_S3_YML ./conf/pbm/gcs_hmac.yaml + cp $PBM_AZURE_YML ./conf/pbm/azure.yaml + if [ "${ADD_JENKINS_MARKED_TESTS}" = "true" ]; then JENKINS_FLAG="--jenkins"; else JENKINS_FLAG=""; fi + PSMDB=perconalab/percona-server-mongodb:${PSMDB} docker-compose build --no-cache + docker-compose up -d + if [ -n "${params.PYTEST_PARAMS}" ]; then + FULL_EXPR="${TEST} and ${params.PYTEST_PARAMS}" + else + FULL_EXPR="${TEST}" + fi + KMS_ID="${KMS_ID}" docker-compose run test pytest -s --junitxml=junit.xml \$JENKINS_FLAG -k "\$FULL_EXPR" || true + docker-compose down -v --remove-orphans + curl -H "Content-Type:multipart/form-data" -H "Authorization: Bearer ${ZEPHYR_TOKEN}" -F "file=@junit.xml;type=application/xml" 'https://api.zephyrscale.smartbear.com/v2/automations/executions/junit?projectKey=PBM' -F 'testCycle={"name":"${JOB_NAME}-${BUILD_NUMBER}","customFields": { "PBM branch": "${PBM_BRANCH}","PSMDB docker image": "percona/percona-server-mongodb:${PSMDB}","instance": "${instance}"}};type=application/json' -i || true + """ + } + } + post { + always { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + """ + } + } + } + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: PBM ${PBM_BRANCH} - all tests passed [${BUILD_URL}testReport/]") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: PBM ${PBM_BRANCH} - some tests failed [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: PBM ${PBM_BRANCH} - unexpected failure [${BUILD_URL}]") + } + } +} diff --git a/pbm/hetzner-pbm-functional-tests-full.yml b/pbm/hetzner-pbm-functional-tests-full.yml new file mode 100644 index 0000000000..d21a601da6 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests-full.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pbm-functional-tests-full + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/hetzner-pbm-functional-tests-full.groovy diff --git a/pbm/hetzner-pbm-functional-tests.groovy b/pbm/hetzner-pbm-functional-tests.groovy new file mode 100644 index 0000000000..44fbc799e0 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests.groovy @@ -0,0 +1,115 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'launcher-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch') + string(name: 'PSMDB', defaultValue: 'percona/percona-server-mongodb', description: 'PSMDB docker image') + string(name: 'GO_VER', defaultValue: 'bookworm', description: 'GOLANG docker image for building PBM from sources') + choice(name: 'instance', choices: ['docker-x64','docker-aarch64'], description: 'Instance type for running tests') + string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') + string(name: 'PYTEST_PARAMS', defaultValue: '', description: 'Extra args passed to pytest') + booleanParam(name: 'ADD_JENKINS_MARKED_TESTS', defaultValue: false, description: 'Include tests with jenkins marker') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PBM_BRANCH}-${params.PSMDB}" + } + } + } + stage ('Run tests') { + matrix { + agent { + label "${params.instance}" + } + axes { + axis { + name 'TEST' + values 'logical', 'physical', 'incremental', 'external', 'load' + } + } + stages { + stage ('Run tests') { + steps { + withCredentials([string(credentialsId: 'olexandr_zephyr_token', variable: 'ZEPHYR_TOKEN'), + string(credentialsId: 'KMS_ID', variable: 'KMS_ID'), + file(credentialsId: 'PBM-AWS-S3', variable: 'PBM_AWS_S3_YML'), + file(credentialsId: 'PBM-GCS-S3', variable: 'PBM_GCS_S3_YML'), + file(credentialsId: 'PBM-GCS-HMAC-S3', variable: 'PBM_GCS_HMAC_S3_YML'), + file(credentialsId: 'PBM-AZURE', variable: 'PBM_AZURE_YML')]) { + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + + if [ ! -f "/usr/local/bin/docker-compose" ] ; then + if [ ${params.instance} = "docker-aarch64" ]; then + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-aarch64 -o /usr/local/bin/docker-compose + else + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + fi + sudo chmod +x /usr/local/bin/docker-compose + fi + + git clone https://github.com/Percona-QA/psmdb-testing + cd psmdb-testing + git checkout ${params.TESTING_BRANCH} + + cd pbm-functional/pytest + cp $PBM_AWS_S3_YML ./conf/pbm/aws.yaml + cp $PBM_GCS_S3_YML ./conf/pbm/gcs.yaml + cp $PBM_GCS_HMAC_S3_YML ./conf/pbm/gcs_hmac.yaml + cp $PBM_AZURE_YML ./conf/pbm/azure.yaml + if [ "${ADD_JENKINS_MARKED_TESTS}" = "true" ]; then JENKINS_FLAG="--jenkins"; else JENKINS_FLAG=""; fi + docker-compose build + docker-compose up -d + if [ -n "${params.PYTEST_PARAMS}" ]; then + FULL_EXPR="${TEST} and ${params.PYTEST_PARAMS}" + else + FULL_EXPR="${TEST}" + fi + KMS_ID="${KMS_ID}" docker-compose run test pytest -s --junitxml=junit.xml \$JENKINS_FLAG -k "\$FULL_EXPR" || true + docker-compose down -v --remove-orphans + curl -H "Content-Type:multipart/form-data" -H "Authorization: Bearer ${ZEPHYR_TOKEN}" -F "file=@junit.xml;type=application/xml" 'https://api.zephyrscale.smartbear.com/v2/automations/executions/junit?projectKey=PBM' -F 'testCycle={"name":"${JOB_NAME}-${BUILD_NUMBER}","customFields": { "PBM branch": "${PBM_BRANCH}","PSMDB docker image": "${PSMDB}","instance": "${instance}"}};type=application/json' -i || true + """ + } + } + post { + always { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + """ + } + } + } + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: PBM ${PBM_BRANCH} with ${PSMDB} - all tests passed") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: PBM ${PBM_BRANCH} with ${PSMDB} - some tests failed [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: PBM ${PBM_BRANCH} with ${PSMDB} - unexpected failure [${BUILD_URL}]") + } + } +} diff --git a/pbm/hetzner-pbm-functional-tests.yml b/pbm/hetzner-pbm-functional-tests.yml new file mode 100644 index 0000000000..4749f20161 --- /dev/null +++ b/pbm/hetzner-pbm-functional-tests.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pbm-functional-tests + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/hetzner-pbm-functional-tests.groovy diff --git a/pbm/hetzner-pbm-site-check.groovy b/pbm/hetzner-pbm-site-check.groovy new file mode 100644 index 0000000000..8a6267f84b --- /dev/null +++ b/pbm/hetzner-pbm-site-check.groovy @@ -0,0 +1,68 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'docker-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string( + defaultValue: '2.3.0', + description: 'PBM Version for tests', + name: 'PBM_VERSION') + string( + defaultValue: 'main', + description: 'Branch for testing repository', + name: 'TESTING_BRANCH') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PBM_VERSION}" + } + } + } + stage('Checkout') { + steps { + deleteDir() + git poll: false, branch: TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + } + } + stage('Test') { + steps { + script { + sh """ + cd site_checks + docker run --env PBM_VERSION=${params.PBM_VERSION} --rm -v `pwd`:/tmp -w /tmp python bash -c 'pip3 install requests pytest setuptools && pytest -s --junitxml=junit.xml test_pbm.py || [ \$? = 1 ] ' + """ + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: checking packages on the main site for PBM ${PBM_VERSION} - ok [${BUILD_URL}testReport/]") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: checking packages on the main site for PBM ${PBM_VERSION} - some links are broken [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: checking packages on the main site for PBM ${PBM_VERSION} - failed [${BUILD_URL}]" ) + } + always { + script { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } + } +} diff --git a/pbm/hetzner-pbm-site-check.yml b/pbm/hetzner-pbm-site-check.yml new file mode 100644 index 0000000000..07b2a3507f --- /dev/null +++ b/pbm/hetzner-pbm-site-check.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pbm-site-check + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/hetzner-pbm-site-check.groovy diff --git a/pbm/jenkins/get-pbm-branches.groovy b/pbm/jenkins/get-pbm-branches.groovy index f9b256f199..693d80c057 100644 --- a/pbm/jenkins/get-pbm-branches.groovy +++ b/pbm/jenkins/get-pbm-branches.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-backup-mongodb.git', description: 'URL for percona-mongodb-backup repository', @@ -22,50 +26,50 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sh """ - EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties || EC=\$? + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + EC=0 + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 ls s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || EC=\$? - if [ \${EC} = 1 ]; then - LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) - BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) - COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + if [ \${EC} = 1 ]; then + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) - echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id.properties - echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id.properties + echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id.properties - aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ - echo "START_NEW_BUILD=NO" > startBuild - else - aws s3 cp s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties . - source branch_commit_id.properties + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + echo "START_NEW_BUILD=NO" > startBuild + else + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp s3://percona-jenkins-artifactory/percona-backup-mongodb/branch_commit_id.properties . ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + source branch_commit_id.properties - LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) - LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) - LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) - if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then - echo "START_NEW_BUILD=YES" > startBuild - else - echo "START_NEW_BUILD=NO" > startBuild - fi + if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then + echo "START_NEW_BUILD=YES" > startBuild + else + echo "START_NEW_BUILD=NO" > startBuild + fi - echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id.properties - echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id.properties - aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ - fi - """ - } - script { + echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id.properties + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-backup-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + fi + """ + } START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${COMMIT_ID}").trim() VERSION = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME} | sed s:release\\-::").trim() } - } - } stage('Build needed') { when { @@ -78,8 +82,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'pbm-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'VERSION', value: VERSION), string(name: 'COMPONENT', value: 'testing')] - build job: 'pbm-release-test-run', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: BRANCH_NAME), string(name: 'PBM_VERSION', value: VERSION)] + build job: 'hetzner-pbm-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'VERSION', value: VERSION), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-pbm-release-test-run', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: BRANCH_NAME), string(name: 'PBM_VERSION', value: VERSION)] } } stage('Build skipped') { diff --git a/pbm/jenkins/get-pbm-branches.yml b/pbm/jenkins/get-pbm-branches.yml index e6b0065d57..23a503b6b9 100644 --- a/pbm/jenkins/get-pbm-branches.yml +++ b/pbm/jenkins/get-pbm-branches.yml @@ -1,5 +1,5 @@ - job: - name: pbm-autobuild + name: hetzner-pbm-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/jenkins/get-pbm-branches.groovy diff --git a/pbm/jenkins/pbm_generate_sbom.groovy b/pbm/jenkins/pbm_generate_sbom.groovy new file mode 100644 index 0000000000..7900ed0564 --- /dev/null +++ b/pbm/jenkins/pbm_generate_sbom.groovy @@ -0,0 +1,411 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([ + string(credentialsId: 'SNYK_TOKEN', variable: 'SNYK_TOKEN'), + string(credentialsId: 'SNYK_ORG_TOKEN', variable: 'SNYK_ORG_TOKEN') + ]){ + sh """ + echo "Starting Build Stage" + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/packaging/pbm_sbom/pbm_generate_sbom.sh -O pbm_generate_sbom.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} -e SNYK_TOKEN=${SNYK_TOKEN} -e SNYK_ORG_TOKEN=${SNYK_ORG_TOKEN} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + ls -laR + uname -a + bash -x ./pbm_generate_sbom.sh --builddir=\${build_dir} --pbm_version=${PBM_VERSION} --repo_type=${REPO_TYPE} --git_repo=${GIT_REPO} --git_branch=${GIT_BRANCH} ${STAGE_PARAM}" + curl -fsSL https://raw.githubusercontent.com/EvgeniyPatlan/sbom_verifier/main/install_sbom_verifier.sh | bash + bash sbom_verifier.sh pbm_sbom/*.json + """ + } +} + +void uploadPBMSBOMToTestingDownloadServer(String productName, String packageVersion, String SBOMType) { + + script { + try { + uploadSBOMToDownloadsTesting(params.CLOUD, productName, packageVersion, SBOMType) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-backup-mongodb.git', + description: 'URL for pbm_sbom repository', + name: 'GIT_REPO') + string( + defaultValue: '2.10.0', + description: 'Version of Percona Backup MongoDB', + name: 'PBM_VERSION') + string( + defaultValue: 'main', + description: 'Tag/Branch for percona-backup-mongodb repository', + name: 'GIT_BRANCH') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Packaging repository type', + name: 'REPO_TYPE') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Generate SBOM') { + parallel { + stage('Generate PBM SBOM OL/8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + echo "Ran stash includes" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM OL/8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM OL/9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM OL/9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM AL 2023 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("amazonlinux:2023", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM AL 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("amazonlinux:2023", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } +/* stage('Generate PBM SBOM OL/10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM OL/10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } */ + stage('Generate PBM SBOM Jammy AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM Jammy ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM Noble AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM Noble ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM Bullseye AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PBM SBOM Bookworm AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PBM_SBOM/${PBM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PBM_VERSION} + cat uploadPath-${PBM_VERSION} + """ + stash includes: "uploadPath-${PBM_VERSION}", name: "uploadPath-${PBM_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH) + } + } + } + } //parallel + } // stage + stage('Upload SBOMS from AWS') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + uploadSBOMfromAWS(params.CLOUD, "pbm_sbom/", AWS_STASH_PATH, "json", "${PBM_VERSION}") + } + } + stage('Push SBOMS to TESTING downloads area') { + steps { + cleanUpWS() + uploadPBMSBOMToTestingDownloadServer("pbm_sbom", "${PBM_VERSION}", "json") + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/pbm/jenkins/pbm_generate_sbom.yml b/pbm/jenkins/pbm_generate_sbom.yml new file mode 100644 index 0000000000..9dc6105756 --- /dev/null +++ b/pbm/jenkins/pbm_generate_sbom.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pbm_generate_sbom + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pbm/jenkins/pbm_generate_sbom.groovy + diff --git a/pbm/jenkins/percona-mongodb-backup.groovy b/pbm/jenkins/percona-mongodb-backup.groovy index 9ad4ef2c3a..53399f55d4 100644 --- a/pbm/jenkins/percona-mongodb-backup.groovy +++ b/pbm/jenkins/percona-mongodb-backup.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,15 +29,19 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-backup-mongodb.git', description: 'URL for percona-mongodb-backup repository', name: 'GIT_REPO') string( - defaultValue: 'master', + defaultValue: 'dev', description: 'Tag/Branch for percona-mongodb-backup repository', name: 'GIT_BRANCH') string( @@ -49,7 +53,7 @@ pipeline { description: 'DEB release value', name: 'DEB_RELEASE') string( - defaultValue: '1.3.2', + defaultValue: '2.10.0', description: 'VERSION value', name: 'VERSION') string( @@ -57,9 +61,25 @@ pipeline { description: 'PBM repo name', name: 'PBM_REPO') choice( - choices: 'laboratory\ntesting\nexperimental', + choices: 'experimental\nlaboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PBM_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') } options { skipDefaultCheckout() @@ -69,8 +89,11 @@ pipeline { } stages { stage('Create PBM source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -88,179 +111,452 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PBM generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PBM generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PBM generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PBM RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9(x86_64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + /* stage('Oracle Linux 10(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } */ // Commented out in scope of PKG-1083 + /* stage('Oracle Linux 10(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } */ // Commented out in scope of PKG-1083 + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Ubuntu Jammy 22.04(x86_64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy 22.04(aarch64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble 24.04(x86_64)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble 24.04(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oraclelinux 8 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages - sync2ProdAutoBuild(PBM_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PBM_REPO, COMPONENT) + } + } + stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + steps { + uploadTarballToDownloadsTesting(params.CLOUD, "pbm", "${VERSION}") } } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-backup-mongodb + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-backup-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-backup-mongodb-amd64 perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-backup-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-backup-mongodb-arm64 perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + PBM_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PBM_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PBM_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + + sudo docker manifest create --amend perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} perconalab/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} + + sudo docker manifest push perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + sudo docker manifest push perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + sudo docker manifest push perconalab/percona-backup-mongodb:${PBM_MAJOR_VERSION} + ''' + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-backup-mongodb + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-backup-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-backup-mongodb-amd64 percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + + sed -i "s/ENV PBM_VERSION.*/ENV PBM_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PBM_REPO_CH.*/ENV PBM_REPO_CH ${PBM_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-backup-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-backup-mongodb-arm64 percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 + sudo docker push percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + PBM_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PBM_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PBM_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + + sudo docker manifest create --amend percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 \ + percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} percona/percona-backup-mongodb:${VERSION}-${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} + + sudo docker manifest push percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION}.${PBM_PATCH_VERSION} + sudo docker manifest push percona/percona-backup-mongodb:${PBM_MAJOR_VERSION}.${PBM_MINOR_VERSION} + sudo docker manifest push percona/percona-backup-mongodb:${PBM_MAJOR_VERSION} + ''' + } + } + } + } } post { success { - slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } diff --git a/pbm/jenkins/percona-mongodb-backup.yml b/pbm/jenkins/percona-mongodb-backup.yml index 4403b1c4d4..533d2869c3 100644 --- a/pbm/jenkins/percona-mongodb-backup.yml +++ b/pbm/jenkins/percona-mongodb-backup.yml @@ -1,5 +1,5 @@ - job: - name: pbm-autobuild-RELEASE + name: hetzner-pbm-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/jenkins/percona-mongodb-backup.groovy diff --git a/pbm/pbm-docker-arm.groovy b/pbm/pbm-docker-arm.groovy index ae8198d04e..1e06e37d22 100644 --- a/pbm/pbm-docker-arm.groovy +++ b/pbm/pbm-docker-arm.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-64gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PBM_REPO_CH', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PBM_VERSION', defaultValue: '2.3.0-1', description: 'PBM version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') diff --git a/pbm/pbm-docker-arm.yml b/pbm/pbm-docker-arm.yml index 63f321df3a..00941cf881 100644 --- a/pbm/pbm-docker-arm.yml +++ b/pbm/pbm-docker-arm.yml @@ -1,5 +1,5 @@ - job: - name: pbm-docker-arm + name: hetzner-pbm-docker-arm project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-docker-arm.groovy diff --git a/pbm/pbm-docker.groovy b/pbm/pbm-docker.groovy index 5eee84ae00..80d75bc86c 100644 --- a/pbm/pbm-docker.groovy +++ b/pbm/pbm-docker.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PBM_REPO_CH', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PBM_VERSION', defaultValue: '2.0.2-1', description: 'PBM version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') diff --git a/pbm/pbm-docker.yml b/pbm/pbm-docker.yml index e38b2cb68c..bca6a88408 100644 --- a/pbm/pbm-docker.yml +++ b/pbm/pbm-docker.yml @@ -1,5 +1,5 @@ - job: - name: pbm-docker + name: hetzner-pbm-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-docker.groovy diff --git a/pbm/pbm-e2e-tests.groovy b/pbm/pbm-e2e-tests.groovy index 6dc8c6a99e..b5ea37c33e 100644 --- a/pbm/pbm-e2e-tests.groovy +++ b/pbm/pbm-e2e-tests.groovy @@ -17,11 +17,11 @@ void prepareCluster(String TEST_TYPE) { sh """ sudo mkdir -p /usr/local/lib/docker/cli-plugins - sudo curl -SL https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose + sudo curl -SL https://github.com/docker/compose/releases/download/v2.39.3/docker-compose-linux-x86_64 -o /usr/local/lib/docker/cli-plugins/docker-compose sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose docker compose version - wget https://download.docker.com/linux/static/stable/x86_64/docker-27.1.1.tgz -O /tmp/docker.tgz + wget https://download.docker.com/linux/static/stable/x86_64/docker-28.4.0.tgz -O /tmp/docker.tgz tar -xvf /tmp/docker.tgz sudo systemctl stop docker containerd sudo cp docker/* /usr/bin/ @@ -30,17 +30,23 @@ void prepareCluster(String TEST_TYPE) { git poll: false, branch: params.PBM_BRANCH, url: 'https://github.com/percona/percona-backup-mongodb.git' - withCredentials([file(credentialsId: 'PBM-AWS-S3', variable: 'PBM_AWS_S3_YML'), file(credentialsId: 'PBM-GCS-S3', variable: 'PBM_GCS_S3_YML'), file(credentialsId: 'PBM-AZURE', variable: 'PBM_AZURE_YML')]) { + withCredentials([file(credentialsId: 'PBM-AWS-S3', variable: 'PBM_AWS_S3_YML'), file(credentialsId: 'PBM-GCS-S3', variable: 'PBM_GCS_S3_YML'), file(credentialsId: 'PBM-GCS-HMAC-S3', variable: 'PBM_GCS_HMAC_S3_YML'), file(credentialsId: 'PBM-MINIO-S3', variable: 'PBM_MINIO_S3_YML'), file(credentialsId: 'PBM-AZURE', variable: 'PBM_AZURE_YML')]) { sh """ cp $PBM_AWS_S3_YML ./e2e-tests/docker/conf/aws.yaml + cp $PBM_MINIO_S3_YML ./e2e-tests/docker/conf/aws_minio.yaml cp $PBM_GCS_S3_YML ./e2e-tests/docker/conf/gcs.yaml + cp $PBM_GCS_HMAC_S3_YML ./e2e-tests/docker/conf/gcs_hmac.yaml cp $PBM_AZURE_YML ./e2e-tests/docker/conf/azure.yaml sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/aws.yaml + sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/aws_minio.yaml sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/gcs.yaml + sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/gcs_hmac.yaml sed -i s:pbme2etest:pbme2etest-${TEST_TYPE}:g ./e2e-tests/docker/conf/azure.yaml chmod 664 ./e2e-tests/docker/conf/aws.yaml + chmod 664 ./e2e-tests/docker/conf/aws_minio.yaml chmod 664 ./e2e-tests/docker/conf/gcs.yaml + chmod 664 ./e2e-tests/docker/conf/gcs_hmac.yaml chmod 664 ./e2e-tests/docker/conf/azure.yaml @@ -49,7 +55,7 @@ void prepareCluster(String TEST_TYPE) { } } -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) @@ -57,12 +63,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PBM_BRANCH', defaultValue: 'dev', description: 'PBM branch') } triggers { @@ -80,25 +87,16 @@ pipeline { parallel { stage('New cluster 8.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-newc-logic') runTest('run-new-cluster', '8.0', 'logical') } } - stage('New cluster 6.0 logical') { - agent { - label 'docker' - } - steps { - prepareCluster('60-newc-logic') - runTest('run-new-cluster', '6.0', 'logical') - } - } stage('New cluster 7.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-newc-logic') @@ -107,25 +105,16 @@ pipeline { } stage('Sharded 8.0 logical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('80-shrd-logic') runTest('run-sharded', '8.0', 'logical') } } - stage('Sharded 6.0 logical') { - agent { - label 'docker-32gb' - } - steps { - prepareCluster('60-shrd-logic') - runTest('run-sharded', '6.0', 'logical') - } - } stage('Sharded 7.0 logical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('70-shrd-logic') @@ -134,25 +123,16 @@ pipeline { } stage('Non-sharded 8.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-rs-logic') runTest('run-rs', '8.0', 'logical') } } - stage('Non-sharded 6.0 logical') { - agent { - label 'docker' - } - steps { - prepareCluster('60-rs-logic') - runTest('run-rs', '6.0', 'logical') - } - } stage('Non-sharded 7.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-rs-logic') @@ -161,25 +141,16 @@ pipeline { } stage('Single-node 8.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-single-logic') runTest('run-single', '8.0', 'logical') } } - stage('Single-node 6.0 logical') { - agent { - label 'docker' - } - steps { - prepareCluster('60-single-logic') - runTest('run-single', '6.0', 'logical') - } - } stage('Single-node 7.0 logical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-single-logic') @@ -188,25 +159,16 @@ pipeline { } stage('Sharded 8.0 physical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('80-shrd-phys') runTest('run-sharded', '8.0', 'physical') } } - stage('Sharded 6.0 physical') { - agent { - label 'docker-32gb' - } - steps { - prepareCluster('60-shrd-phys') - runTest('run-sharded', '6.0', 'physical') - } - } stage('Sharded 7.0 physical') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { prepareCluster('70-shrd-phys') @@ -215,25 +177,16 @@ pipeline { } stage('Non-sharded 8.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-rs-phys') runTest('run-rs', '8.0', 'physical') } } - stage('Non-sharded 6.0 physical') { - agent { - label 'docker' - } - steps { - prepareCluster('60-rs-phys') - runTest('run-rs', '6.0', 'physical') - } - } stage('Non-sharded 7.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-rs-phys') @@ -242,25 +195,16 @@ pipeline { } stage('Single-node 8.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('80-single-phys') runTest('run-single', '8.0', 'physical') } } - stage('Single-node 6.0 physical') { - agent { - label 'docker' - } - steps { - prepareCluster('60-single-phys') - runTest('run-single', '6.0', 'physical') - } - } stage('Single-node 7.0 physical') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { prepareCluster('70-single-phys') diff --git a/pbm/pbm-e2e-tests.yml b/pbm/pbm-e2e-tests.yml index 0d62378029..da29844f9b 100644 --- a/pbm/pbm-e2e-tests.yml +++ b/pbm/pbm-e2e-tests.yml @@ -1,5 +1,5 @@ - job: - name: pbm-e2e-tests + name: hetzner-pbm-e2e-tests project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'PBM_psmdb_jenkins' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-e2e-tests.groovy diff --git a/pbm/pbm-functional-tests-full.groovy b/pbm/pbm-functional-tests-full.groovy index 0affd472e8..ef11b58294 100644 --- a/pbm/pbm-functional-tests-full.groovy +++ b/pbm/pbm-functional-tests-full.groovy @@ -12,7 +12,7 @@ pipeline { } parameters { string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch or commit') - string(name: 'GO_VER', defaultValue: '1.22-bullseye', description: 'GOLANG docker image for building PBM from sources') + string(name: 'GO_VER', defaultValue: 'bookworm', description: 'GOLANG docker image for building PBM from sources') choice(name: 'instance', choices: ['docker-64gb','docker-64gb-aarch64'], description: 'Ec2 instance type for running tests') string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') } diff --git a/pbm/pbm-functional-tests.groovy b/pbm/pbm-functional-tests.groovy index 6c10d447d4..8ef0cc2a2f 100644 --- a/pbm/pbm-functional-tests.groovy +++ b/pbm/pbm-functional-tests.groovy @@ -13,7 +13,7 @@ pipeline { parameters { string(name: 'PBM_BRANCH', defaultValue: 'main', description: 'PBM branch') string(name: 'PSMDB', defaultValue: 'percona/percona-server-mongodb', description: 'PSMDB docker image') - string(name: 'GO_VER', defaultValue: '1.22-bullseye', description: 'GOLANG docker image for building PBM from sources') + string(name: 'GO_VER', defaultValue: 'bookworm', description: 'GOLANG docker image for building PBM from sources') choice(name: 'instance', choices: ['docker-64gb','docker-64gb-aarch64'], description: 'Ec2 instance type for running tests') string(name: 'TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') } diff --git a/pbm/pbm-release-test-run.groovy b/pbm/pbm-release-test-run.groovy index 8a424c3733..98078f096f 100644 --- a/pbm/pbm-release-test-run.groovy +++ b/pbm/pbm-release-test-run.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'master' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'master' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PBM_VERSION', defaultValue: '2.0.5', description: 'PBM Version') string(name: 'PBM_BRANCH', defaultValue: 'release-2.0.5', description: 'PBM Branch') } @@ -20,7 +21,7 @@ pipeline { stages { stage ('Run e2e tests') { steps { - build job: 'pbm-functional-tests-full', propagate: false, wait: true, parameters: [ string(name: 'PBM_BRANCH', value: params.PBM_BRANCH) ] + build job: 'hetzner-pbm-functional-tests-full', propagate: false, wait: true, parameters: [ string(name: 'PBM_BRANCH', value: params.PBM_BRANCH), booleanParam(name: 'ADD_JENKINS_MARKED_TESTS', value: true) ] } } stage ('Run package tests') { @@ -32,7 +33,7 @@ pipeline { steps { script { def version = params.PBM_VERSION + '-1' - build job: 'pbm-docker', parameters: [string(name: 'PBM_REPO_CH', value: "testing"), string(name: 'PBM_VERSION', value: version ), string(name: 'LATEST', value: "no") ] + build job: 'hetzner-pbm-docker', parameters: [string(name: 'PBM_REPO_CH', value: "testing"), string(name: 'PBM_VERSION', value: version ), string(name: 'LATEST', value: "no") ] } } } diff --git a/pbm/pbm-release-test-run.yml b/pbm/pbm-release-test-run.yml index aae98ef360..b369215e88 100644 --- a/pbm/pbm-release-test-run.yml +++ b/pbm/pbm-release-test-run.yml @@ -1,5 +1,5 @@ - job: - name: pbm-release-test-run + name: hetzner-pbm-release-test-run project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pbm/pbm-release-test-run.groovy diff --git a/pdmdb/pdmdb-site-check.groovy b/pdmdb/pdmdb-site-check.groovy index 30bba1c312..cc1d0dd23c 100644 --- a/pdmdb/pdmdb-site-check.groovy +++ b/pdmdb/pdmdb-site-check.groovy @@ -5,12 +5,16 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice( + name: 'CLOUD', + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build') string( defaultValue: '6.0.9-7', description: 'PSMDB Version for tests', diff --git a/pdmdb/pdmdb-site-check.yml b/pdmdb/pdmdb-site-check.yml index 255590420a..f9697e0834 100644 --- a/pdmdb/pdmdb-site-check.yml +++ b/pdmdb/pdmdb-site-check.yml @@ -1,5 +1,5 @@ - job: - name: pdmdb-site-check + name: hetzner-pdmdb-site-check project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pdmdb/pdmdb-site-check.groovy diff --git a/pdps/perl-DBD-Mysql.groovy b/pdps/perl-DBD-Mysql.groovy index 505c8ebfe3..48e5e69063 100644 --- a/pdps/perl-DBD-Mysql.groovy +++ b/pdps/perl-DBD-Mysql.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -11,6 +11,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { export build_dir=\$(pwd -P) docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " set -o xtrace + sed -i "s:archive.ubuntu.com:mirrors.edge.kernel.org:g" /etc/apt/sources.list cd \${build_dir} sed -i "s/^RPM_RELEASE=.*/RPM_RELEASE=${RPM_RELEASE}/g" builder.sh sed -i "s/^DEB_RELEASE=.*/DEB_RELEASE=${DEB_RELEASE}/g" builder.sh @@ -29,15 +30,19 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/perl5-dbi/DBD-mysql.git', description: 'URL for perl-DBD-MySQL repository', name: 'REPO') string( - defaultValue: '4_050', + defaultValue: '5_013', description: 'Tag/Branch for orchestartor repository', name: 'BRANCH') string( @@ -49,7 +54,7 @@ pipeline { description: 'Tag/Branch for perl-DBD-MySQL packaging repository', name: 'PACKAGE_REPO_BRANCH') string( - defaultValue: '4_050', + defaultValue: '5_013', description: 'General version of the product', name: 'VERSION') string( @@ -88,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build perl-DBD-MySQL generic source packages') { parallel { stage('Build perl-DBD-MySQL generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build perl-DBD-MySQL generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -126,184 +131,210 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -318,7 +349,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild("tools", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "tools", COMPONENT) } } @@ -333,6 +364,9 @@ pipeline { } failure { slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${BRANCH} by ${PACKAGE_REPO_BRANCH}" + } deleteDir() } always { diff --git a/pdps/perl-DBD-Mysql.yml b/pdps/perl-DBD-Mysql.yml index 9626d00dd4..46d1929026 100644 --- a/pdps/perl-DBD-Mysql.yml +++ b/pdps/perl-DBD-Mysql.yml @@ -1,5 +1,5 @@ - job: - name: perl-DBD-MySQL-RELEASE + name: hetzner-perl-DBD-MySQL-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pdps/perl-DBD-Mysql.groovy diff --git a/pdpxc/percona-replication-manager.groovy b/pdpxc/percona-replication-manager.groovy index 01d298c02b..dc02d7b04f 100644 --- a/pdpxc/percona-replication-manager.groovy +++ b/pdpxc/percona-replication-manager.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/replication-manager.git', description: 'URL for percona-replication-manager repository', @@ -66,7 +70,7 @@ pipeline { stages { stage('Create PRM source tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -85,36 +89,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PRM generic source packages') { parallel { stage('Build PRM generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PRM generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -123,197 +127,249 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -328,7 +384,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PRM_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PRM_REPO, COMPONENT) } } diff --git a/pdpxc/percona-replication-manager.yml b/pdpxc/percona-replication-manager.yml index 9af712b329..6c3911ff7c 100644 --- a/pdpxc/percona-replication-manager.yml +++ b/pdpxc/percona-replication-manager.yml @@ -1,5 +1,5 @@ - job: - name: prm-autobuild-RELEASE + name: hetzner-prm-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pdpxc/percona-replication-manager.groovy diff --git a/percona-release/jenkins/percona-release.groovy b/percona-release/jenkins/percona-release.groovy index 16043cbdc6..563328c930 100644 --- a/percona-release/jenkins/percona-release.groovy +++ b/percona-release/jenkins/percona-release.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -113,9 +113,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-repositories.git', description: 'URL for mysql-shell packaging repository', @@ -161,19 +165,19 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build percona-release packages') { parallel { stage('RPM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -182,38 +186,38 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('DEB') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:bionic", "DEB") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "DEB") sh ''' pwd ls -la test/deb cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } /* stage('RPM ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -221,17 +225,17 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('DEB ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "DEB") sh ''' pwd @@ -239,8 +243,8 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadRPMfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } */ @@ -249,15 +253,15 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild('prel', COMPONENT) + sync2ProdAutoBuild(params.CLOUD, 'prel', COMPONENT) } } @@ -266,7 +270,7 @@ pipeline { success { // slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - currentBuild.description = "Built on ${SHELL_BRANCH}, path to packages: experimental/${AWS_STASH_PATH}" + currentBuild.description = "Built on ${BUILD_BRANCH}, path to packages: experimental/${AWS_STASH_PATH}" } deleteDir() } diff --git a/percona-release/jenkins/percona-release.yml b/percona-release/jenkins/percona-release.yml index 34a1cf2d92..61257f56e8 100644 --- a/percona-release/jenkins/percona-release.yml +++ b/percona-release/jenkins/percona-release.yml @@ -1,5 +1,5 @@ - job: - name: percona-release-RELEASE + name: hetzner-percona-release-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: percona-release/jenkins/percona-release.groovy diff --git a/percona-telemetry-agent/percona-telemetry-agent.groovy b/percona-telemetry-agent/percona-telemetry-agent.groovy index 0e4c1481bf..4a5fd13e02 100644 --- a/percona-telemetry-agent/percona-telemetry-agent.groovy +++ b/percona-telemetry-agent/percona-telemetry-agent.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/telemetry-agent.git', description: 'URL for telemetry-agent repository', @@ -66,7 +70,7 @@ pipeline { stages { stage('Create telemetry agent source tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build telemetry agent generic source packages') { parallel { stage('Build telemetry agent generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build telemetry agent generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("debian:buster", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,210 +126,236 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Oracle Linux 10') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("amazonlinux:2023", "--build_rpm=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023 ARM') { + stage('Oracle Linux 10 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("amazonlinux:2023", "--build_rpm=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Amazon Linux 2023') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04) ARM') { + stage('Amazon Linux 2023 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -333,14 +363,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PTA_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PTA_REPO, COMPONENT) } } diff --git a/percona-telemetry-agent/percona-telemetry-agent.yml b/percona-telemetry-agent/percona-telemetry-agent.yml index f30cbc4a58..3123fa8cf1 100644 --- a/percona-telemetry-agent/percona-telemetry-agent.yml +++ b/percona-telemetry-agent/percona-telemetry-agent.yml @@ -1,5 +1,5 @@ - job: - name: percona-telemetry-agent-RELEASE + name: hetzner-percona-telemetry-agent-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: percona-telemetry-agent/percona-telemetry-agent.groovy diff --git a/plm/get-plm-branches.groovy b/plm/get-plm-branches.groovy new file mode 100644 index 0000000000..42ff411cee --- /dev/null +++ b/plm/get-plm-branches.groovy @@ -0,0 +1,109 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') + string( + defaultValue: 'https://github.com/percona/percona-link-mongodb.git', + description: 'URL for percona-link-mongodb repository', + name: 'GIT_REPO') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Get release branches') { + steps { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + EC=0 + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 ls s3://percona-jenkins-artifactory/percona-link-mongodb/branch_commit_id.properties ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || EC=\$? + + if [ \${EC} = 1 ]; then + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + + echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id.properties + + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-link-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + echo "START_NEW_BUILD=NO" > startBuild + else + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp s3://percona-jenkins-artifactory/percona-link-mongodb/branch_commit_id.properties . ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + source branch_commit_id.properties + + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release\\* | tail -1) + LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + + if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then + echo "START_NEW_BUILD=YES" > startBuild + else + echo "START_NEW_BUILD=NO" > startBuild + fi + + echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id.properties + echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id.properties + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id.properties s3://percona-jenkins-artifactory/percona-link-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 + fi + """ + } + START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() + BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME}").trim() + COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${COMMIT_ID}").trim() + VERSION = sh(returnStdout: true, script: "source branch_commit_id.properties; echo \${BRANCH_NAME} | sed s:release\\-::").trim() + } + } + } + stage('Build needed') { + when { + expression { START_NEW_BUILD == 'YES' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD}: build required + """ + } + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") + build job: 'hetzner-plm-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'VERSION', value: VERSION), string(name: 'COMPONENT', value: 'testing')] + } + } + stage('Build skipped') { + when { + expression { START_NEW_BUILD == 'NO' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD} build required + """ + } + } + } + } + post { + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/plm/get-plm-branches.yml b/plm/get-plm-branches.yml new file mode 100644 index 0000000000..d63d994de9 --- /dev/null +++ b/plm/get-plm-branches.yml @@ -0,0 +1,17 @@ +- job: + name: hetzner-plm-autobuild + project-type: pipeline + description: | + Do not edit this job through the web! + Check PLM repo for new release branches to build packages + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/get-plm-branches.groovy + triggers: + - timed: "H/15 * * * *" diff --git a/plm/hetzner-plm-site-check.groovy b/plm/hetzner-plm-site-check.groovy new file mode 100644 index 0000000000..d9b34e7115 --- /dev/null +++ b/plm/hetzner-plm-site-check.groovy @@ -0,0 +1,68 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'docker-x64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + string( + defaultValue: '0.6.0', + description: 'PLM Version for tests', + name: 'PLM_VERSION') + string( + defaultValue: 'main', + description: 'Branch for testing repository', + name: 'TESTING_BRANCH') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PLM_VERSION}" + } + } + } + stage('Checkout') { + steps { + deleteDir() + git poll: false, branch: TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + } + } + stage('Test') { + steps { + script { + sh """ + cd site_checks + docker run --env PLM_VERSION=${params.PLM_VERSION} --rm -v `pwd`:/tmp -w /tmp python bash -c 'pip3 install requests pytest setuptools && pytest -s --junitxml=junit.xml test_plm.py || [ \$? = 1 ] ' + """ + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: checking packages on the main site for PLM ${PLM_VERSION} - ok [${BUILD_URL}testReport/]") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: checking packages on the main site for PLM ${PLM_VERSION} - some links are broken [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: checking packages on the main site for PLM ${PLM_VERSION} - failed [${BUILD_URL}]" ) + } + always { + script { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } + } +} diff --git a/plm/hetzner-plm-site-check.yml b/plm/hetzner-plm-site-check.yml new file mode 100644 index 0000000000..fbd03960da --- /dev/null +++ b/plm/hetzner-plm-site-check.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-plm-site-check + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/hetzner-plm-site-check.groovy diff --git a/plm/percona-link-mongodb.groovy b/plm/percona-link-mongodb.groovy new file mode 100644 index 0000000000..83d7ed0558 --- /dev/null +++ b/plm/percona-link-mongodb.groovy @@ -0,0 +1,389 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/packaging/scripts/percona-link-mongodb_builder.sh -O percona-link-mongodb_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./percona-link-mongodb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./percona-link-mongodb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${GIT_BRANCH} --rpm_release=${RELEASE} --deb_release=${RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-link-mongodb.git', + description: 'URL for percona-link-mongodb repository', + name: 'GIT_REPO') + string( + defaultValue: 'main', + description: 'Tag/Branch for percona-link-mongodb repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM/DEB release value', + name: 'RELEASE') + string( + defaultValue: '1.0.1', + description: 'VERSION value', + name: 'VERSION') + string( + defaultValue: 'plm', + description: 'PTA repo name', + name: 'PTA_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Create Percona Link for MongoDB source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-link-mongodb.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-link-mongodb.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build Percona Link for MongoDB generic source packages') { + parallel { + stage('Build Percona Link for MongoDB generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build Percona Link for MongoDB generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build Percona Link for MongoDB RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + /* stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } */ // Commented out in scope of PKG-1083 + /* stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } */ // Commented out in scope of PKG-1083 + stage('Amazon Linux 2023') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Oraclelinux 8 tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_tarball=1") + + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PTA_REPO, COMPONENT) + } + } + stage('Push Tarballs to TESTING download area') { + steps { + uploadTarballToDownloadsTesting(params.CLOUD, "plm", "${VERSION}") + } + } + + } + post { + success { + //slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } + deleteDir() + } + failure { + //slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/plm/percona-link-mongodb.yml b/plm/percona-link-mongodb.yml new file mode 100644 index 0000000000..3c8f70b8ee --- /dev/null +++ b/plm/percona-link-mongodb.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-plm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/percona-link-mongodb.groovy + diff --git a/plm/plm-functional-tests.groovy b/plm/plm-functional-tests.groovy new file mode 100644 index 0000000000..574bdb31ca --- /dev/null +++ b/plm/plm-functional-tests.groovy @@ -0,0 +1,122 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +def set_agent(cloud, arch) { + if (arch == 'x86') { + if (cloud == 'Hetzner') { + return 'docker-x64' + } else { + return 'docker-64gb' + } + } else if (arch == 'arm') { + if (cloud == 'Hetzner') { + return 'docker-aarch64' + } else { + return 'docker-64gb-aarch64' + } + } +} + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'master' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PLM_BRANCH', defaultValue: 'main', description: 'PLM branch') + string(name: 'GO_VER', defaultValue: 'latest', description: 'GOLANG docker image for building PBM from sources') + choice(name: 'ARCH', choices: ['x86','arm'], description: 'Ec2 instance type for running tests') + string(name: 'PSMDB_TESTING_BRANCH', defaultValue: 'main', description: 'psmdb-testing repo branch') + string(name: 'TEST_FILTER', defaultValue: '', description: 'Optional pytest filter, f.e. T2 or T3') + booleanParam(name: 'ADD_JENKINS_MARKED_TESTS', defaultValue: true, description: 'Include tests with jenkins marker') + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PLM_BRANCH}" + } + } + } + stage ('Run tests') { + matrix { + agent { + label set_agent(params.CLOUD, params.ARCH) + } + axes { + axis { + name 'MONGODB_IMAGE' + values 'percona/percona-server-mongodb:6.0', 'percona/percona-server-mongodb:7.0', 'percona/percona-server-mongodb:8.0' + } + } + stages { + stage ('Run tests') { + steps { + withCredentials([string(credentialsId: 'olexandr_zephyr_token', variable: 'ZEPHYR_TOKEN')]) { + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + if [ ! -f "/usr/local/bin/docker-compose" ] ; then + if [ ${params.INSTANCE} = "docker-64gb-aarch64" ]; then + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-aarch64 -o /usr/local/bin/docker-compose + else + sudo curl -SL https://github.com/docker/compose/releases/download/v2.16.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose + fi + sudo chmod +x /usr/local/bin/docker-compose + fi + """ + + dir('psmdb-testing') { + git poll: false, branch: params.PSMDB_TESTING_BRANCH, url: 'https://github.com/Percona-QA/psmdb-testing.git' + } + + sh """ + cd psmdb-testing/plm-pytest + docker-compose build --no-cache + docker-compose up -d + if [ "${ADD_JENKINS_MARKED_TESTS}" = "true" ]; then JENKINS_FLAG="--jenkins"; else JENKINS_FLAG=""; fi + if [ -n "${params.TEST_FILTER}" ]; then + docker-compose run test pytest -v -s \$JENKINS_FLAG -k "${params.TEST_FILTER}" --junitxml=junit.xml || true + else + docker-compose run test pytest -v -s \$JENKINS_FLAG --junitxml=junit.xml || true + fi + docker-compose down -v --remove-orphans + curl -H "Content-Type:multipart/form-data" -H "Authorization: Bearer ${ZEPHYR_TOKEN}" -F "file=@junit.xml;type=application/xml" 'https://api.zephyrscale.smartbear.com/v2/automations/executions/junit?projectKey=PLM' -F 'testCycle={"name":"${JOB_NAME}-${BUILD_NUMBER}","customFields": { "PLM branch": "${PLM_BRANCH}","PSMDB docker image": "${MONGODB_IMAGE}","Instance": "${params.INSTANCE}"}};type=application/json' -i || true + """ + } + } + post { + always { + junit testResults: "**/junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true + sh """ + docker kill \$(docker ps -a -q) || true + docker rm \$(docker ps -a -q) || true + docker rmi -f \$(docker images -q | uniq) || true + sudo rm -rf ./* + """ + } + } + } + } + } + } + } + post { + success { + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: PLM ${PLM_BRANCH} - all tests passed") + } + unstable { + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: PLM ${PLM_BRANCH} - some tests failed [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: PLM ${PLM_BRANCH} - unexpected failure [${BUILD_URL}]") + } + } +} diff --git a/plm/plm-functional-tests.yml b/plm/plm-functional-tests.yml new file mode 100644 index 0000000000..bd1023ffeb --- /dev/null +++ b/plm/plm-functional-tests.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-plm-functional-tests + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/plm-functional-tests.groovy diff --git a/plm/plm_generate_sbom.groovy b/plm/plm_generate_sbom.groovy new file mode 100644 index 0000000000..87c833685e --- /dev/null +++ b/plm/plm_generate_sbom.groovy @@ -0,0 +1,448 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([ + string(credentialsId: 'SNYK_TOKEN', variable: 'SNYK_TOKEN'), + string(credentialsId: 'SNYK_ORG_TOKEN', variable: 'SNYK_ORG_TOKEN') + ]){ + sh """ + echo "Starting Build Stage" + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/packaging/plm_sbom/plm_generate_sbom.sh -O plm_generate_sbom.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} -e SNYK_TOKEN=${SNYK_TOKEN} -e SNYK_ORG_TOKEN=${SNYK_ORG_TOKEN} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + ls -laR + uname -a + bash -x ./plm_generate_sbom.sh --builddir=\${build_dir} --plm_version=${PLM_VERSION} --repo_type=${REPO_TYPE} --git_repo=${GIT_REPO} --git_branch=${GIT_BRANCH} ${STAGE_PARAM}" + curl -fsSL https://raw.githubusercontent.com/EvgeniyPatlan/sbom_verifier/main/install_sbom_verifier.sh | bash + bash sbom_verifier.sh plm_sbom/*.json + """ + } +} + +void uploadPLMSBOMToTestingDownloadServer(String productName, String packageVersion, String SBOMType) { + + script { + try { + uploadSBOMToDownloadsTesting(params.CLOUD, productName, packageVersion, SBOMType) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-link-mongodb.git', + description: 'URL for plm_sbom repository', + name: 'GIT_REPO') + string( + defaultValue: '0.5.0', + description: 'Version of Percona Link MongoDB', + name: 'PLM_VERSION') + string( + defaultValue: 'main', + description: 'Tag/Branch for percona-link-mongodb repository', + name: 'GIT_BRANCH') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Packaging repository type', + name: 'REPO_TYPE') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Generate SBOM') { + parallel { + stage('Generate PLM SBOM OL/8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + echo "Ran stash includes" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM OL/8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM OL/9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM OL/9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM AL 2023 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("amazonlinux:2023", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM AL 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("amazonlinux:2023", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } +/* stage('Generate PLM SBOM OL/10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM OL/10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } */ + stage('Generate PLM SBOM Jammy AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Jammy ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Noble AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Noble ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Bullseye AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Bullseye ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Bookworm AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PLM SBOM Bookworm ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PLM_SBOM/${PLM_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PLM_VERSION} + cat uploadPath-${PLM_VERSION} + """ + stash includes: "uploadPath-${PLM_VERSION}", name: "uploadPath-${PLM_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "plm_sbom/", AWS_STASH_PATH) + } + } + } + } //parallel + } // stage + stage('Upload SBOMS from AWS') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + uploadSBOMfromAWS(params.CLOUD, "plm_sbom/", AWS_STASH_PATH, "json", "${PLM_VERSION}") + } + } + stage('Push SBOMS to TESTING downloads area') { + steps { + cleanUpWS() + uploadPLMSBOMToTestingDownloadServer("plm_sbom", "${PLM_VERSION}", "json") + } + } + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/plm/plm_generate_sbom.yml b/plm/plm_generate_sbom.yml new file mode 100644 index 0000000000..5de371fb5f --- /dev/null +++ b/plm/plm_generate_sbom.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-plm_generate_sbom + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: plm/plm_generate_sbom.groovy + diff --git a/pmm/v3/pmm3-migration-tests.groovy b/pmm/v3/pmm3-migration-tests.groovy new file mode 100644 index 0000000000..716decd9e1 --- /dev/null +++ b/pmm/v3/pmm3-migration-tests.groovy @@ -0,0 +1,357 @@ +library changelog: false, identifier: 'lib@master', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +def versionsList = pmmVersion('list').reverse() +def latestVersion = versionsList.first() +def getMinorVersion(VERSION) { + return VERSION.split("\\.")[1].toInteger() +} + +pipeline { + agent { + label 'agent-amd64-ol9' + } + environment { + REMOTE_AWS_MYSQL_USER=credentials('pmm-dev-mysql-remote-user') + REMOTE_AWS_MYSQL_PASSWORD=credentials('pmm-dev-remote-password') + REMOTE_AWS_MYSQL57_HOST=credentials('pmm-dev-mysql57-remote-host') + REMOTE_MYSQL_HOST=credentials('mysql-remote-host') + REMOTE_MYSQL_USER=credentials('mysql-remote-user') + REMOTE_MYSQL_PASSWORD=credentials('mysql-remote-password') + GCP_SERVER_IP=credentials('GCP_SERVER_IP') + GCP_USER=credentials('GCP_USER') + GCP_USER_PASSWORD=credentials('GCP_USER_PASSWORD') + REMOTE_MONGODB_HOST=credentials('qa-remote-mongodb-host') + REMOTE_MONGODB_USER=credentials('qa-remote-mongodb-user') + REMOTE_MONGODB_PASSWORD=credentials('qa-remote-mongodb-password') + REMOTE_POSTGRESQL_HOST=credentials('qa-remote-pgsql-host') + REMOTE_POSTGRESQL_USER=credentials('qa-remote-pgsql-user') + REMOTE_POSTGRESSQL_PASSWORD=credentials('qa-remote-pgsql-password') + REMOTE_PROXYSQL_HOST=credentials('qa-remote-proxysql-host') + REMOTE_PROXYSQL_USER=credentials('qa-remote-proxysql-user') + REMOTE_PROXYSQL_PASSWORD=credentials('qa-remote-proxysql-password') + INFLUXDB_ADMIN_USER=credentials('influxdb-admin-user') + INFLUXDB_ADMIN_PASSWORD=credentials('influxdb-admin-password') + INFLUXDB_USER=credentials('influxdb-user') + INFLUXDB_USER_PASSWORD=credentials('influxdb-user-password') + MONITORING_HOST=credentials('monitoring-host') + PMM_QA_AURORA2_MYSQL_HOST=credentials('PMM_QA_AURORA2_MYSQL_HOST') + PMM_QA_AURORA2_MYSQL_PASSWORD=credentials('PMM_QA_AURORA2_MYSQL_PASSWORD') + PMM_QA_AWS_ACCESS_KEY_ID=credentials('PMM_QA_AWS_ACCESS_KEY_ID') + PMM_QA_AWS_ACCESS_KEY=credentials('PMM_QA_AWS_ACCESS_KEY') + MAILOSAUR_API_KEY=credentials('MAILOSAUR_API_KEY') + MAILOSAUR_SERVER_ID=credentials('MAILOSAUR_SERVER_ID') + MAILOSAUR_SMTP_PASSWORD=credentials('MAILOSAUR_SMTP_PASSWORD') + ZEPHYR_PMM_API_KEY=credentials('ZEPHYR_PMM_API_KEY') + } + parameters { + string( + defaultValue: 'v3', + description: 'Tag/Branch for UI Tests repository PMM V3', + name: 'PMM_V3_UI_GIT_BRANCH') + string( + defaultValue: 'main', + description: 'Tag/Branch for UI Tests repository PMM V2', + name: 'PMM_V2_UI_GIT_BRANCH') + string( + defaultValue: latestVersion, + description: 'PMM Server Version to test for Upgrade', + name: 'DOCKER_VERSION') + string( + defaultValue: latestVersion, + description: 'PMM Client Version to test for Upgrade', + name: 'CLIENT_VERSION') + string( + defaultValue: 'pmm3admin!', + description: 'pmm-server admin user default password', + name: 'ADMIN_PASSWORD') + string( + defaultValue: 'main', + description: 'Tag/Branch for pmm-qa repository', + name: 'PMM_QA_GIT_BRANCH') + choice( + choices: ['experimental', 'testing', 'release'], + description: 'Select pmm repo to upgrate to', + name: 'UPGRADE_TAG') + } + options { + skipDefaultCheckout() + } + stages { + stage('Prepare') { + steps { + // fetch pmm-ui-tests repository + git poll: false, + branch: PMM_V3_UI_GIT_BRANCH, + url: 'https://github.com/percona/pmm-ui-tests.git' + + slackSend channel: '#pmm-notifications', color: '#0000FF', message: "[${JOB_NAME}]: build started - ${BUILD_URL}" + sh ''' + sudo mkdir -p /srv/pmm-qa || : + pushd /srv/pmm-qa + sudo git clone --single-branch --branch ${PMM_QA_GIT_BRANCH} https://github.com/percona/pmm-qa.git . + sudo git checkout ${PMM_QA_GIT_COMMIT_HASH} + popd + sudo ln -s /usr/bin/chromium-browser /usr/bin/chromium + ''' + } + } + stage('Start Server Instance') { + steps { + sh ''' + docker network create pmm-qa || true + git checkout ${PMM_V2_UI_GIT_BRANCH} + PWD=$(pwd) PMM_SERVER_IMAGE=percona/pmm-server:${DOCKER_VERSION} docker-compose up -d + + ''' + waitForContainer('pmm-server', 'pmm-managed entered RUNNING state') + waitForContainer('pmm-agent_mongo', 'waiting for connections on port 27017') + waitForContainer('pmm-agent_mysql_5_7', "Server hostname (bind-address):") + waitForContainer('pmm-agent_postgres', 'PostgreSQL init process complete; ready for start up.') + sleep 20 + sh """ + bash -x testdata/db_setup.sh + """ + script { + env.SERVER_IP = "127.0.0.1" + env.PMM_UI_URL = "http://${env.SERVER_IP}/" + env.PMM_URL = "http://admin:${env.ADMIN_PASSWORD}@${env.SERVER_IP}" + } + } + } + stage('Change admin password') { + when { + expression { getMinorVersion(DOCKER_VERSION) >= 27 } + } + steps { + sh ''' + docker exec pmm-server change-admin-password ${ADMIN_PASSWORD} + ''' + } + } + stage('Enable Repo') { + parallel { + stage('Enable Experimental Repo') { + when { + expression { env.UPGRADE_TAG == "experimental" } + } + steps { + script { + sh """ + set -o errexit + set -o xtrace + docker exec pmm-server sed -i'' -e 's^/release/^/experimental/^' /etc/yum.repos.d/pmm2-server.repo + docker exec pmm-server percona-release enable pmm2-client experimental + docker exec pmm-server yum clean all + docker exec pmm-server yum clean metadata + """ + } + } + } + stage('Enable Testing Repo') { + when { + expression { env.UPGRADE_TAG == "testing" } + } + steps { + script { + sh """ + set -o errexit + set -o xtrace + docker exec pmm-server sed -i'' -e 's^/release/^/testing/^' /etc/yum.repos.d/pmm2-server.repo + docker exec pmm-server percona-release enable pmm2-client testing + docker exec pmm-server yum clean all + docker exec pmm-server yum clean metadata + """ + } + } + } + stage('Enable Release Repo') { + when { + expression { env.UPGRADE_TAG == "release" } + } + steps { + script { + sh """ + set -o errexit + set -o xtrace + docker exec pmm-server yum clean all + docker exec pmm-server yum clean metadata + """ + } + } + } + } + } + stage('Setup Client for PMM-Server') { + steps { + script{ + if(env.UPGRADE_TAG == "experimental") { + setupPMMClient(SERVER_IP, CLIENT_VERSION.trim(), "pmm2", "no", "no", "no", 'compose_setup', ADMIN_PASSWORD) + } else if (env.UPGRADE_TAG == "testing") { + setupPMMClient(SERVER_IP, CLIENT_VERSION.trim(), "pmm2", "no", "yes", "no", 'compose_setup', ADMIN_PASSWORD, "no") + } else { + setupPMMClient(SERVER_IP, CLIENT_VERSION.trim(), "pmm2", "no", "no", "no", 'compose_setup', ADMIN_PASSWORD, "no") + } + } + sh """ + set -o errexit + set -o xtrace + export PATH=$PATH:/usr/sbin + export PMM_CLIENT_VERSION=${CLIENT_VERSION} + bash /srv/pmm-qa/pmm-tests/pmm-framework.sh \ + --download \ + --pdpgsql-version 17 --ps-version 8.0 --mo-version 8.0 --addclient=pdpgsql,1 --addclient=ps,1 --mongo-replica-for-backup \ + --pmm2 + sleep 20 + """ + } + } + stage('Sanity check') { + steps { + sh ''' + echo \${PMM_URL} + timeout 100 bash -c \'while [[ "$(curl -s -o /dev/null -w \'\'%{http_code}\'\' \${PMM_URL}/ping)" != "200" ]]; do sleep 5; done\' || false + ''' + } + } + stage('Prepare nightly tests on migrated pmm.') { + steps { + script { + sh """ + curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash - + sudo yum install -y nodejs + node --version + npm ci + npx playwright install + envsubst < env.list > env.generated.list + """ + } + } + } + stage('Run pre migration Tests') { + options { + timeout(time: 150, unit: "MINUTES") + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'PMM_AWS_DEV', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js + export PWD=\$(pwd); + npx codeceptjs run --reporter mocha-multi -c pr.codecept.js --grep '@pmm-pre-migration' + git checkout ${PMM_V3_UI_GIT_BRANCH} + """ + } + } + } + stage('Migrate pmm2 to pmm3') { + steps { + script { + sh """ + echo "\$UPGRADE_TAG" + if [[ "\$UPGRADE_TAG" == "experimental" ]]; then + export PERCONA_REPOSITORY="experimental" + export DOCKER_TAG="3-dev-latest" + export DOCKER_REPO="perconalab/pmm-server" + elif [[ "\$UPGRADE_TAG" == "testing" ]]; then + export DOCKER_TAG=\$(wget -q "https://registry.hub.docker.com/v2/repositories/perconalab/pmm-server/tags?page_size=25&name=rc" -O - | jq -r .results[].name | grep 3.*.*-rc\$ | sort -V | tail -n1) + export DOCKER_REPO="perconalab/pmm-server" + elif [[ "\$UPGRADE_TAG" == "release" ]]; then + export DOCKER_TAG=\$(wget -q "https://registry.hub.docker.com/v2/repositories/percona/pmm-server/tags?page_size=25" -O - | jq -r .results[].name | grep 3.*.* | sort -V | tail -n1) + export DOCKER_REPO="percona/pmm-server" + fi + + echo "Percona repository is: \$PERCONA_REPOSITORY" + echo "Docker tag is: \$DOCKER_TAG" + + wget https://raw.githubusercontent.com/percona/pmm/refs/heads/v3/get-pmm.sh + chmod +x get-pmm.sh + ./get-pmm.sh -n pmm-server -b --network-name pmm-qa --tag "\$DOCKER_TAG" --repo "\$DOCKER_REPO" + + sudo percona-release enable pmm3-client \$UPGRADE_TAG + sudo yum install -y pmm-client + + listVar="rs101 rs102 rs103 rs201 rs202 rs203" + + for i in \$listVar; do + echo "\$i" + docker exec \$i percona-release enable pmm3-client \$UPGRADE_TAG + docker exec \$i yum install -y pmm-client + docker exec \$i sed -i "s/443/8443/g" /usr/local/percona/pmm/config/pmm-agent.yaml + docker exec \$i cat /usr/local/percona/pmm/config/pmm-agent.yaml + docker exec \$i systemctl restart pmm-agent + done + """ + env.SERVER_IP = "127.0.0.1" + env.PMM_UI_URL = "https://${env.SERVER_IP}/" + env.PMM_URL = "https://admin:${env.ADMIN_PASSWORD}@${env.SERVER_IP}" + } + } + } + stage('Sleep') { + steps { + sleep 300 + } + } + stage('Run Tests') { + options { + timeout(time: 150, unit: "MINUTES") + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'PMM_AWS_DEV', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + sed -i 's+http://localhost/+${PMM_UI_URL}/+g' pr.codecept.js + export PWD=\$(pwd); + npx codeceptjs run --reporter mocha-multi -c pr.codecept.js --grep '@pmm-migration' + """ + } + } + } + } + post { + always { + sh ''' + # fetch all the logs from PMM server + curl --insecure ${PMM_URL}/logs.zip --output logs.zip || true + + # get logs from systemd pmm-agent.service + if [[ ${CLIENT_VERSION} != http* ]]; then + journalctl -u pmm-agent.service > ./pmm-agent.log + fi + + # get logs from managed and update-perform + echo --- pmm-managed logs from pmm-server --- >> pmm-managed-full.log + docker exec pmm-server cat /srv/logs/pmm-managed.log >> pmm-managed-full.log || true + docker exec pmm-server cat /srv/logs/pmm-update-perform.log >> pmm-update-perform.log || true + echo --- pmm-update-perform logs from pmm-server --- >> pmm-update-perform.log + docker cp pmm-server:/srv/logs srv-logs + tar -zcvf srv-logs.tar.gz srv-logs + + # stop the containers + docker-compose down || true + docker rm -f $(sudo docker ps -a -q) || true + docker volume rm $(sudo docker volume ls -q) || true + sudo chown -R ec2-user:ec2-user . || true + ''' + script { + archiveArtifacts artifacts: 'pmm-managed-full.log' + archiveArtifacts artifacts: 'pmm-update-perform.log' + archiveArtifacts artifacts: 'pmm-agent.log' + archiveArtifacts artifacts: 'logs.zip' + archiveArtifacts artifacts: 'srv-logs.tar.gz' + + if (currentBuild.result == null || currentBuild.result == 'SUCCESS') { + junit 'tests/output/*.xml' + slackSend botUser: true, channel: '#pmm-notifications', color: '#00FF00', message: "[${JOB_NAME}]: build finished - ${BUILD_URL}" + archiveArtifacts artifacts: 'logs.zip' + } else { + junit 'tests/output/*.xml' + slackSend botUser: true, channel: '#pmm-notifications', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result} - ${BUILD_URL}" + archiveArtifacts artifacts: 'logs.zip' + archiveArtifacts artifacts: 'tests/output/*.png' + } + } + } + } +} diff --git a/ppg/etcd-arm.groovy b/ppg/etcd-arm.groovy new file mode 100644 index 0000000000..1e6ad382ea --- /dev/null +++ b/ppg/etcd-arm.groovy @@ -0,0 +1,261 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/etcd/etcd_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for packaging repository', + name: 'GIT_REPO') + string( + defaultValue: '17.6', + description: 'Tag/Branch for etcd packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: 'ppg-17.6', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create etcd source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/etcd.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/etcd.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build etcd generic source packages') { + parallel { + stage('Build etcd generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build etcd generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build etcd RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/etcd-arm.yml b/ppg/etcd-arm.yml new file mode 100644 index 0000000000..1afefa456a --- /dev/null +++ b/ppg/etcd-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-etcd-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/etcd-arm.groovy + diff --git a/ppg/etcd.groovy b/ppg/etcd.groovy index c2a222dd64..e914216855 100644 --- a/ppg/etcd.groovy +++ b/ppg/etcd.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/etcd/etcd_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --pg_version=${GIT_BRANCH} --etcd_version=${ETCD_VERSION} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for etcd repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '3.5.13', - description: 'etcd version', - name: 'ETCD_VERSION') - string( - defaultValue: '16.2', + defaultValue: '17.6', description: 'Tag/Branch for etcd packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.2', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,222 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build etcd generic source packages') { parallel { stage('Build etcd generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build etcd generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build etcd RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + //stage('Oracle Linux 8 ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + // buildStage("oraclelinux:8", "--build_rpm=1") + + // pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + // uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + // } + //} + stage('Oracle Linux 9 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + //stage('Oracle Linux 9 ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + // buildStage("oraclelinux:9", "--build_rpm=1") + + // pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + // uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + // } + //} + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + //stage('Ubuntu Jammy(22.04) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("ubuntu:jammy", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + //stage('Ubuntu Noble(24.04) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("ubuntu:noble", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Debian Bullseye(11) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + //stage('Debian Bullseye(11) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("debian:bullseye", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Debian bookworm(12) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + //stage('Debian bookworm(12) ARM') { + // agent { + // label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + // } + // steps { + // cleanUpWS() + // popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + // buildStage("debian:bookworm", "--build_deb=1") + + // pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // } + //} + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +302,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/etcd.yml b/ppg/etcd.yml index 6ac3ccc49c..d867dc8b11 100644 --- a/ppg/etcd.yml +++ b/ppg/etcd.yml @@ -1,5 +1,5 @@ - job: - name: etcd-RELEASE + name: hetzner-etcd-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/etcd.groovy diff --git a/ppg/gdal311.groovy b/ppg/gdal311.groovy new file mode 100644 index 0000000000..6d92bd7c86 --- /dev/null +++ b/ppg/gdal311.groovy @@ -0,0 +1,135 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_IMAGE, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir -p test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/master/gdal311/builder.sh -O builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_IMAGE} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --rpm_release=${RPM_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh "sudo rm -rf ./*" +} + +def AWS_STASH_PATH + +pipeline { + agent none + + parameters { + choice(name: 'CLOUD', choices: ['Hetzner', 'AWS'], description: 'Cloud infra for build') + string(name: 'GIT_REPO', defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for gdal311 repository') + string(name: 'GIT_BRANCH', defaultValue: '16.3', description: 'Tag/Branch for postgresql packaging repository') + string(name: 'RPM_RELEASE', defaultValue: '1', description: 'RPM release value') + string(name: 'PPG_REPO', defaultValue: 'ppg-16.3', description: 'PPG repo name') + choice(name: 'COMPONENT', choices: ['laboratory', 'testing', 'experimental'], description: 'Repo component to push packages to') + } + + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + + stages { + stage('Build SRPM (generic)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + buildStage("oraclelinux:8", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", "") + popArtifactFolder(params.CLOUD, "srpm/", "") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/gdal311.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/gdal311.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + stash includes: 'test/**', name: 'srpmContents' + } + } + stage('Build RPMs in Parallel') { + parallel { + stage('Build Oracle Linux 9 x86_64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:9", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + + stage('Build Oracle Linux 9 aarch64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:9", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Sign packages') { + agent { label 'docker' } + steps { + signRPM(params.CLOUD) + } + } + + stage('Push to public repository') { + agent { label 'docker' } + steps { + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } + + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } + always { + cleanUpWS() + deleteDir() + } + } +} \ No newline at end of file diff --git a/ppg/gdal311.yml b/ppg/gdal311.yml new file mode 100644 index 0000000000..f6fa21a532 --- /dev/null +++ b/ppg/gdal311.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-gdal311-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/gdal311.groovy \ No newline at end of file diff --git a/ppg/gdal385.groovy b/ppg/gdal385.groovy new file mode 100644 index 0000000000..57c2cd3239 --- /dev/null +++ b/ppg/gdal385.groovy @@ -0,0 +1,135 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_IMAGE, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir -p test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/master/gdal385/builder.sh -O builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_IMAGE} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --rpm_release=${RPM_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh "sudo rm -rf ./*" +} + +def AWS_STASH_PATH + +pipeline { + agent none + + parameters { + choice(name: 'CLOUD', choices: ['Hetzner', 'AWS'], description: 'Cloud infra for build') + string(name: 'GIT_REPO', defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for gdal385 repository') + string(name: 'GIT_BRANCH', defaultValue: '16.3', description: 'Tag/Branch for postgresql packaging repository') + string(name: 'RPM_RELEASE', defaultValue: '1', description: 'RPM release value') + string(name: 'PPG_REPO', defaultValue: 'ppg-16.3', description: 'PPG repo name') + choice(name: 'COMPONENT', choices: ['laboratory', 'testing', 'experimental'], description: 'Repo component to push packages to') + } + + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + + stages { + stage('Build SRPM (generic)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + buildStage("oraclelinux:8", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", "") + popArtifactFolder(params.CLOUD, "srpm/", "") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/gdal385.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/gdal385.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + stash includes: 'test/**', name: 'srpmContents' + } + } + stage('Build RPMs in Parallel') { + parallel { + stage('Build Oracle Linux 8 x86_64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:8", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + + stage('Build Oracle Linux 8 aarch64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:8", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Sign packages') { + agent { label 'docker' } + steps { + signRPM(params.CLOUD) + } + } + + stage('Push to public repository') { + agent { label 'docker' } + steps { + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } + + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } + always { + cleanUpWS() + deleteDir() + } + } +} \ No newline at end of file diff --git a/ppg/gdal385.yml b/ppg/gdal385.yml new file mode 100644 index 0000000000..1d88acd133 --- /dev/null +++ b/ppg/gdal385.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-gdal385-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/gdal385.groovy \ No newline at end of file diff --git a/ppg/haproxy-arm.groovy b/ppg/haproxy-arm.groovy new file mode 100644 index 0000000000..828c724e8a --- /dev/null +++ b/ppg/haproxy-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/haproxy-builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --build_branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/haproxy-packaging.git', + description: 'URL for haproxy repository', + name: 'GIT_REPO') + string( + defaultValue: 'v2.6.14', + description: 'Tag/Branch for haproxy repository', + name: 'PG_BRANCH') + string( + defaultValue: 'main', + description: 'Tag/Branch for haproxy packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create haproxy source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/haproxy.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/haproxy.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build haproxy generic source packages') { + parallel { + stage('Build haproxy generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build haproxy generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build haproxy RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/haproxy-arm.yml b/ppg/haproxy-arm.yml new file mode 100644 index 0000000000..9b50aa6372 --- /dev/null +++ b/ppg/haproxy-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-haproxy-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/haproxy-arm.groovy diff --git a/ppg/haproxy.groovy b/ppg/haproxy.groovy index cc98a2b312..b30b1867b5 100644 --- a/ppg/haproxy.groovy +++ b/ppg/haproxy.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/haproxy-packaging.git', description: 'URL for haproxy repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build haproxy generic source packages') { parallel { stage('Build haproxy generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build haproxy generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,184 +126,210 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04) ARM') { + stage('Oracle Linux 10 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -307,14 +337,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/haproxy.yml b/ppg/haproxy.yml index db6a0de85a..2130ec0a11 100644 --- a/ppg/haproxy.yml +++ b/ppg/haproxy.yml @@ -1,5 +1,5 @@ - job: - name: haproxy-RELEASE + name: hetzner-haproxy-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/haproxy.groovy diff --git a/ppg/llvm-arm.groovy b/ppg/llvm-arm.groovy new file mode 100644 index 0000000000..119975fb76 --- /dev/null +++ b/ppg/llvm-arm.groovy @@ -0,0 +1,181 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir -p test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/llvm/llvm_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --rpm_release=${RPM_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for llvm repository', + name: 'GIT_REPO') + string( + defaultValue: '16.3', + description: 'Tag/Branch for postgresql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: 'ppg-16.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Build LLVM RPMs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + buildStage("oraclelinux:8", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/llvm.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + buildStage("oraclelinux:8", "--build_rpm=1") + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + buildStage("oraclelinux:9", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/llvm.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + buildStage("oraclelinux:9", "--build_rpm=1") + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + buildStage("oraclelinux:10", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/llvm.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + buildStage("oraclelinux:10", "--build_rpm=1") + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/llvm-arm.yml b/ppg/llvm-arm.yml new file mode 100644 index 0000000000..34b53e66a7 --- /dev/null +++ b/ppg/llvm-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-llvm-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/llvm-arm.groovy + diff --git a/ppg/llvm.groovy b/ppg/llvm.groovy index 724bbded75..2d892fc7e6 100644 --- a/ppg/llvm.groovy +++ b/ppg/llvm.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for llvm repository', @@ -63,13 +67,13 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() buildStage("oraclelinux:8", "--get_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - popArtifactFolder("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -83,19 +87,19 @@ pipeline { } stash includes: 'uploadPath', name: 'uploadPath' buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() buildStage("oraclelinux:9", "--get_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - popArtifactFolder("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -109,8 +113,34 @@ pipeline { } stash includes: 'uploadPath', name: 'uploadPath' buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + buildStage("oraclelinux:10", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/llvm.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/llvm.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + buildStage("oraclelinux:10", "--build_rpm=1") + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -118,13 +148,13 @@ pipeline { stage('Sign packages') { steps { - signRPM() + signRPM(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/llvm.yml b/ppg/llvm.yml index 367225bf20..b01532d210 100644 --- a/ppg/llvm.yml +++ b/ppg/llvm.yml @@ -1,5 +1,5 @@ - job: - name: llvm-RELEASE + name: hetzner-llvm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/llvm.groovy diff --git a/ppg/patroni-arm.groovy b/ppg/patroni-arm.groovy new file mode 100644 index 0000000000..3f3d8b0344 --- /dev/null +++ b/ppg/patroni-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/patroni/patroni_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for patroni repository', + name: 'GIT_REPO') + string( + defaultValue: '3.1.0', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for patroni repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PATRONI source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/patroni.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/patroni.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PATRONI generic source packages') { + parallel { + stage('Build PATRONI generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PATRONI generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PATRONI RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/patroni-arm.yml b/ppg/patroni-arm.yml new file mode 100644 index 0000000000..f887959734 --- /dev/null +++ b/ppg/patroni-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-patroni-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/patroni-arm.groovy + diff --git a/ppg/patroni.groovy b/ppg/patroni.groovy index cc2a44385c..34e6649c08 100644 --- a/ppg/patroni.groovy +++ b/ppg/patroni.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/patroni/patroni_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for patroni repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '3.1.0', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', - description: 'Tag/Branch for patroni repository', + defaultValue: '17.6', + description: 'Tag/Branch for patroni packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,183 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PATRONI generic source packages') { parallel { stage('Build PATRONI generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PATRONI generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PATRONI RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +263,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/patroni.yml b/ppg/patroni.yml index 89c355e8b3..84fd41cf85 100644 --- a/ppg/patroni.yml +++ b/ppg/patroni.yml @@ -1,5 +1,5 @@ - job: - name: patroni-RELEASE + name: hetzner-patroni-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/patroni.groovy diff --git a/ppg/percona-postgis-arm.groovy b/ppg/percona-postgis-arm.groovy new file mode 100644 index 0000000000..2dbdb390be --- /dev/null +++ b/ppg/percona-postgis-arm.groovy @@ -0,0 +1,248 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgis/postgis_builder.sh -O postgis_builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./postgis_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./postgis_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --postgis_gitrepo=${POSTGIS_GITREPO} --postgis_branch=${POSTGIS_BRANCH} --postgis_ver=${POSTGIS_VERSION} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for postgis packaging repository', + name: 'GIT_REPO') + string( + defaultValue: '15.4', + description: 'Tag/Branch for postgis packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: 'https://github.com/postgis/postgis.git', + description: 'URL for postgis repository', + name: 'POSTGIS_GITREPO') + string( + defaultValue: 'stable-3.3', + description: 'Tag/Branch for postgis repository', + name: 'POSTGIS_BRANCH') + string( + defaultValue: '3.3', + description: 'POSTGIS release value', + name: 'POSTGIS_VERSION') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-15.4', + description: 'POSTGIS repo name', + name: 'POSTGIS_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create POSTGIS source tarball') { + steps { + slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgis.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgis.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build POSTGIS generic source packages') { + parallel { + stage('Build POSTGIS generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build POSTGIS generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build POSTGIS RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, POSTGIS_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/percona-postgis-arm.yml b/ppg/percona-postgis-arm.yml new file mode 100644 index 0000000000..4ea7523458 --- /dev/null +++ b/ppg/percona-postgis-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-percona-postgis-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/percona-postgis-arm.groovy + diff --git a/ppg/percona-postgis.groovy b/ppg/percona-postgis.groovy index 97d750e3ec..3760d3f8d3 100644 --- a/ppg/percona-postgis.groovy +++ b/ppg/percona-postgis.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,13 +8,16 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgis/postgis_builder.sh -O postgis_builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P export build_dir=\$(pwd -P) docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " set -o xtrace cd \${build_dir} bash -x ./postgis_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./postgis_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --postgis_gitrepo=${POSTGIS_GITREPO} --postgis_branch=${POSTGIS_BRANCH} --postgis_ver=${POSTGIS_VERSION} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./postgis_builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -28,41 +31,25 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for postgis packaging repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '15.4', + defaultValue: '17.6', description: 'Tag/Branch for postgis packaging repository', name: 'GIT_BRANCH') string( - defaultValue: 'https://github.com/postgis/postgis.git', - description: 'URL for postgis repository', - name: 'POSTGIS_GITREPO') - string( - defaultValue: 'stable-3.3', - description: 'Tag/Branch for postgis repository', - name: 'POSTGIS_BRANCH') - string( - defaultValue: '3.3', - description: 'POSTGIS release value', - name: 'POSTGIS_VERSION') - string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-15.4', - description: 'POSTGIS repo name', - name: 'POSTGIS_REPO') + defaultValue: 'ppg-17.6', + description: 'PPG repo name', + name: 'PPG_REPO') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', @@ -91,131 +78,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build POSTGIS generic source packages') { parallel { stage('Build POSTGIS generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build POSTGIS generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build POSTGIS RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } - stage('Ubuntu Focal(20.04)') { + } + stage('Oracle Linux 10 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bookworm(12)') { + stage('Debian Bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -223,14 +327,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(POSTGIS_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/percona-postgis.yml b/ppg/percona-postgis.yml index 398e3d1bdc..9e8c673c14 100644 --- a/ppg/percona-postgis.yml +++ b/ppg/percona-postgis.yml @@ -1,5 +1,5 @@ - job: - name: percona-postgis-RELEASE + name: hetzner-percona-postgis-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/percona-postgis.groovy diff --git a/ppg/pg17_autobuild.groovy b/ppg/pg17_autobuild.groovy new file mode 100644 index 0000000000..fa3dc01742 --- /dev/null +++ b/ppg/pg17_autobuild.groovy @@ -0,0 +1,110 @@ +library changelog: false, identifier: 'lib@master', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +pipeline { + agent { + label 'docker' + } + parameters { + string( + defaultValue: 'https://github.com/percona/postgres.git', + description: 'URL for pg repository', + name: 'GIT_REPO') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Get release branches') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + EC=0 + aws s3 ls s3://percona-jenkins-artifactory/pg17/branch_commit_id_17.properties || EC=\$? + + if [ \${EC} = 1 ]; then + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-17\\* | tail -1) + BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + + echo "BRANCH_NAME=\${BRANCH_NAME}" > branch_commit_id_17.properties + echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_17.properties + + aws s3 cp branch_commit_id_17.properties s3://percona-jenkins-artifactory/pg17/ + echo "START_NEW_BUILD=NO" > startBuild + else + aws s3 cp s3://percona-jenkins-artifactory/pg17/branch_commit_id_17.properties . + source ./branch_commit_id_17.properties + cat ./branch_commit_id_17.properties + + LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-17\\* | tail -1) + LATEST_BRANCH_NAME=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d "/" -f 3) + LATEST_COMMIT_ID=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d " " -f 1) + VERSION=\$(echo \${LATEST_RELEASE_BRANCH} | cut -d - -f 2-3) + + if [ "x\${COMMIT_ID}" != "x\${LATEST_COMMIT_ID}" ] || [ "x\${BRANCH_NAME}" != "x\${LATEST_BRANCH_NAME}" ]; then + echo "START_NEW_BUILD=YES" > startBuild + else + echo "START_NEW_BUILD=NO" > startBuild + fi + + echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_17.properties + echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_17.properties + echo "VERSION=\${VERSION}" >> branch_commit_id_17.properties + aws s3 cp branch_commit_id_17.properties s3://percona-jenkins-artifactory/pg17/ + fi + """ + } + script { + START_NEW_BUILD = sh(returnStdout: true, script: "source ./startBuild; echo \${START_NEW_BUILD}").trim() + BRANCH_NAME = sh(returnStdout: true, script: "source ./branch_commit_id_17.properties; echo \${BRANCH_NAME}").trim() + COMMIT_ID = sh(returnStdout: true, script: "source ./branch_commit_id_17.properties; echo \${COMMIT_ID}").trim() + VERSION = sh(returnStdout: true, script: "source ./branch_commit_id_17.properties; echo \${BRANCH_NAME} | cut -d - -f 2-3 ").trim() + } + + } + + } + stage('Build packages needed') { + when { + expression { START_NEW_BUILD == 'YES' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD}: build required + """ + } + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") + build job: 'hetzner-postgresql-server-17_nightly' + build job: 'hetzner-pg_tarballs-nightly' + + } + } + stage('Build skipped') { + when { + expression { START_NEW_BUILD == 'NO' } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + echo ${START_NEW_BUILD} build required + """ + } + } + } + } + post { + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} \ No newline at end of file diff --git a/ppg/pg17_autobuild.yml b/ppg/pg17_autobuild.yml new file mode 100644 index 0000000000..3d9ab10cb3 --- /dev/null +++ b/ppg/pg17_autobuild.yml @@ -0,0 +1,17 @@ +- job: + name: pg17-autobuild + project-type: pipeline + description: | + Do not edit this job through the web! + Check PG repo for new release branches to build packages + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pg/pg17_autobuild.groovy + triggers: + - timed: "H/15 * * * *" diff --git a/ppg/pg_cron-arm.groovy b/ppg/pg_cron-arm.groovy new file mode 100644 index 0000000000..ad1ec17ed7 --- /dev/null +++ b/ppg/pg_cron-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_cron/pg_cron_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_cron repository', + name: 'GIT_REPO') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_cron repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_cron packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_cron source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_cron.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_cron.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_cron generic source packages') { + parallel { + stage('Build pg_cron generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_cron generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_cron RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_cron-arm.yml b/ppg/pg_cron-arm.yml new file mode 100644 index 0000000000..e3ee342076 --- /dev/null +++ b/ppg/pg_cron-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_cron-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_cron-arm.groovy + diff --git a/ppg/pg_cron.groovy b/ppg/pg_cron.groovy index ee050a9606..b54a69a92a 100644 --- a/ppg/pg_cron.groovy +++ b/ppg/pg_cron.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_cron repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_cron generic source packages') { parallel { stage('Build pg_cron generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_cron generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_cron.yml b/ppg/pg_cron.yml index 9b2100ee73..3082d35395 100644 --- a/ppg/pg_cron.yml +++ b/ppg/pg_cron.yml @@ -1,5 +1,5 @@ - job: - name: pg_cron-RELEASE + name: hetzner-pg_cron-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_cron.groovy diff --git a/ppg/pg_gather-arm.groovy b/ppg/pg_gather-arm.groovy new file mode 100644 index 0000000000..eac114f681 --- /dev/null +++ b/ppg/pg_gather-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_gather/pggather_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_gather repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for pg_gather repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_gather packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_gather source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_gather.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_gather.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_gather generic source packages') { + parallel { + stage('Build pg_gather generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_gather generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_gather RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_gather-arm.yml b/ppg/pg_gather-arm.yml new file mode 100644 index 0000000000..db7739ea75 --- /dev/null +++ b/ppg/pg_gather-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_gather-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_gather-arm.groovy + diff --git a/ppg/pg_gather.groovy b/ppg/pg_gather.groovy index eebe5d5392..00ebd48cec 100644 --- a/ppg/pg_gather.groovy +++ b/ppg/pg_gather.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_gather/pggather_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_gather repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for pg_gather repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', + defaultValue: '17.6', description: 'Tag/Branch for pg_gather packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,247 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_gather generic source packages') { parallel { stage('Build pg_gather generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_gather generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build pg_gather RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +327,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_gather.yml b/ppg/pg_gather.yml index 8000fc283f..f99ca8b797 100644 --- a/ppg/pg_gather.yml +++ b/ppg/pg_gather.yml @@ -1,5 +1,5 @@ - job: - name: pg_gather-RELEASE + name: hetzner-pg_gather-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_gather.groovy diff --git a/ppg/pg_generate_sbom.groovy b/ppg/pg_generate_sbom.groovy new file mode 100644 index 0000000000..5b6b9c7767 --- /dev/null +++ b/ppg/pg_generate_sbom.groovy @@ -0,0 +1,420 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([ + string(credentialsId: 'SNYK_TOKEN', variable: 'SNYK_TOKEN'), + string(credentialsId: 'SNYK_ORG_TOKEN', variable: 'SNYK_ORG_TOKEN') + ]){ + sh """ + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_sbom/pg_generate_sbom.sh -O pg_generate_sbom.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} -e SNYK_TOKEN=${SNYK_TOKEN} -e SNYK_ORG_TOKEN=${SNYK_ORG_TOKEN} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pg_generate_sbom.sh --pg_version=${PG_VERSION} --repo_type=${REPO_TYPE} ${STAGE_PARAM}" + curl -fsSL https://raw.githubusercontent.com/EvgeniyPatlan/sbom_verifier/main/install_sbom_verifier.sh | bash + bash sbom_verifier.sh pg_sbom/*.json + + """ + } +} + +void uploadPGSBOMToTestingDownloadServer(String productName, String packageVersion, String SBOMType) { + + script { + try { + uploadSBOMToDownloadsTesting(params.CLOUD, productName, packageVersion, SBOMType) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_sbom repository', + name: 'GIT_REPO') + string( + defaultValue: '17.4', + description: 'Version of PostgreSQL server', + name: 'PG_VERSION') + string( + defaultValue: 'main', + description: 'Tag/Branch for postgres packaging repository', + name: 'GIT_BRANCH') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Packaging repository type', + name: 'REPO_TYPE') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Generate SBOM') { + parallel { + stage('Generate PG SBOM OL/8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM OL/10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Jammy AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Jammy ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Noble AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Noble ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bullseye AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bullseye ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bookworm AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + stage('Generate PG SBOM Bookworm ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PG_SBOM/${PG_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PG_VERSION} + cat uploadPath-${PG_VERSION} + """ + stash includes: "uploadPath-${PG_VERSION}", name: "uploadPath-${PG_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "pg_sbom/", AWS_STASH_PATH) + uploadSBOMfromAWS(params.CLOUD, "pg_sbom/", AWS_STASH_PATH, "json", "${PG_VERSION}") + uploadPGSBOMToTestingDownloadServer("pg_sbom", "${PG_VERSION}", "json") + } + } + } + } //parallel + } // stage + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_generate_sbom.yml b/ppg/pg_generate_sbom.yml new file mode 100644 index 0000000000..32966dd55c --- /dev/null +++ b/ppg/pg_generate_sbom.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_generate_sbom + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_generate_sbom.groovy + diff --git a/ppg/pg_percona_telemetry_autobuild.groovy b/ppg/pg_percona_telemetry_autobuild.groovy index 7791e6aba8..7c2e3b8eb8 100644 --- a/ppg/pg_percona_telemetry_autobuild.groovy +++ b/ppg/pg_percona_telemetry_autobuild.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) @@ -24,17 +24,18 @@ void installCli(String PLATFORM) { void buildStage(String DOCKER_OS, String STAGE_PARAM) { sh """ - echo "Docker: $DOCKER_OS, Release: PG$PG_RELEASE, Stage: $STAGE_PARAM" set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_percona_telemetry_builder.sh -O ppt_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_percona_telemetry_builder.sh -o ppt_builder.sh + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/percona-packaging/scripts/pg_percona_telemetry_builder.sh -O ppt_builder.sh pwd -P + ls -laR export build_dir=\$(pwd -P) - set -o xtrace - cd \${build_dir} - sudo bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 - bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} "$STAGE_PARAM" - """ + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=${PG_RELEASE} --ppg_repo_name=${PPG_REPO} ${STAGE_PARAM}" + """ } void cleanUpWS() { @@ -47,9 +48,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: '1.0.0', description: 'General version of the product', @@ -78,7 +83,7 @@ pipeline { choice( name: 'PG_RELEASE', description: 'PPG major version to test', - choices: ['11', '12', '13', '14', '15', '16'] + choices: ['11', '12', '13', '14', '15', '16', '17'] ) string( defaultValue: 'ppg-16.0', @@ -98,14 +103,13 @@ pipeline { stages { stage('Download source from github') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo '====> Source will be downloaded from github' //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") cleanUpWS() - installCli("deb") - buildStage("ubuntu:focal", "--get_sources=1") + buildStage("ubuntu:jammy", "--get_sources=1") sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-percona-telemetry.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -120,170 +124,312 @@ pipeline { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/pg-percona-telemetry.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } //stage stage('Build percona_pg_telemetry generic source packages') { parallel { stage('Source rpm') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry generic source rpm" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Source deb') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry generic source deb" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_source_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } //stage } //parallel } //stage stage('Build percona_pg_telemetry RPMs') { parallel { - stage('OL 8') { + stage('OL 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 8 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 8 ARM') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { echo "====> Build percona_pg_telemetry rpm on OL 8 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } //stage - stage('OL 9') { + stage('OL 9 AMD') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } //stage } //parallel } //stage stage('Build percona_pg_telemetry DEBs') { parallel { - stage('Ubuntu 20.04') { + stage('Ubuntu 20.04 AMD') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry deb on Ubuntu 20.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu 20.04 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 20.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu 22.04 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 22.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Ubuntu 22.04') { + stage('Ubuntu 22.04 ARM') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { echo "====> Build percona_pg_telemetry deb on Ubuntu 22.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { echo "====> Build percona_pg_telemetry deb on Ubuntu 24.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Debian 11') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 24.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 11 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 11 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 11 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { echo "====> Build percona_pg_telemetry deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 12 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 12 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage - stage('Debian 12') { + stage('Debian 12 ARM') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { echo "====> Build percona_pg_telemetry deb on Debian 12 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 13 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 13 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 13 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 13 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } //stage } //parallel } //stage + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } } //stages diff --git a/ppg/pg_percona_telemetry_autobuild.yml b/ppg/pg_percona_telemetry_autobuild.yml index d731397664..f16cfc2893 100644 --- a/ppg/pg_percona_telemetry_autobuild.yml +++ b/ppg/pg_percona_telemetry_autobuild.yml @@ -1,5 +1,5 @@ - job: - name: pg_percona_telemetry-autobuild-RELEASE + name: hetzner-pg_percona_telemetry-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_percona_telemetry_autobuild.groovy diff --git a/ppg/pg_percona_telemetry_autobuild_arm.groovy b/ppg/pg_percona_telemetry_autobuild_arm.groovy new file mode 100644 index 0000000000..0094fc944a --- /dev/null +++ b/ppg/pg_percona_telemetry_autobuild_arm.groovy @@ -0,0 +1,300 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +void installCli(String PLATFORM) { + sh """ + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip gnupg2 + elif [ ${PLATFORM} = "rpm" ]; then + sudo yum -y install wget curl unzip gnupg2 + fi + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + """ +} +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/percona-packaging/scripts/pg_percona_telemetry_builder.sh -O ppt_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./ppt_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=${PG_RELEASE} --ppg_repo_name=${PPG_REPO} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: '1.0.0', + description: 'General version of the product', + name: 'VERSION' + ) + string( + defaultValue: 'https://github.com/Percona-Lab/percona_pg_telemetry.git', + description: 'percona_pg_telemetry repo', + name: 'GIT_REPO' + ) + string( + defaultValue: 'main', + description: 'Branch for tests', + name: 'BRANCH' + ) + string( + defaultValue: '1', + description: 'rpm release number', + name: 'RPM_RELEASE' + ) + string( + defaultValue: '1', + description: 'deb release number', + name: 'DEB_RELEASE' + ) + choice( + name: 'PG_RELEASE', + description: 'PPG major version to test', + choices: ['11', '12', '13', '14', '15', '16', '17'] + ) + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Download source from github') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + echo '====> Source will be downloaded from github' + //slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("ubuntu:jammy", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-percona-telemetry.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg-percona-telemetry.properties + cat uploadPath + cat awsUploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + stash includes: 'test/pg-percona-telemetry.properties', name: 'properties' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } //stage + stage('Build percona_pg_telemetry generic source packages') { + parallel { + stage('Source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry generic source rpm" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry generic source deb" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } //stage + } //parallel + } //stage + stage('Build percona_pg_telemetry RPMs') { + parallel { + stage('OL 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 8 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + stage('OL 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry rpm on OL 9 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } //stage + } //parallel + } //stage + stage('Build percona_pg_telemetry DEBs') { + parallel { + stage('Ubuntu 20.04') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 20.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu 22.04') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 22.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Ubuntu 24.04 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 11') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 11 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + stage('Debian 12') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + echo "====> Build percona_pg_telemetry deb on Debian 12 PG${PG_RELEASE}" + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } //stage + } //parallel + } //stage + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } //stages + post { + success { + //slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Success" + } + failure { + //slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Failure" + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + echo "Always" + } + } +} diff --git a/ppg/pg_percona_telemetry_autobuild_arm.yml b/ppg/pg_percona_telemetry_autobuild_arm.yml new file mode 100644 index 0000000000..ae2d3479f8 --- /dev/null +++ b/ppg/pg_percona_telemetry_autobuild_arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pg_percona_telemetry-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_percona_telemetry_autobuild_arm.groovy diff --git a/ppg/pg_source_tarballs.groovy b/ppg/pg_source_tarballs.groovy index bab623f6da..85a7398c35 100644 --- a/ppg/pg_source_tarballs.groovy +++ b/ppg/pg_source_tarballs.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -20,7 +20,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { def packageVersion = sh(script: """ # Download the configure file - if [[ "${BRANCH_NAME}" == *TDE* ]]; then + if echo "${BRANCH_NAME}" | grep -q "release-"; then wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} else wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} @@ -41,7 +41,7 @@ void uploadTarballToTestingDownloadServer(String tarballDirectory, String packag script { try { - uploadPGTarballToDownloadsTesting(tarballDirectory, packageVersion) + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) } catch (err) { echo "Caught: ${err}" currentBuild.result = 'UNSTABLE' @@ -59,10 +59,14 @@ def TIMESTAMP pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tarballs repository', @@ -89,7 +93,7 @@ pipeline { stage('Create timestamp') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -110,13 +114,13 @@ pipeline { parallel { stage('Build source tarball for PG 17') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() script { def PG_VERSION=17 - def BRANCH_NAME = "TDE_REL_17_STABLE" + def BRANCH_NAME = "release-17.6.1" def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" def PRODUCT="Percona-PostgreSQL-Source-Tarballs" @@ -128,15 +132,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 16') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -154,15 +158,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 15') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -180,15 +184,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 14') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -206,15 +210,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build source tarball for PG 13') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -232,8 +236,8 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } diff --git a/ppg/pg_source_tarballs.yml b/ppg/pg_source_tarballs.yml index 62c71c2e9f..738a316524 100644 --- a/ppg/pg_source_tarballs.yml +++ b/ppg/pg_source_tarballs.yml @@ -1,5 +1,5 @@ - job: - name: pg_source_tarballs-RELEASE + name: hetzner-pg_source_tarballs-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_source_tarballs.groovy diff --git a/ppg/pg_stat_monitor-autobuild-arm.groovy b/ppg/pg_stat_monitor-autobuild-arm.groovy new file mode 100644 index 0000000000..085c6736c7 --- /dev/null +++ b/ppg/pg_stat_monitor-autobuild-arm.groovy @@ -0,0 +1,254 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: '2.0.0', + description: 'General version of the product', + name: 'VERSION' + ) + string( + defaultValue: 'https://github.com/percona/pg_stat_monitor.git', + description: 'pg_stat_monitor repo', + name: 'GIT_REPO' + ) + string( + defaultValue: 'main', + description: 'Branch for tests', + name: 'BRANCH' + ) + string( + defaultValue: '1', + description: 'rpm release number', + name: 'RPM_RELEASE' + ) + string( + defaultValue: '1', + description: 'deb release number', + name: 'DEB_RELEASE' + ) + choice( + name: 'PG_RELEASE', + description: 'PPG major version to test', + choices: ['11', '12', '13', '14', '15', '16', '17'] + ) + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO' + ) + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Download source from github') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-stat-monitor.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg-stat-monitor.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } //stage + stage('Build pg_stat_monitor generic source packages') { + parallel { + stage('Source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } //stage + stage('Build pg_stat_monitor RPMs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } //parallel + } //stage + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } //stages + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Success" + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + deleteDir() + echo "Failure" + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + echo "Always" + } + } +} diff --git a/ppg/pg_stat_monitor-autobuild-arm.yml b/ppg/pg_stat_monitor-autobuild-arm.yml new file mode 100644 index 0000000000..a0ef8e1bd5 --- /dev/null +++ b/ppg/pg_stat_monitor-autobuild-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pg_stat_monitor-autobuild-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_stat_monitor-autobuild-arm.groovy diff --git a/ppg/pg_stat_monitor-autobuild.groovy b/ppg/pg_stat_monitor-autobuild.groovy index c64f09bce7..3ffdff9d98 100644 --- a/ppg/pg_stat_monitor-autobuild.groovy +++ b/ppg/pg_stat_monitor-autobuild.groovy @@ -1,40 +1,22 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) - -void installCli(String PLATFORM) { - sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip gnupg2 - elif [ ${PLATFORM} = "rpm" ]; then - sudo yum -y install wget curl unzip gnupg2 - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true - """ -} - void buildStage(String DOCKER_OS, String STAGE_PARAM) { sh """ - echo "Docker: $DOCKER_OS, Release: PG$PG_RELEASE, Stage: $STAGE_PARAM" set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -O psm_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/main/percona-packaging/scripts/pg_stat_monitor_builder.sh -o psm_builder.sh + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/release-2.2.0/percona-packaging/scripts/pg_stat_monitor_builder.sh -O builder.sh pwd -P + ls -laR export build_dir=\$(pwd -P) - set -o xtrace - cd \${build_dir} - sudo bash -x ./psm_builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 - bash -x ./psm_builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} "$STAGE_PARAM" - """ + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --version=\${VERSION} --branch=\${BRANCH} --repo=\${GIT_REPO} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --pg_release=\${PG_RELEASE} --ppg_repo_name=\${PPG_REPO} ${STAGE_PARAM}" + """ } void cleanUpWS() { @@ -47,11 +29,15 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( - defaultValue: '1.0.0', + defaultValue: '2.0.0', description: 'General version of the product', name: 'VERSION' ) @@ -81,9 +67,10 @@ pipeline { choices: ['11', '12', '13', '14', '15', '16', '17'] ) string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-16.1', description: 'PPG repo name', - name: 'PPG_REPO') + name: 'PPG_REPO' + ) choice( choices: 'laboratory\ntesting\nexperimental\nrelease', description: 'Repo component to push packages to', @@ -98,204 +85,286 @@ pipeline { stages { stage('Download source from github') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo '====> Source will be downloaded from github' - slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") cleanUpWS() - installCli("deb") - buildStage("ubuntu:focal", "--get_sources=1") - sh ''' + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg-stat-monitor.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") echo ${REPO_UPLOAD_PATH} > uploadPath echo ${AWS_STASH_PATH} > awsUploadPath cat test/pg-stat-monitor.properties cat uploadPath - cat awsUploadPath - ''' + ''' script { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - stash includes: 'test/pg-stat-monitor.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } //stage stage('Build pg_stat_monitor generic source packages') { parallel { stage('Source rpm') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor generic source rpm" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } - stage('Source deb') { + stage('Source deb') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor generic source deb" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_source_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } - } //stage + } } //parallel } //stage stage('Build pg_stat_monitor RPMs') { parallel { - stage('OL 8') { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor rpm on OL 8 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - stage('OL 9') { + } + stage('Oracle Linux 9 AMD') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor rpm on OL 9 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - } //parallel - } //stage - stage('Build pg_stat_monitor DEBs') { - parallel { - stage('Ubuntu 20.04') { + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor deb on Ubuntu 20.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 22.04') { + } + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor deb on Ubuntu 22.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } - } //stage - stage('Ubuntu 24.04') { + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor deb on Ubuntu 24.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 11') { + } + stage('Debian Bullseye(11) AMD') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pg_stat_monitor deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 12') { + } + stage('Debian bookworm(12) ARM') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pg_stat_monitor deb on Debian 12 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } - } //stage + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } } //parallel } //stage stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } } //stages post { success { - slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") deleteDir() echo "Success" } diff --git a/ppg/pg_stat_monitor-autobuild.yml b/ppg/pg_stat_monitor-autobuild.yml index 9539125fd9..94b70bbcdc 100644 --- a/ppg/pg_stat_monitor-autobuild.yml +++ b/ppg/pg_stat_monitor-autobuild.yml @@ -1,5 +1,5 @@ - job: - name: pg_stat_monitor-autobuild-RELEASE + name: hetzner-pg_stat_monitor-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_stat_monitor-autobuild.groovy diff --git a/ppg/pg_tarballs-arm.groovy b/ppg/pg_tarballs-arm.groovy new file mode 100644 index 0000000000..e8971c8841 --- /dev/null +++ b/ppg/pg_tarballs-arm.groovy @@ -0,0 +1,532 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tarballs/pg_tarballs_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --build_dependencies=${BUILD_DEPENDENCIES} ${STAGE_PARAM}" + """ +} + +String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { + def packageVersion = sh(script: """ + # Download the configure file + if echo "${BRANCH_NAME}" | grep -q "release-"; then + wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + else + wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + fi + # Read the PACKAGE_VERSION value from the configure file + packageVersion=\$(grep -r 'PACKAGE_VERSION=' ${configureFileName} | tr -dc '[. [:digit:]]') + + # Delete configure file + rm -f ${configureFileName} + + echo "\$packageVersion" + """, returnStdout: true).trim() + + return packageVersion +} + +void uploadTarballToTestingDownloadServer(String tarballDirectory, String packageVersion) { + + script { + try { + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tarballs repository', + name: 'GIT_REPO') +/* string( + defaultValue: '16.1', + description: 'Version of PostgreSQL server', + name: 'PG_VERSION')*/ + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tarballs packaging repository', + name: 'GIT_BRANCH') + choice( + choices: '1\n0', + description: 'Build third party dependencies', + name: 'BUILD_DEPENDENCIES') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo destination to push packages to', + name: 'DESTINATION') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Build pg_tarballs') { + parallel { + stage('Build pg_tarball 17 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.6.1" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.6.1" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = 'release-17.6.1' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 16 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_16_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 16 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=16 + def BRANCH_NAME = "REL_16_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_16="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_16} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 16 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=16 + def BRANCH_NAME = 'REL_16_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_16="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_16} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 15 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_15_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 15 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=15 + def BRANCH_NAME = 'REL_15_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_15="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_15} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 15 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=15 + def BRANCH_NAME = 'REL_15_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_15="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_15} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 14 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_14_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 14 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=14 + def BRANCH_NAME = 'REL_14_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_14="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_14} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 14 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=14 + def BRANCH_NAME = 'REL_14_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_14="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_14} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 13 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_13_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 13 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=13 + def BRANCH_NAME = 'REL_13_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_13="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_13} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 13 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=13 + def BRANCH_NAME = 'REL_13_STABLE' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_13="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_13} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + } //parallel + } // stage + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tarballs-arm.yml b/ppg/pg_tarballs-arm.yml new file mode 100644 index 0000000000..e780ee3959 --- /dev/null +++ b/ppg/pg_tarballs-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_tarballs-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tarballs-arm.groovy + diff --git a/ppg/pg_tarballs.groovy b/ppg/pg_tarballs.groovy index f4956712d4..4c8fa1f795 100644 --- a/ppg/pg_tarballs.groovy +++ b/ppg/pg_tarballs.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -20,7 +20,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { def packageVersion = sh(script: """ # Download the configure file - if [[ "${BRANCH_NAME}" == *TDE* ]]; then + if echo "${BRANCH_NAME}" | grep -q "release-"; then wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} else wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} @@ -41,7 +41,7 @@ void uploadTarballToTestingDownloadServer(String tarballDirectory, String packag script { try { - uploadPGTarballToDownloadsTesting(tarballDirectory, packageVersion) + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) } catch (err) { echo "Caught: ${err}" currentBuild.result = 'UNSTABLE' @@ -59,10 +59,14 @@ def TIMESTAMP pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tarballs repository', @@ -93,7 +97,7 @@ pipeline { stage('Create timestamp') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -112,15 +116,41 @@ pipeline { stage('Build pg_tarballs') { parallel { + stage('Build pg_tarball 17 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.6.1" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } stage('Build pg_tarball 17 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() script { def PG_VERSION=17 - def BRANCH_NAME = "TDE_REL_17_STABLE" + def BRANCH_NAME = "release-17.6.1" def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" def PRODUCT="Percona-PostgreSQL-Tarballs" @@ -132,21 +162,21 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 17 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() script { def PG_VERSION=17 - def BRANCH_NAME = 'TDE_REL_17_STABLE' + def BRANCH_NAME = 'release-17.6.1' def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" def PRODUCT="Percona-PostgreSQL-Tarballs" @@ -158,15 +188,41 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 16 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_16_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 16 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -184,15 +240,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 16 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -210,15 +266,41 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_16, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } + stage('Build pg_tarball 15 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_15_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } stage('Build pg_tarball 15 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -236,15 +318,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 15 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -262,15 +344,41 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_15, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } + stage('Build pg_tarball 14 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_14_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } stage('Build pg_tarball 14 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -288,15 +396,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 14 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -314,15 +422,41 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_14, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } + stage('Build pg_tarball 13 for OpenSSL 3.5') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "REL_13_STABLE" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl35") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_ssl35=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } stage('Build pg_tarball 13 for OpenSSL 3') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -340,15 +474,15 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } } stage('Build pg_tarball 13 for OpenSSL 1.1') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -366,8 +500,8 @@ pipeline { """ stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") - pushArtifactFolder("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) - uploadPGTarballfromAWS("tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_13, "binary", "${PACKAGE_VERSION}") uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") } } @@ -375,7 +509,7 @@ pipeline { } //parallel } // stage - } + } post { success { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") diff --git a/ppg/pg_tarballs.yml b/ppg/pg_tarballs.yml index fe37a9b4da..d51295963b 100644 --- a/ppg/pg_tarballs.yml +++ b/ppg/pg_tarballs.yml @@ -1,5 +1,5 @@ - job: - name: pg_tarballs-RELEASE + name: hetzner-pg_tarballs-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_tarballs.groovy diff --git a/ppg/pg_tarballs_17_nightly.groovy b/ppg/pg_tarballs_17_nightly.groovy new file mode 100644 index 0000000000..4fc96b60da --- /dev/null +++ b/ppg/pg_tarballs_17_nightly.groovy @@ -0,0 +1,246 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tarballs/pg_tarballs_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --build_dependencies=${BUILD_DEPENDENCIES} ${STAGE_PARAM}" + """ +} + +String getPostgreSQLVersion(String BRANCH_NAME, String configureFileName) { + def packageVersion = sh(script: """ + # Download the configure file + if echo "${BRANCH_NAME}" | grep -q "release-"; then + wget https://raw.githubusercontent.com/Percona-Lab/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + else + wget https://raw.githubusercontent.com/postgres/postgres/${BRANCH_NAME}/configure -O ${configureFileName} + fi + # Read the PACKAGE_VERSION value from the configure file + packageVersion=\$(grep -r 'PACKAGE_VERSION=' ${configureFileName} | tr -dc '[. [:digit:]]') + + # Delete configure file + rm -f ${configureFileName} + + echo "\$packageVersion" + """, returnStdout: true).trim() + + return packageVersion +} + +void uploadTarballToTestingDownloadServer(String tarballDirectory, String packageVersion) { + + script { + try { + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tarballs repository', + name: 'GIT_REPO') +/* string( + defaultValue: '16.1', + description: 'Version of PostgreSQL server', + name: 'PG_VERSION')*/ + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tarballs packaging repository', + name: 'GIT_BRANCH') + choice( + choices: '1\n0', + description: 'Build third party dependencies', + name: 'BUILD_DEPENDENCIES') + choice( + choices: 'testing\nlaboratory\nexperimental', + description: 'Repo destination to push packages to', + name: 'DESTINATION') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Build pg_tarballs') { + parallel { + stage('Build pg_tarball 17 for OpenSSL 3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.5.3" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 1.1') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = 'release-17.5.3' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 3 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = "release-17.5.3" + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl3") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION}") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + stage('Build pg_tarball 17 for OpenSSL 1.1 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + def PG_VERSION=17 + def BRANCH_NAME = 'release-17.5.3' + def PACKAGE_VERSION = getPostgreSQLVersion(BRANCH_NAME, "configure.${PG_VERSION}.ssl1.1") + println "Returned PACKAGE_VERSION: ${PACKAGE_VERSION}" + def PRODUCT="Percona-PostgreSQL-Tarballs" + unstash 'timestamp' + AWS_STASH_PATH_17="/srv/UPLOAD/${DESTINATION}/BUILDS/${PRODUCT}/${PRODUCT}-${PACKAGE_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH_17} > uploadPath-${PACKAGE_VERSION} + cat uploadPath-${PACKAGE_VERSION} + """ + stash includes: "uploadPath-${PACKAGE_VERSION}", name: "uploadPath-${PACKAGE_VERSION}" + buildStage("oraclelinux:8", "--version=${PACKAGE_VERSION} --use_system_ssl=1") + pushArtifactFolder(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17) + uploadPGTarballfromAWS(params.CLOUD, "tarballs-${PACKAGE_VERSION}/", AWS_STASH_PATH_17, "binary", "${PACKAGE_VERSION}") + uploadTarballToTestingDownloadServer("pg_tarballs", "${PACKAGE_VERSION}") + } + } + } + } //parallel + } // stage + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tarballs_17_nightly.yml b/ppg/pg_tarballs_17_nightly.yml new file mode 100644 index 0000000000..5c4d328cf5 --- /dev/null +++ b/ppg/pg_tarballs_17_nightly.yml @@ -0,0 +1,17 @@ +- job: + name: hetzner-pg_tarballs-nightly + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tarballs_17_nightly.groovy + triggers: + - timed: "0 0 * * *" + diff --git a/ppg/pg_tde-arm.groovy b/ppg/pg_tde-arm.groovy new file mode 100644 index 0000000000..5dc10ff052 --- /dev/null +++ b/ppg/pg_tde-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tde/pg_tde_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tde repository', + name: 'GIT_REPO') + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tde repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.2', + description: 'Tag/Branch for pg_tde packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.2', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_tde source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_tde.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_tde.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_tde generic source packages') { + parallel { + stage('Build pg_tde generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_tde generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_tde RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tde-arm.yml b/ppg/pg_tde-arm.yml new file mode 100644 index 0000000000..0a2e655e1c --- /dev/null +++ b/ppg/pg_tde-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pg_tde-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tde-arm.groovy + diff --git a/ppg/pg_tde.groovy b/ppg/pg_tde.groovy index 1d6767c32d..ac1d67ed55 100644 --- a/ppg/pg_tde.groovy +++ b/ppg/pg_tde.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tde repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_tde generic source packages') { parallel { stage('Build pg_tde generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_tde generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,80 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - } - } - stage('Ubuntu Focal(20.04)') { - agent { - label 'docker' - } - steps { - cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +207,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_tde.yml b/ppg/pg_tde.yml index b6f72a92c9..efb9debf24 100644 --- a/ppg/pg_tde.yml +++ b/ppg/pg_tde.yml @@ -1,5 +1,5 @@ - job: - name: pg_tde-RELEASE + name: hetzner-pg_tde-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_tde.groovy diff --git a/ppg/pg_tde_arm_nightly.groovy b/ppg/pg_tde_arm_nightly.groovy new file mode 100644 index 0000000000..72f228cc8f --- /dev/null +++ b/ppg/pg_tde_arm_nightly.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_tde/pg_tde_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --nightly=1 ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_tde repository', + name: 'GIT_REPO') + string( + defaultValue: 'main', + description: 'Tag/Branch for pg_tde repository', + name: 'PG_BRANCH') + string( + defaultValue: '17.5.2', + description: 'Tag/Branch for pg_tde packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '2', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '2', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-17.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'experimental\nlaboratory\ntesting', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pg_tde source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_tde.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_tde.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pg_tde generic source packages') { + parallel { + stage('Build pg_tde generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pg_tde generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pg_tde RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pg_tde_arm_nightly.yml b/ppg/pg_tde_arm_nightly.yml new file mode 100644 index 0000000000..5ccb8eae5c --- /dev/null +++ b/ppg/pg_tde_arm_nightly.yml @@ -0,0 +1,16 @@ +- job: + name: hetzner-pg_tde_arm-nightly + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pg_tde_arm_nightly.groovy + triggers: + - timed: "0 0 * * *" diff --git a/ppg/pg_tde_nightly.groovy b/ppg/pg_tde_nightly.groovy index b4033b3627..60a90286a0 100644 --- a/ppg/pg_tde_nightly.groovy +++ b/ppg/pg_tde_nightly.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for pg_tde repository', @@ -41,19 +45,19 @@ pipeline { description: 'Tag/Branch for pg_tde repository', name: 'PG_BRANCH') string( - defaultValue: '17.0', + defaultValue: '17.5.2', description: 'Tag/Branch for pg_tde packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', + defaultValue: '2', description: 'RPM release value', name: 'RPM_RELEASE') string( - defaultValue: '1', + defaultValue: '2', description: 'DEB release value', name: 'DEB_RELEASE') string( - defaultValue: 'ppg-17.0', + defaultValue: 'ppg-17.5', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pg_tde generic source packages') { parallel { stage('Build pg_tde generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pg_tde generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,80 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - } - } - stage('Ubuntu Focal(20.04)') { - agent { - label 'docker' - } - steps { - cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +207,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pg_tde_nightly.yml b/ppg/pg_tde_nightly.yml index ec8649cef9..09808bfeec 100644 --- a/ppg/pg_tde_nightly.yml +++ b/ppg/pg_tde_nightly.yml @@ -1,5 +1,5 @@ - job: - name: pg_tde-nightly + name: hetzner-pg_tde-nightly project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pg_tde_nightly.groovy diff --git a/ppg/pgaudit-arm.groovy b/ppg/pgaudit-arm.groovy new file mode 100644 index 0000000000..b2bdb346d6 --- /dev/null +++ b/ppg/pgaudit-arm.groovy @@ -0,0 +1,240 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgaudit/pgaudit_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_audit repository', + name: 'GIT_REPO') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_audit repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_audit packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_AUDIT source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgaudit.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgaudit.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_AUDIT generic source packages') { + parallel { + stage('Build PG_AUDIT generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_AUDIT generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgaudit-arm.yml b/ppg/pgaudit-arm.yml new file mode 100644 index 0000000000..fd157e864d --- /dev/null +++ b/ppg/pgaudit-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pgaudit-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgaudit-arm.groovy + diff --git a/ppg/pgaudit.groovy b/ppg/pgaudit.groovy index fb8c84fede..52bc8c16ec 100644 --- a/ppg/pgaudit.groovy +++ b/ppg/pgaudit.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgaudit/pgaudit_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_audit repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for pg_audit repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', + defaultValue: '17.6', description: 'Tag/Branch for pg_audit packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_AUDIT generic source packages') { parallel { stage('Build PG_AUDIT generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_AUDIT generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgaudit.yml b/ppg/pgaudit.yml index aef6bd3500..a2592e23f7 100644 --- a/ppg/pgaudit.yml +++ b/ppg/pgaudit.yml @@ -1,5 +1,5 @@ - job: - name: pgaudit-RELEASE + name: hetzner-pgaudit-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgaudit.groovy diff --git a/ppg/pgaudit_set_user-arm.groovy b/ppg/pgaudit_set_user-arm.groovy new file mode 100644 index 0000000000..207c5d6dbc --- /dev/null +++ b/ppg/pgaudit_set_user-arm.groovy @@ -0,0 +1,240 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgaudit_set_user/pgaudit_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_audit repository', + name: 'GIT_REPO') + string( + defaultValue: '4.0.1', + description: 'Tag/Branch for pg_audit-set-user', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for pg_audit-set-user packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_AUDIT source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgaudit.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgaudit.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_AUDIT generic source packages') { + parallel { + stage('Build PG_AUDIT generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_AUDIT generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgaudit_set_user-arm.yml b/ppg/pgaudit_set_user-arm.yml new file mode 100644 index 0000000000..b11b4adf86 --- /dev/null +++ b/ppg/pgaudit_set_user-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgaudit_set_user-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgaudit_set_user-arm.groovy \ No newline at end of file diff --git a/ppg/pgaudit_set_user.groovy b/ppg/pgaudit_set_user.groovy index 5eb8855c0e..fcdd53e064 100644 --- a/ppg/pgaudit_set_user.groovy +++ b/ppg/pgaudit_set_user.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgaudit_set_user/pgaudit_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_audit repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', - description: 'Tag/Branch for pg_audit repository', + defaultValue: '17.6', + description: 'Tag/Branch for pg_audit packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_AUDIT generic source packages') { parallel { stage('Build PG_AUDIT generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_AUDIT generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_AUDIT RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgaudit_set_user.yml b/ppg/pgaudit_set_user.yml index 78d1efa15f..9c23675724 100644 --- a/ppg/pgaudit_set_user.yml +++ b/ppg/pgaudit_set_user.yml @@ -1,5 +1,5 @@ - job: - name: pgaudit_set_user-RELEASE + name: hetzner-pgaudit_set_user-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgaudit_set_user.groovy \ No newline at end of file diff --git a/ppg/pgbackrest-arm.groovy b/ppg/pgbackrest-arm.groovy new file mode 100644 index 0000000000..fb3b425f4d --- /dev/null +++ b/ppg/pgbackrest-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbackrest/pgbackrest_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_BACKREST repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_BACKREST repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_BACKREST source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgbackrest.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgbackrest.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_BACKREST generic source packages') { + parallel { + stage('Build PG_BACKREST generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_BACKREST generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_BACKREST RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgbackrest-arm.yml b/ppg/pgbackrest-arm.yml new file mode 100644 index 0000000000..af2bf133e9 --- /dev/null +++ b/ppg/pgbackrest-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgbackrest-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgbackrest-arm.groovy diff --git a/ppg/pgbackrest.groovy b/ppg/pgbackrest.groovy index f82f4c24ca..a02ddcc1a7 100644 --- a/ppg/pgbackrest.groovy +++ b/ppg/pgbackrest.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbackrest/pgbackrest_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_BACKREST repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', - description: 'Tag/Branch for pg_BACKREST repository', + defaultValue: '17.6', + description: 'Tag/Branch for pg_BACKREST packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_BACKREST generic source packages') { parallel { stage('Build PG_BACKREST generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_BACKREST generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_BACKREST RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgbackrest.yml b/ppg/pgbackrest.yml index 8178ce4242..5e3208db0d 100644 --- a/ppg/pgbackrest.yml +++ b/ppg/pgbackrest.yml @@ -1,5 +1,5 @@ - job: - name: pgbackrest-RELEASE + name: hetzner-pgbackrest-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgbackrest.groovy \ No newline at end of file diff --git a/ppg/pgbadger-arm.groovy b/ppg/pgbadger-arm.groovy new file mode 100644 index 0000000000..19237c362b --- /dev/null +++ b/ppg/pgbadger-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbadger/pgbadger_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_badger repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_badger repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_badger source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgbadger.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgbadger.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_badger generic source packages') { + parallel { + stage('Build PG_badger generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_badger generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_badger RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgbadger-arm.yml b/ppg/pgbadger-arm.yml new file mode 100644 index 0000000000..22773e6dd0 --- /dev/null +++ b/ppg/pgbadger-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgbadger-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgbadger-arm.groovy diff --git a/ppg/pgbadger.groovy b/ppg/pgbadger.groovy index 0556a563c4..d681262a19 100644 --- a/ppg/pgbadger.groovy +++ b/ppg/pgbadger.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbadger/pgbadger_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_badger repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', - description: 'Tag/Branch for pg_badger repository', + defaultValue: '17.6', + description: 'Tag/Branch for pg_badger packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_badger generic source packages') { parallel { stage('Build PG_badger generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_badger generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_badger RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgbadger.yml b/ppg/pgbadger.yml index 15816a7e91..3a50b2916f 100644 --- a/ppg/pgbadger.yml +++ b/ppg/pgbadger.yml @@ -1,5 +1,5 @@ - job: - name: pgbadger-RELEASE + name: hetzner-pgbadger-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgbadger.groovy \ No newline at end of file diff --git a/ppg/pgbouncer-arm.groovy b/ppg/pgbouncer-arm.groovy new file mode 100644 index 0000000000..ad8c8f9467 --- /dev/null +++ b/ppg/pgbouncer-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbouncer/pgbouncer_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_bouncer repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_bouncer repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_bouncer source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgbouncer.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgbouncer.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_bouncer generic source packages') { + parallel { + stage('Build PG_bouncer generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_bouncer generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_bouncer RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgbouncer-arm.yml b/ppg/pgbouncer-arm.yml new file mode 100644 index 0000000000..4fa82e885b --- /dev/null +++ b/ppg/pgbouncer-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgbouncer-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgbouncer-arm.groovy diff --git a/ppg/pgbouncer.groovy b/ppg/pgbouncer.groovy index 544dfda8fb..571aff8520 100644 --- a/ppg/pgbouncer.groovy +++ b/ppg/pgbouncer.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgbouncer/pgbouncer_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_bouncer repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', - description: 'Tag/Branch for pg_bouncer repository', + defaultValue: '17.6', + description: 'Tag/Branch for pg_bouncer packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_bouncer generic source packages') { parallel { stage('Build PG_bouncer generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_bouncer generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_bouncer RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgbouncer.yml b/ppg/pgbouncer.yml index 1fc522eb90..06d128a77b 100644 --- a/ppg/pgbouncer.yml +++ b/ppg/pgbouncer.yml @@ -1,5 +1,5 @@ - job: - name: pgbouncer-RELEASE + name: hetzner-pgbouncer-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgbouncer.groovy \ No newline at end of file diff --git a/ppg/pgpool-arm.groovy b/ppg/pgpool-arm.groovy new file mode 100644 index 0000000000..a0c321cd37 --- /dev/null +++ b/ppg/pgpool-arm.groovy @@ -0,0 +1,256 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -O pgpool2_builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --branch=\${BRANCH} --repo=\${GIT_REPO} --pp_branch=\${BUILD_BRANCH} --pp_repo=\${GIT_BUILD_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} $STAGE_PARAM" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: '4.4.2', + description: 'General version of the product', + name: 'VERSION' + ) + string( + defaultValue: 'https://git.postgresql.org/git/pgpool2.git', + description: 'pgpool2 repo', + name: 'GIT_REPO' + ) + string( + defaultValue: 'V4_4_STABLE', + description: 'Branch for pgpool2 repo', + name: 'BRANCH' + ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'Build pgpool2 repo', + name: 'GIT_BUILD_REPO' + ) + string( + defaultValue: 'main', + description: 'Branch for build repo', + name: 'BUILD_BRANCH' + ) + string( + defaultValue: '1', + description: 'rpm release number', + name: 'RPM_RELEASE' + ) + string( + defaultValue: '1', + description: 'deb release number', + name: 'DEB_RELEASE' + ) + string( + defaultValue: '16.1', + description: 'PPG major version to test', + name: 'PG_RELEASE' + ) + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pgpool2 source tarball') { + steps { + slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${BUILD_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgpool2.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgpool2.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pgpool2 generic source packages') { + parallel { + stage('Build pgpool2 generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pgpool2 generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pgpool2 RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, "ppg-${PG_RELEASE}", COMPONENT) + } + } + + } + post { + success { + slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BUILD_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${BUILD_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${BUILD_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgpool-arm.yml b/ppg/pgpool-arm.yml new file mode 100644 index 0000000000..33d386d847 --- /dev/null +++ b/ppg/pgpool-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgpool-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgpool-arm.groovy diff --git a/ppg/pgpool2-build-RELEASE.yml b/ppg/pgpool2-build-RELEASE.yml index ba54061b7e..a562beb9eb 100644 --- a/ppg/pgpool2-build-RELEASE.yml +++ b/ppg/pgpool2-build-RELEASE.yml @@ -1,5 +1,5 @@ - job: - name: pgpool2-build-RELEASE + name: hetzner-pgpool2-build-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgpool2-build.groovy diff --git a/ppg/pgpool2-build.groovy b/ppg/pgpool2-build.groovy index 2ec22a575d..10ebd36ee6 100644 --- a/ppg/pgpool2-build.groovy +++ b/ppg/pgpool2-build.groovy @@ -1,76 +1,24 @@ -/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' -]) - -void installCli(String PLATFORM) { - sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip gnupg2 - elif [ ${PLATFORM} = "rpm" ]; then - sudo yum -y install wget curl unzip gnupg2 - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true - """ -} +]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { sh """ - echo "Docker: $DOCKER_OS, Release: PG$PG_RELEASE, Stage: $STAGE_PARAM" set -o xtrace - mkdir -p test - wget \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -O pg2_builder.sh || curl \$(echo ${GIT_BUILD_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BUILD_BRANCH}/pgpool2/pgpool2_builder.sh -o pg2_builder.sh - #git clone ${GIT_BUILD_REPO} - #cd postgres-packaging - #git checkout ${BUILD_BRANCH} - #cd .. - #cp postgres-packaging/pgpool2/pgpool2_builder.sh pg2_builder.sh - if [ -f /etc/redhat-release ]; then - sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true - else - DEBIAN_VERSION=\$(lsb_release -sc) - # sed -e '/.*backports.*/d' /etc/apt/sources.list > sources.list.new - # sudo mv -vf sources.list.new /etc/apt/sources.list - if [ \${DEBIAN_VERSION} = bionic ]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" | sudo tee -a /etc/apt/sources.list - fi - if [ \${DEBIAN_VERSION} = buster ]; then - sudo apt-get -y update --allow-releaseinfo-change || true - fi - until sudo apt-get update; do - sleep 30 - echo "Waiting ..." - done - until sudo apt-get -y install gpgv curl clang gnupg; do - sleep 30 - echo "Waiting ..." - done - sudo wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb - sudo dpkg -i percona-release_latest.generic_all.deb - fi - sudo percona-release enable ppg-${PG_RELEASE} release + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgpool2/pgpool2_builder.sh -O pgpool2_builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P export build_dir=\$(pwd -P) - set -o xtrace - cd \${build_dir} - if [ -f ./test/pgpool2.properties ]; then - . ./test/pgpool2.properties - fi - sed -i "s:VERSION=\\"1.0.0:VERSION=\\"$VERSION:" pg2_builder.sh - sed -i "s:PG_RELEASE=11:PG_RELEASE=\"${PG_RELEASE}\":" pg2_builder.sh - - sudo bash -x ./pg2_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./pg2_builder.sh --builddir=\${build_dir}/test --branch=\${BRANCH} --repo=\${GIT_REPO} --pp_branch=\${BUILD_BRANCH} --pp_repo=\${GIT_BUILD_REPO} --rpm_release=\${RPM_RELEASE} --deb_release=\${DEB_RELEASE} --pg_release=\${PG_RELEASE} "$STAGE_PARAM" - """ + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./pgpool2_builder.sh --builddir=\${build_dir}/test $STAGE_PARAM" + """ } void cleanUpWS() { @@ -83,275 +31,334 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { - string( - defaultValue: '4.4.2', - description: 'General version of the product', - name: 'VERSION' - ) - string( - defaultValue: 'https://git.postgresql.org/git/pgpool2.git', - description: 'pgpool2 repo', - name: 'GIT_REPO' - ) - string( - defaultValue: 'V4_4_STABLE', - description: 'Branch for pgpool2 repo', - name: 'BRANCH' - ) + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'Build pgpool2 repo', - name: 'GIT_BUILD_REPO' - ) - string( - defaultValue: 'main', - description: 'Branch for build repo', - name: 'BUILD_BRANCH' - ) - string( - defaultValue: '1', - description: 'rpm release number', - name: 'RPM_RELEASE' + description: 'URL for packaging repository', + name: 'GIT_REPO' ) string( - defaultValue: '1', - description: 'deb release number', - name: 'DEB_RELEASE' + defaultValue: '17.6', + description: 'Tag/Branch for pgpool2 packaging repository', + name: 'GIT_BRANCH' ) string( - defaultValue: '16.1', - description: 'PPG major version to test', - name: 'PG_RELEASE' + defaultValue: '17.6', + description: 'PPG repo name', + name: 'PPG_REPO' ) choice( choices: 'laboratory\ntesting\nexperimental\nrelease', description: 'Repo component to push packages to', name: 'COMPONENT') - } + } options { skipDefaultCheckout() disableConcurrentBuilds() buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) - timestamps () } stages { - stage('Download source from github') { - agent { - label 'min-focal-x64' - } + stage('Create pgpool2 source tarball') { steps { - echo '====> Source will be downloaded from github' - // slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") + slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - installCli("deb") - buildStage("ubuntu:focal", "--get_sources=1") - sh ''' + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgpool2.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") echo ${REPO_UPLOAD_PATH} > uploadPath echo ${AWS_STASH_PATH} > awsUploadPath cat test/pgpool2.properties cat uploadPath - cat awsUploadPath - ''' + ''' script { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - stash includes: 'test/pgpool2.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } - } //stage + } stage('Build pgpool2 generic source packages') { parallel { - stage('Source rpm') { + stage('Build pgpool2 generic source rpm') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 generic source rpm" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } - stage('Source deb') { + stage('Build pgpool2 generic source deb') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 generic source deb" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_source_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } - } //stage + } } //parallel - } //stage - stage('Build pgpool2 RPMs') { + } // stage + stage('Build pgpool2 RPMs/DEBs/Binary tarballs') { parallel { - stage('OL 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 rpm on Centos 8 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - stage('OL 9') { + } + stage('Oracle Linux 8 ARM') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 rpm on OL9 PG${PG_RELEASE}" cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - } //stage - } //parallel - } //stage - stage('Build pgpool2 DEBs') { - parallel { - stage('Ubuntu 20.04') { + } + stage('Oracle Linux 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 deb on Ubuntu 20.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 22.04') { + } + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Ubuntu 22.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Ubuntu 24.04') { + } + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Ubuntu 24.04 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 11') { + } + stage('Debian Bullseye(11) AMD') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { - echo "====> Build pgpool2 deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } - } //stage - stage('Debian 12') { + } + stage('Debian Bullseye(11) ARM') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { - echo "====> Build pgpool2 deb on Debian 11 PG${PG_RELEASE}" cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } - } //stage - } //parallel - } //stage + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild("ppg-${PG_RELEASE}", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ppg-${PPG_REPO}", COMPONENT) } } - } //stages + + } post { success { -// slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") - deleteDir() - echo "Success" + slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() } failure { -// slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for PG${PG_RELEASE}, repo branch: ${BRANCH} - [${BUILD_URL}]") - deleteDir() - echo "Failure" + slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() } always { sh ''' sudo rm -rf ./* ''' deleteDir() - echo "Always" } } } diff --git a/ppg/pgrepack.groovy b/ppg/pgrepack.groovy index cd24c62427..1bfcbab42d 100644 --- a/ppg/pgrepack.groovy +++ b/ppg/pgrepack.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_repack/pgrepack_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pg_repack repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', - description: 'Tag/Branch for pg_repack repository', + defaultValue: '17.6', + description: 'Tag/Branch for pg_repack packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PG_REPACK generic source packages') { parallel { stage('Build PG_REPACK generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PG_REPACK generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PG_REPACK RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgrepack.yml b/ppg/pgrepack.yml index 02c9d26d7e..e7b0c3a7c2 100644 --- a/ppg/pgrepack.yml +++ b/ppg/pgrepack.yml @@ -1,5 +1,5 @@ - job: - name: pgrepack-RELEASE + name: hetzner-pgrepack-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgrepack.groovy diff --git a/ppg/pgrepack_arm.groovy b/ppg/pgrepack_arm.groovy new file mode 100644 index 0000000000..36ac0d7a4a --- /dev/null +++ b/ppg/pgrepack_arm.groovy @@ -0,0 +1,240 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pg_repack/pgrepack_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pg_repack repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'General version of the product', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pg_repack repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PG_REPACK source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pg_repack.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pg_repack.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PG_REPACK generic source packages') { + parallel { + stage('Build PG_REPACK generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PG_REPACK generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PG_REPACK RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgrepack_arm.yml b/ppg/pgrepack_arm.yml new file mode 100644 index 0000000000..cad913b5f3 --- /dev/null +++ b/ppg/pgrepack_arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pgrepack-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgrepack_arm.groovy \ No newline at end of file diff --git a/ppg/pgvector-arm.groovy b/ppg/pgvector-arm.groovy new file mode 100644 index 0000000000..d305c9e939 --- /dev/null +++ b/ppg/pgvector-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgvector/pgvector_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pgvector repository', + name: 'GIT_REPO') + string( + defaultValue: 'ver_1.4.4', + description: 'Tag/Branch for pgvector repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.0', + description: 'Tag/Branch for pgvector packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.0', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create pgvector source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pgvector.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pgvector.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build pgvector generic source packages') { + parallel { + stage('Build pgvector generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build pgvector generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build pgvector RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pgvector-arm.yml b/ppg/pgvector-arm.yml new file mode 100644 index 0000000000..8de028f67f --- /dev/null +++ b/ppg/pgvector-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pgvector-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pgvector-arm.groovy + diff --git a/ppg/pgvector.groovy b/ppg/pgvector.groovy index 1e8251ecd2..c70e301b18 100644 --- a/ppg/pgvector.groovy +++ b/ppg/pgvector.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pgvector/pgvector_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pgvector repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for pgvector repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', + defaultValue: '17.6', description: 'Tag/Branch for pgvector packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build pgvector generic source packages') { parallel { stage('Build pgvector generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build pgvector generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build pgvector RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pgvector.yml b/ppg/pgvector.yml index bc31e98e3f..2779caff35 100644 --- a/ppg/pgvector.yml +++ b/ppg/pgvector.yml @@ -1,5 +1,5 @@ - job: - name: pgvector-RELEASE + name: hetzner-pgvector-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pgvector.groovy diff --git a/ppg/postgis_tarballs.groovy b/ppg/postgis_tarballs.groovy index fd0bb8ec91..9e689d94cd 100644 --- a/ppg/postgis_tarballs.groovy +++ b/ppg/postgis_tarballs.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -27,7 +27,7 @@ void uploadTarballToTestingDownloadServer(String tarballDirectory, String packag script { try { - uploadPGTarballToDownloadsTesting(tarballDirectory, packageVersion) + uploadPGTarballToDownloadsTesting(params.CLOUD, tarballDirectory, packageVersion) } catch (err) { echo "Caught: ${err}" currentBuild.result = 'UNSTABLE' @@ -40,8 +40,8 @@ void buildTarball(String platform, String architecture){ script { unstash "uploadPath-${PG_VERSION}" buildStage("${PG_VERSION}", platform, architecture) - pushArtifactFolder("postgis_output/", AWS_STASH_PATH) - uploadPGTarballfromAWS("postgis_output/", AWS_STASH_PATH, "binary", "${PG_VERSION}") + pushArtifactFolder(params.CLOUD, "postgis_output/", AWS_STASH_PATH) + uploadPGTarballfromAWS(params.CLOUD, "postgis_output/", AWS_STASH_PATH, "binary", "${PG_VERSION}") uploadTarballToTestingDownloadServer("postgis_tarballs", "${PG_VERSION}") } } @@ -56,10 +56,14 @@ def TIMESTAMP pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) //string( // defaultValue: 'https://github.com/percona/postgis-tarballs.git', // description: 'URL for postgis-tarballs repository', @@ -86,7 +90,7 @@ pipeline { stage('Create timestamp') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${PG_VERSION} - [${BUILD_URL}]") @@ -114,7 +118,7 @@ pipeline { parallel { stage('Build postgis-tarball for OL8 amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -125,7 +129,7 @@ pipeline { } stage('Build postgis-tarball for OL8 arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -136,7 +140,7 @@ pipeline { } stage('Build postgis-tarball for OL9 amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -147,7 +151,7 @@ pipeline { } stage('Build postgis-tarball for OL9 arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -158,7 +162,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu focal amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -169,7 +173,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu focal arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -180,7 +184,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu jammy amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -191,7 +195,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu jammy arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -202,7 +206,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu noble amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -213,7 +217,7 @@ pipeline { } stage('Build postgis-tarball for Ubuntu noble arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -224,7 +228,7 @@ pipeline { } stage('Build postgis-tarball for Debian bullseye amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -235,7 +239,7 @@ pipeline { } stage('Build postgis-tarball for Debian bullseye arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -246,7 +250,7 @@ pipeline { } stage('Build postgis-tarball for Debian bookworm amd64') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() @@ -257,7 +261,7 @@ pipeline { } stage('Build postgis-tarball for Debian bookworm arm64') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() diff --git a/ppg/postgis_tarballs.yml b/ppg/postgis_tarballs.yml index 970930e542..f6a0f11a60 100644 --- a/ppg/postgis_tarballs.yml +++ b/ppg/postgis_tarballs.yml @@ -1,5 +1,5 @@ - job: - name: postgis_tarballs-RELEASE + name: hetzner-postgis_tarballs-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgis_tarballs.groovy diff --git a/ppg/postgresql-common.groovy b/ppg/postgresql-common.groovy index c25ea38265..205aa71e37 100644 --- a/ppg/postgresql-common.groovy +++ b/ppg/postgresql-common.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres-common/ppg-common_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for postgresql-common repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'debian/256', - description: 'Tag/Branch for postgresql-common repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', + defaultValue: '17.6', description: 'Tag/Branch for postgresql-common packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,36 +79,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build postgresql-common generic source packages') { parallel { stage('Build postgresql-common generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build postgresql-common generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +117,106 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +224,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/postgresql-common.yml b/ppg/postgresql-common.yml index f0f10482dc..f501be8a68 100644 --- a/ppg/postgresql-common.yml +++ b/ppg/postgresql-common.yml @@ -1,5 +1,5 @@ - job: - name: postgresql-common-RELEASE + name: hetzner-postgresql-common-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgresql-common.groovy diff --git a/ppg/postgresql-ivee-arm.groovy b/ppg/postgresql-ivee-arm.groovy new file mode 100644 index 0000000000..9dc81ac7bc --- /dev/null +++ b/ppg/postgresql-ivee-arm.groovy @@ -0,0 +1,190 @@ +library changelog: false, identifier: "lib@master", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '12.19-1', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + echo \$MAJ_VER + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + echo \$MIN_VER + git clone https://github.com/Percona-Lab/postgresql-docker-ivee + cd postgresql-docker-ivee + sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile + sed -E "s/ENV PPG_MAJOR_VERSION (.+)/ENV PPG_MAJOR_VERSION \$MAJ_VER/" -i Dockerfile + sed -E "s/ENV PPG_MINOR_VERSION (.+)/ENV PPG_MINOR_VERSION \$MIN_VER/" -i Dockerfile + docker build . -t postgresql-ivee -f Dockerfile + """ + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag postgresql-ivee perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker tag postgresql-ivee perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker tag postgresql-ivee perconalab/postgresql-ivee:\$MAJ_VER-arm64 + docker push perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker push perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker push perconalab/postgresql-ivee:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/postgresql-ivee:${params.PPG_VERSION} \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-amd64 \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker manifest annotate perconalab/postgresql-ivee:${params.PPG_VERSION} \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:${params.PPG_VERSION} \ + perconalab/postgresql-ivee:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:${params.PPG_VERSION} + docker manifest push perconalab/postgresql-ivee:${params.PPG_VERSION} + + docker manifest create --amend perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER + docker manifest push perconalab/postgresql-ivee:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend perconalab/postgresql-ivee:\$MAJ_VER \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:\$MAJ_VER \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:\$MAJ_VER + docker manifest push perconalab/postgresql-ivee:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/postgresql-ivee:latest \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate perconalab/postgresql-ivee:latest \ + perconalab/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/postgresql-ivee:latest \ + perconalab/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/postgresql-ivee:latest + docker manifest push perconalab/postgresql-ivee:latest + fi + + """ + } + } + } + stage ('Push images to percona') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag postgresql-ivee percona/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker tag postgresql-ivee percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker tag postgresql-ivee percona/postgresql-ivee:\$MAJ_VER-arm64 + docker push percona/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker push percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker push percona/postgresql-ivee:\$MAJ_VER-arm64 + + docker manifest create --amend percona/postgresql-ivee:${params.PPG_VERSION} \ + percona/postgresql-ivee:${params.PPG_VERSION}-amd64 \ + percona/postgresql-ivee:${params.PPG_VERSION}-arm64 + docker manifest annotate percona/postgresql-ivee:${params.PPG_VERSION} \ + percona/postgresql-ivee:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:${params.PPG_VERSION} \ + percona/postgresql-ivee:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:${params.PPG_VERSION} + docker manifest push percona/postgresql-ivee:${params.PPG_VERSION} + + docker manifest create --amend percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER \ + percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER + docker manifest push percona/postgresql-ivee:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend percona/postgresql-ivee:\$MAJ_VER \ + percona/postgresql-ivee:\$MAJ_VER-amd64 \ + percona/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER \ + percona/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:\$MAJ_VER \ + percona/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:\$MAJ_VER + docker manifest push percona/postgresql-ivee:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/postgresql-ivee:latest \ + percona/postgresql-ivee:\$MAJ_VER-amd64 \ + percona/postgresql-ivee:\$MAJ_VER-arm64 + docker manifest annotate percona/postgresql-ivee:latest \ + percona/postgresql-ivee:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/postgresql-ivee:latest \ + percona/postgresql-ivee:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/postgresql-ivee:latest + docker manifest push percona/postgresql-ivee:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} fro ARM, repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/postgresql-ivee-arm.yml b/ppg/postgresql-ivee-arm.yml new file mode 100644 index 0000000000..7e4bc1b878 --- /dev/null +++ b/ppg/postgresql-ivee-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-postgresql-ivee-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql-ivee-arm.groovy diff --git a/ppg/postgresql_common_arm.groovy b/ppg/postgresql_common_arm.groovy new file mode 100644 index 0000000000..97dfd7f31e --- /dev/null +++ b/ppg/postgresql_common_arm.groovy @@ -0,0 +1,256 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres-common/ppg-common_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: '256', + description: 'Tag/Branch for postgresql-common', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgresql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgresql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build postgresql-common generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/postgresql_common_arm.yml b/ppg/postgresql_common_arm.yml new file mode 100644 index 0000000000..703baa26e9 --- /dev/null +++ b/ppg/postgresql_common_arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-postgresql-common-arm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql_common_arm.groovy + diff --git a/ppg/postgresql_server.groovy b/ppg/postgresql_server.groovy index 7430e2ff0a..3c0001bdd1 100644 --- a/ppg/postgresql_server.groovy +++ b/ppg/postgresql_server.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres/ppg_builder.sh -O ppg-server_builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for ppg-server repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'REL_15_RELEASE', - description: 'Tag/Branch for postgresql', - name: 'PG_BRANCH') - string( - defaultValue: '15.3', - description: 'Tag/Branch for ppg-server repository', + defaultValue: '17.6', + description: 'Tag/Branch for ppg-server packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-15.3', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER generic source packages') { parallel { stage('Build PPG-SERVER generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/postgresql_server.yml b/ppg/postgresql_server.yml index 56c1629bac..af42fc261c 100644 --- a/ppg/postgresql_server.yml +++ b/ppg/postgresql_server.yml @@ -1,5 +1,5 @@ - job: - name: postgresql-server-autobuild-RELEASE + name: hetzner-postgresql-server-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgresql_server.groovy diff --git a/ppg/postgresql_server_17_nightly.groovy b/ppg/postgresql_server_17_nightly.groovy new file mode 100644 index 0000000000..0eaac6f6fe --- /dev/null +++ b/ppg/postgresql_server_17_nightly.groovy @@ -0,0 +1,345 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres/ppg_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: 'TDE_REL_17_STABLE', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '17.6', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-17.6', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'testing\nlaboratory\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgresql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgresql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/postgresql_server_17_nightly.yml b/ppg/postgresql_server_17_nightly.yml new file mode 100644 index 0000000000..93556dd19f --- /dev/null +++ b/ppg/postgresql_server_17_nightly.yml @@ -0,0 +1,17 @@ +- job: + name: hetzner-postgresql-server-17_nightly + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql_server_17_nightly.groovy + triggers: + - timed: "0 0 * * *" + diff --git a/ppg/postgresql_server_arm.groovy b/ppg/postgresql_server_arm.groovy new file mode 100644 index 0000000000..a3b0889c0b --- /dev/null +++ b/ppg/postgresql_server_arm.groovy @@ -0,0 +1,240 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/postgres/ppg_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: 'REL_15_RELEASE', + description: 'Tag/Branch for postgresql', + name: 'PG_BRANCH') + string( + defaultValue: '15.3', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-15.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-postgresql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-postgresql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/postgresql_server_arm.yml b/ppg/postgresql_server_arm.yml new file mode 100644 index 0000000000..ff9e0e50d0 --- /dev/null +++ b/ppg/postgresql_server_arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-postgresql-server-arm_autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/postgresql_server_arm.groovy + diff --git a/ppg/postgresql_server_nightly.groovy b/ppg/postgresql_server_nightly.groovy index 94867da246..ab135c5960 100644 --- a/ppg/postgresql_server_nightly.groovy +++ b/ppg/postgresql_server_nightly.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for ppg-server repository', @@ -41,19 +45,19 @@ pipeline { description: 'Tag/Branch for postgresql', name: 'PG_BRANCH') string( - defaultValue: '17.0', + defaultValue: '17.5.2', description: 'Tag/Branch for ppg-server repository', name: 'GIT_BRANCH') string( - defaultValue: '1', + defaultValue: '2', description: 'RPM release value', name: 'RPM_RELEASE') string( - defaultValue: '1', + defaultValue: '2', description: 'DEB release value', name: 'DEB_RELEASE') string( - defaultValue: 'ppg-17.0', + defaultValue: 'ppg-17.5', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -106,8 +110,8 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER generic source packages') { @@ -117,28 +121,28 @@ pipeline { parallel { stage('Build PPG-SERVER generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -150,28 +154,28 @@ pipeline { parallel { stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -182,8 +186,8 @@ pipeline { expression { currentBuild.result != 'NOT_BUILT' } } steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { @@ -192,7 +196,7 @@ pipeline { } steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/postgresql_server_nightly.yml b/ppg/postgresql_server_nightly.yml index 5182d83774..f9d623b8e0 100644 --- a/ppg/postgresql_server_nightly.yml +++ b/ppg/postgresql_server_nightly.yml @@ -1,5 +1,5 @@ - job: - name: postgresql-server-nightly + name: hetzner-postgresql-server-nightly project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/postgresql_server_nightly.groovy diff --git a/ppg/ppg-controller-trigger.groovy b/ppg/ppg-controller-trigger.groovy new file mode 100644 index 0000000000..89b31f4b39 --- /dev/null +++ b/ppg/ppg-controller-trigger.groovy @@ -0,0 +1,177 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git', + credentialsId: '' // force anonymous for shared library +]) _ + +void cleanUpWS() { + sh "sudo rm -rf ./* || true" +} + +def AWS_STASH_PATH +def jobsConfig = [:] + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + + parameters { + string(name: 'CONFIG_REPO', defaultValue: 'https://github.com/Percona-Lab/postgres-packaging', description: 'Job config repo') + string(name: 'CONFIG_BRANCH', defaultValue: 'main', description: 'Job config branch') + string(name: 'CLOUD', defaultValue: 'Hetzner', description: 'Cloud target') + } + + environment { + CONFIG_FILE = 'postgres-packaging/job_configs/all-jobs-full.yaml' + } + + stages { + stage('Checkout Job Config Repo') { + steps { + script { + echo "[INFO] Installing jq and yq" + sh "sudo apt-get update -qq" + sh "sudo apt-get install -y jq" + sh "wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64" + sh "sudo mv yq_linux_amd64 /usr/local/bin/yq" + sh "sudo chmod +x /usr/local/bin/yq" + sh "echo '[✓] jq version: \$(jq --version)'" + sh "echo '[✓] yq version: \$(yq --version)'" + + echo "[INFO] Cloning CONFIG_REPO: ${params.CONFIG_REPO} (${params.CONFIG_BRANCH})" + dir('postgres-packaging') { + withEnv(['GIT_ASKPASS=', 'GIT_TERMINAL_PROMPT=0']) { + try { + sh ''' + set -eu + rm -rf ./* + git -c credential.helper= clone \ + --depth=1 --no-tags --single-branch \ + --branch "${CONFIG_BRANCH}" \ + "${CONFIG_REPO}.git" . + ''' + } catch (e) { + // If user passed Percona-Lab repo and it requires auth, fallback to public mirror + if ("${params.CONFIG_REPO}".toLowerCase().endsWith('/percona-lab/postgres-packaging')) { + echo "[WARN] ${params.CONFIG_REPO} might require auth. Retrying with public repo: https://github.com/percona/postgres-packaging" + sh ''' + set -eu + rm -rf ./* + git -c credential.helper= clone \ + --depth=1 --no-tags --single-branch \ + --branch "${CONFIG_BRANCH}" \ + "https://github.com/percona/postgres-packaging.git" . + ''' + } else { + throw e + } + } + } + } + + if (!fileExists(env.CONFIG_FILE)) { + error "❌ Config file not found: ${env.CONFIG_FILE}" + } + + echo "[✓] Repo checked out and config file exists." + def criticalJobs = [ + 'hetzner-postgresql-common-RELEASE', + 'hetzner-postgresql-server-autobuild-RELEASE', + 'hetzner-pg_percona_telemetry-autobuild-RELEASE' + ] + + criticalJobs.each { jobKey -> + def exists = sh(script: "yq eval '.\"${jobKey}\"' ${env.CONFIG_FILE} > /dev/null 2>&1", returnStatus: true) == 0 + if (!exists) { + echo "[SKIP] ${jobKey} not found in config." + return + } + + def paramsJSON = sh(script: "yq eval -o=json '.\"${jobKey}\".parameters // {}' ${env.CONFIG_FILE}", returnStdout: true).trim() + def paramKeys = sh(script: "echo '${paramsJSON}' | jq -r 'keys[]' 2>/dev/null || echo ''", returnStdout: true).trim().split('\n').findAll { it } + def buildParams = [] + + paramKeys.each { key -> + def value = sh(script: "echo '${paramsJSON}' | jq -r '.\"${key}\"'", returnStdout: true).trim() + def rawValue = sh(script: "echo '${paramsJSON}' | jq '.\"${key}\"'", returnStdout: true).trim() + if (rawValue == "true" || rawValue == "false") { + buildParams.add(booleanParam(name: key, value: rawValue == "true")) + } else { + buildParams.add(string(name: key, value: value)) + } + } + + echo "[▶] Running critical job: ${jobKey}" + build job: jobKey, + parameters: buildParams, + wait: true, + propagate: true + } + + def jobKeys = sh(script: "yq eval 'keys' ${env.CONFIG_FILE}", returnStdout: true) + .split('\n') + .collect { it.trim().replaceAll(/^[- ]/, '') } + .findAll { it && !criticalJobs.contains(it) } + + def parallelJobs = [:] + + jobKeys.each { jobKey -> + def trigger = sh(script: "yq eval '.\"${jobKey}\".trigger // false' ${env.CONFIG_FILE}", returnStdout: true).trim() + if (trigger != "true") return + + def paramsJSON = sh(script: "yq eval -o=json '.\"${jobKey}\".parameters // {}' ${env.CONFIG_FILE}", returnStdout: true).trim() + def paramKeys = sh(script: "echo '${paramsJSON}' | jq -r 'keys[]' 2>/dev/null || echo ''", returnStdout: true).trim().split('\n').findAll { it } + def buildParams = [] + + paramKeys.each { key -> + def value = sh(script: "echo '${paramsJSON}' | jq -r '.\"${key}\"'", returnStdout: true).trim() + def valueType = sh(script: "echo '${paramsJSON}' | jq -r 'type_of(.\"${key}\")'", returnStdout: true).trim() + if (valueType == "boolean") { + buildParams.add(booleanParam(name: key, value: value == "true")) + } else { + buildParams.add(string(name: key, value: value)) + } + } + + parallelJobs[jobKey] = { + echo "[▶] Running parallel job: ${jobKey}" + build job: jobKey, + parameters: buildParams, + wait: true, + propagate: true + } + } + + if (parallelJobs) { + parallel parallelJobs + } else { + echo "[✓] No parallel jobs to run." + } + } + } + } + } + + post { + success { + slackNotify( + "#releases-ci", + "#00FF00", + "[${env.JOB_NAME}]: ✅ Build finished successfully for branch ${params.CONFIG_BRANCH} → ${env.BUILD_URL}" + ) + script { currentBuild.description = "Built from ${params.CONFIG_BRANCH}" } + cleanUpWS() + } + failure { + slackNotify( + "#FF0000", + "[${env.JOB_NAME}]: ❌ Build failed for branch ${params.CONFIG_BRANCH} → ${env.BUILD_URL}" + ) + cleanUpWS() + } + always { + cleanUpWS() + } + } +} diff --git a/ppg/ppg-controller-trigger.yml b/ppg/ppg-controller-trigger.yml new file mode 100644 index 0000000000..4b7762b2cc --- /dev/null +++ b/ppg/ppg-controller-trigger.yml @@ -0,0 +1,28 @@ +- job: + name: ppg-controller-trigger + project-type: pipeline + description: | + 🔧 Orchestrates all PPG component jobs. + ⚠️ Do not edit this job via the Jenkins UI! + parameters: + - string: + name: CLOUD + default: Hetzner + description: "Cloud infra: Hetzner or AWS" + - string: + name: CONFIG_REPO + default: "https://github.com/percona/postgres-packaging.git" + description: "Git repo containing job_configs/all-jobs-full.yaml" + - string: + name: CONFIG_BRANCH + default: "main" + description: "Branch to pull the configuration from" + pipeline-scm: + scm: + - git: + url: "$CONFIG_REPO" + branches: + - "$CONFIG_BRANCH" + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-controller-trigger.groovy diff --git a/ppg/ppg-docker-arm.groovy b/ppg/ppg-docker-arm.groovy new file mode 100644 index 0000000000..ee371c2646 --- /dev/null +++ b/ppg/ppg-docker-arm.groovy @@ -0,0 +1,192 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '12.19-1', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + echo \$MAJ_VER + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + echo \$MIN_VER + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-distribution-postgresql-\$MAJ_VER + sed -E "s/ENV PPG_VERSION (.+)/ENV PPG_VERSION ${params.PPG_VERSION}/" -i Dockerfile.aarch64 + sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile.aarch64 + sed -E "s/ENV PPG_MAJOR_VERSION (.+)/ENV PPG_MAJOR_VERSION \$MAJ_VER/" -i Dockerfile.aarch64 + sed -E "s/ENV PPG_MINOR_VERSION (.+)/ENV PPG_MINOR_VERSION \$MIN_VER/" -i Dockerfile.aarch64 + docker build . -t percona-distribution-postgresql -f Dockerfile.aarch64 + """ + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag percona-distribution-postgresql perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker tag percona-distribution-postgresql perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker tag percona-distribution-postgresql perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker push perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker push perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker push perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/percona-distribution-postgresql:${params.PPG_VERSION} \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:${params.PPG_VERSION} \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:${params.PPG_VERSION} \ + perconalab/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:${params.PPG_VERSION} + docker manifest push perconalab/percona-distribution-postgresql:${params.PPG_VERSION} + + docker manifest create --amend perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + docker manifest push perconalab/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend perconalab/percona-distribution-postgresql:\$MAJ_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:\$MAJ_VER \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:\$MAJ_VER + docker manifest push perconalab/percona-distribution-postgresql:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/percona-distribution-postgresql:latest \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-distribution-postgresql:latest \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-distribution-postgresql:latest \ + perconalab/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-distribution-postgresql:latest + docker manifest push perconalab/percona-distribution-postgresql:latest + fi + + """ + } + } + } + stage ('Push images to percona') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f1 -d'.') + MIN_VER=\$(echo ${params.PPG_VERSION} | cut -f1 -d'-' | cut -f2 -d'.') + docker tag percona-distribution-postgresql percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker tag percona-distribution-postgresql percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker tag percona-distribution-postgresql percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker push percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker push percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker push percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + + docker manifest create --amend percona/percona-distribution-postgresql:${params.PPG_VERSION} \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 + docker manifest annotate percona/percona-distribution-postgresql:${params.PPG_VERSION} \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:${params.PPG_VERSION} \ + percona/percona-distribution-postgresql:${params.PPG_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:${params.PPG_VERSION} + docker manifest push percona/percona-distribution-postgresql:${params.PPG_VERSION} + + docker manifest create --amend percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + docker manifest push percona/percona-distribution-postgresql:\$MAJ_VER.\$MIN_VER + + docker manifest create --amend percona/percona-distribution-postgresql:\$MAJ_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:\$MAJ_VER \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:\$MAJ_VER + docker manifest push percona/percona-distribution-postgresql:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/percona-distribution-postgresql:latest \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-distribution-postgresql:latest \ + percona/percona-distribution-postgresql:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-distribution-postgresql:latest \ + percona/percona-distribution-postgresql:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-distribution-postgresql:latest + docker manifest push percona/percona-distribution-postgresql:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} fro ARM, repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of PPG ${PPG_VERSION} for ARM, repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/ppg-docker-arm.yml b/ppg/ppg-docker-arm.yml new file mode 100644 index 0000000000..d789254aba --- /dev/null +++ b/ppg/ppg-docker-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-ppg-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-docker-arm.groovy diff --git a/ppg/ppg-docker.groovy b/ppg/ppg-docker.groovy index 6af0fdcd97..179af39dbb 100644 --- a/ppg/ppg-docker.groovy +++ b/ppg/ppg-docker.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PPG_VERSION', defaultValue: '16.3-1', description: 'PPG version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') diff --git a/ppg/ppg-docker.yml b/ppg/ppg-docker.yml index 63c482693d..a70341aa47 100644 --- a/ppg/ppg-docker.yml +++ b/ppg/ppg-docker.yml @@ -1,5 +1,5 @@ - job: - name: ppg-docker + name: hetzner-ppg-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-docker.groovy diff --git a/ppg/ppg-pgbackrest-docker-arm.groovy b/ppg/ppg-pgbackrest-docker-arm.groovy new file mode 100644 index 0000000000..b47e91ca17 --- /dev/null +++ b/ppg/ppg-pgbackrest-docker-arm.groovy @@ -0,0 +1,183 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PGBACKREST_VERSION', defaultValue: '2.51-1', description: 'pgBackrest version') + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-pgbackrest + sed -E "s/ARG PG_VERSION=(.+)/ARG PG_VERSION=${params.PPG_VERSION}/" -i Dockerfile + sed -E "s/ARG PPG_REPO=(.+)/ARG PPG_REPO=${params.PPG_REPO}/" -i Dockerfile + docker build . -t percona-pgbackrest + """ + } + } + stage ('Push image to aws ecr') { + when { + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} + docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} + if [ ${params.LATEST} = "yes" ]; then + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest + docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest + fi + """ + } + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbackrest perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker push perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker tag percona-pgbackrest perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + docker push perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker manifest annotate perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} + docker manifest push perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION} + + docker manifest create --amend perconalab/percona-pgbackrest:\$MAJ_VER \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbackrest:\$MAJ_VER \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbackrest:\$MAJ_VER \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbackrest:\$MAJ_VER + docker manifest push perconalab/percona-pgbackrest:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/percona-pgbackrest:latest \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbackrest:latest \ + perconalab/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbackrest:latest \ + perconalab/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbackrest:latest + docker manifest push perconalab/percona-pgbackrest:latest + fi + + """ + } + } + } + stage ('Push images to official percona docker registry') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbackrest percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker push percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker tag percona-pgbackrest percona/percona-pgbackrest:\$MAJ_VER-arm64 + docker push percona/percona-pgbackrest:\$MAJ_VER-arm64 + + docker manifest create --amend percona/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 + docker manifest annotate percona/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbackrest:${params.PGBACKREST_VERSION} \ + percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbackrest:${params.PGBACKREST_VERSION} + docker manifest push percona/percona-pgbackrest:${params.PGBACKREST_VERSION} + + docker manifest create --amend percona/percona-pgbackrest:\$MAJ_VER \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbackrest:\$MAJ_VER \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbackrest:\$MAJ_VER \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbackrest:\$MAJ_VER + docker manifest push percona/percona-pgbackrest:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/percona-pgbackrest:latest \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbackrest:latest \ + percona/percona-pgbackrest:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbackrest:latest \ + percona/percona-pgbackrest:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbackrest:latest + docker manifest push percona/percona-pgbackrest:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of Percona pgBackrest ${PGBACKREST_VERSION} repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of Percona pgBackrest ${PGBACKREST_VERSION} repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of Percona pgBackrest ${PGBACKREST_VERSION} repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/ppg-pgbackrest-docker-arm.yml b/ppg/ppg-pgbackrest-docker-arm.yml new file mode 100644 index 0000000000..577b1b07e2 --- /dev/null +++ b/ppg/ppg-pgbackrest-docker-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-percona-pgbackrest-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-pgbackrest-docker-arm.groovy diff --git a/ppg/ppg-pgbackrest-docker.groovy b/ppg/ppg-pgbackrest-docker.groovy index eef8f70697..0e1df0183e 100644 --- a/ppg/ppg-pgbackrest-docker.groovy +++ b/ppg/ppg-pgbackrest-docker.groovy @@ -1,17 +1,18 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { - string(name: 'PGBACKREST_VERSION', defaultValue: '2.51-1', description: 'pgBackrest version') + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PGBACKREST_VERSION', defaultValue: '2.51-1', description: 'pgBackrest version') choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') @@ -33,9 +34,9 @@ pipeline { sh """ git clone https://github.com/percona/percona-docker cd percona-docker/percona-pgbackrest - sed -E "s/ENV PG_VERSION (.+)/ENV PG_VERSION ${params.PPG_VERSION}/" -i Dockerfile - sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile - docker build . -t percona-pgbackrest:${params.PGBACKREST_VERSION} + sed -E "s/ARG PG_VERSION=(.+)/ARG PG_VERSION=${params.PPG_VERSION}/" -i Dockerfile + sed -E "s/ARG PPG_REPO=(.+)/ARG PPG_REPO=${params.PPG_REPO}/" -i Dockerfile + docker build . -t percona-pgbackrest """ } } @@ -48,10 +49,10 @@ pipeline { wget https://raw.githubusercontent.com/aquasecurity/trivy/v\${TRIVY_VERSION}/contrib/junit.tpl if [ ${params.PPG_REPO} = "release" ]; then /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbackrest:${params.PGBACKREST_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbackrest else /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbackrest:${params.PGBACKREST_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbackrest fi """ } @@ -73,10 +74,10 @@ pipeline { unzip -o awscliv2.zip sudo ./aws/install aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest-build:percona-pgbackrest:${params.PGBACKREST_VERSION} if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest + docker tag percona-pgbackrest public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest docker push public.ecr.aws/e7j3v3n0/percona-pgbackrest:latest fi """ @@ -92,12 +93,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 + docker tag percona-pgbackrest perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 docker push perconalab/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} perconalab/percona-pgbackrest:\$MAJ_VER-amd64 + docker tag percona-pgbackrest perconalab/percona-pgbackrest:\$MAJ_VER-amd64 docker push perconalab/percona-pgbackrest:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} perconalab/percona-pgbackrest:latest + docker tag percona-pgbackrest perconalab/percona-pgbackrest:latest docker push perconalab/percona-pgbackrest:latest fi """ @@ -113,12 +114,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBACKREST_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 + docker tag percona-pgbackrest percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 docker push percona/percona-pgbackrest:${params.PGBACKREST_VERSION}-amd64 - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} percona/percona-pgbackrest:\$MAJ_VER-amd64 + docker tag percona-pgbackrest percona/percona-pgbackrest:\$MAJ_VER-amd64 docker push percona/percona-pgbackrest:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbackrest:${params.PGBACKREST_VERSION} percona/percona-pgbackrest:latest + docker tag percona-pgbackrest percona/percona-pgbackrest:latest docker push percona/percona-pgbackrest:latest fi """ diff --git a/ppg/ppg-pgbackrest-docker.yml b/ppg/ppg-pgbackrest-docker.yml index c584383a39..57bb7f9efd 100644 --- a/ppg/ppg-pgbackrest-docker.yml +++ b/ppg/ppg-pgbackrest-docker.yml @@ -1,5 +1,5 @@ - job: - name: percona-pgbackrest-docker + name: hetzner-percona-pgbackrest-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-pgbackrest-docker.groovy diff --git a/ppg/ppg-pgbouncer-docker-arm.groovy b/ppg/ppg-pgbouncer-docker-arm.groovy new file mode 100644 index 0000000000..2d21fc2d71 --- /dev/null +++ b/ppg/ppg-pgbouncer-docker-arm.groovy @@ -0,0 +1,182 @@ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + environment { + PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' + } + parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'PGBOUNCER_VERSION', defaultValue: '1.22.1-1', description: 'pgBouncer version') + choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') + string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') + choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') + choice(name: 'LATEST', choices: ['no','yes'], description: 'Tag image as latest') + } + options { + disableConcurrentBuilds() + } + stages { + stage('Set build name'){ + steps { + script { + currentBuild.displayName = "${params.PPG_REPO}-${params.PPG_VERSION}" + } + } + } + stage ('Build image') { + steps { + sh """ + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-pgbouncer + sed -E "s/ARG PG_VERSION=(.+)/ARG PG_VERSION=${params.PPG_VERSION}/" -i Dockerfile + sed -E "s/ARG PPG_REPO=(.+)/ARG PPG_REPO=${params.PPG_REPO}/" -i Dockerfile + docker build . -t percona-pgbouncer + """ + } + } + stage ('Push image to aws ecr') { + when { + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} + if [ ${params.LATEST} = "yes" ]; then + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest + docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest + fi + """ + } + } + } + stage ('Push images to perconalab') { + when { + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbouncer perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker push perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker tag percona-pgbouncer perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + docker push perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + + docker manifest create --amend perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker manifest annotate perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker manifest push perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION} + + docker manifest create --amend perconalab/percona-pgbouncer:\$MAJ_VER \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbouncer:\$MAJ_VER \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbouncer:\$MAJ_VER \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbouncer:\$MAJ_VER + docker manifest push perconalab/percona-pgbouncer:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend perconalab/percona-pgbouncer:latest \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate perconalab/percona-pgbouncer:latest \ + perconalab/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate perconalab/percona-pgbouncer:latest \ + perconalab/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect perconalab/percona-pgbouncer:latest + docker manifest push perconalab/percona-pgbouncer:latest + fi + """ + } + } + } + stage ('Push images to official percona docker registry') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') + docker login -u '${USER}' -p '${PASS}' + docker tag percona-pgbouncer percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker push percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker tag percona-pgbouncer percona/percona-pgbouncer:\$MAJ_VER-arm64 + docker push percona/percona-pgbouncer:\$MAJ_VER-arm64 + + docker manifest create --amend percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 + docker manifest annotate percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} \ + percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker manifest push percona/percona-pgbouncer:${params.PGBOUNCER_VERSION} + + docker manifest create --amend percona/percona-pgbouncer:\$MAJ_VER \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbouncer:\$MAJ_VER \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbouncer:\$MAJ_VER \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbouncer:\$MAJ_VER + docker manifest push percona/percona-pgbouncer:\$MAJ_VER + + if [ ${params.LATEST} = "yes" ]; then + docker manifest create --amend percona/percona-pgbouncer:latest \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-pgbouncer:latest \ + percona/percona-pgbouncer:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-pgbouncer:latest \ + percona/percona-pgbouncer:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-pgbouncer:latest + docker manifest push percona/percona-pgbouncer:latest + fi + """ + } + } + } + } + post { + always { + sh """ + sudo docker rmi -f \$(sudo docker images -q | uniq) || true + sudo rm -rf ${WORKSPACE}/* + """ + deleteDir() + } + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of Percona pgBouncer ${PGBOUNCER_VERSION} repo ${PPG_REPO} succeed") + } + unstable { + slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of Percona pgBouncer ${PGBOUNCER_VERSION} repo ${PPG_REPO} unstable - [${BUILD_URL}testReport/]") + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of Percona pgBouncer ${PGBOUNCER_VERSION} repo ${PPG_REPO} failed - [${BUILD_URL}]") + } + } +} diff --git a/ppg/ppg-pgbouncer-docker-arm.yml b/ppg/ppg-pgbouncer-docker-arm.yml new file mode 100644 index 0000000000..3e99e9246d --- /dev/null +++ b/ppg/ppg-pgbouncer-docker-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-percona-pgbouncer-docker-arm + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-pgbouncer-docker-arm.groovy diff --git a/ppg/ppg-pgbouncer-docker.groovy b/ppg/ppg-pgbouncer-docker.groovy index dfcc2097dd..e69112dff2 100644 --- a/ppg/ppg-pgbouncer-docker.groovy +++ b/ppg/ppg-pgbouncer-docker.groovy @@ -1,16 +1,17 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PGBOUNCER_VERSION', defaultValue: '1.22.1-1', description: 'pgBouncer version') choice(name: 'PPG_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PPG_VERSION', defaultValue: '17.0', description: 'PPG version') @@ -33,9 +34,9 @@ pipeline { sh """ git clone https://github.com/percona/percona-docker cd percona-docker/percona-pgbouncer - sed -E "s/ENV PG_VERSION (.+)/ENV PG_VERSION ${params.PPG_VERSION}/" -i Dockerfile - sed -E "s/ENV PPG_REPO (.+)/ENV PPG_REPO ${params.PPG_REPO}/" -i Dockerfile - docker build . -t percona-pgbouncer:${params.PGBOUNCER_VERSION} + sed -E "s/ARG PG_VERSION=(.+)/ARG PG_VERSION=${params.PPG_VERSION}/" -i Dockerfile + sed -E "s/ARG PPG_REPO=(.+)/ARG PPG_REPO=${params.PPG_REPO}/" -i Dockerfile + docker build . -t percona-pgbouncer """ } } @@ -48,10 +49,10 @@ pipeline { wget https://raw.githubusercontent.com/aquasecurity/trivy/v\${TRIVY_VERSION}/contrib/junit.tpl if [ ${params.PPG_REPO} = "release" ]; then /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbouncer:${params.PGBOUNCER_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-pgbouncer else /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbouncer:${params.PGBOUNCER_VERSION} + --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-pgbouncer fi """ } @@ -73,10 +74,10 @@ pipeline { unzip -o awscliv2.zip sudo ./aws/install aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer-build:percona-pgbouncer:${params.PGBOUNCER_VERSION} if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest + docker tag percona-pgbouncer public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest docker push public.ecr.aws/e7j3v3n0/percona-pgbouncer:latest fi """ @@ -92,12 +93,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 + docker tag percona-pgbouncer perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 docker push perconalab/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} perconalab/percona-pgbouncer:\$MAJ_VER-amd64 + docker tag percona-pgbouncer perconalab/percona-pgbouncer:\$MAJ_VER-amd64 docker push perconalab/percona-pgbouncer:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} perconalab/percona-pgbouncer:latest + docker tag percona-pgbouncer perconalab/percona-pgbouncer:latest docker push perconalab/percona-pgbouncer:latest fi """ @@ -113,12 +114,12 @@ pipeline { sh """ MAJ_VER=\$(echo ${params.PGBOUNCER_VERSION} | cut -f1 -d'-') docker login -u '${USER}' -p '${PASS}' - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 + docker tag percona-pgbouncer percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 docker push percona/percona-pgbouncer:${params.PGBOUNCER_VERSION}-amd64 - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} percona/percona-pgbouncer:\$MAJ_VER-amd64 + docker tag percona-pgbouncer percona/percona-pgbouncer:\$MAJ_VER-amd64 docker push percona/percona-pgbouncer:\$MAJ_VER-amd64 if [ ${params.LATEST} = "yes" ]; then - docker tag percona-pgbouncer:${params.PGBOUNCER_VERSION} percona/percona-pgbouncer:latest + docker tag percona-pgbouncer percona/percona-pgbouncer:latest docker push percona/percona-pgbouncer:latest fi """ diff --git a/ppg/ppg-pgbouncer-docker.yml b/ppg/ppg-pgbouncer-docker.yml index 1d988e6831..e34912d882 100644 --- a/ppg/ppg-pgbouncer-docker.yml +++ b/ppg/ppg-pgbouncer-docker.yml @@ -1,5 +1,5 @@ - job: - name: percona-pgbouncer-docker + name: hetzner-percona-pgbouncer-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-pgbouncer-docker.groovy diff --git a/ppg/ppg-server-arm.groovy b/ppg/ppg-server-arm.groovy new file mode 100644 index 0000000000..3a43201106 --- /dev/null +++ b/ppg/ppg-server-arm.groovy @@ -0,0 +1,237 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ppg-server/ppg-server_builder.sh -O ppg-server_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${PPG_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server repository', + name: 'GIT_REPO') + string( + defaultValue: '14.4', + description: 'Tag/Branch for ppg-server repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-14.4', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ppg-server.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/ppg-server.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER generic source packages') { + parallel { + stage('Build PPG-SERVER generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/ppg-server-arm.yml b/ppg/ppg-server-arm.yml new file mode 100644 index 0000000000..bd4c2daa77 --- /dev/null +++ b/ppg/ppg-server-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-ppg-server-arm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-server-arm.groovy + diff --git a/ppg/ppg-server-ha-arm.groovy b/ppg/ppg-server-ha-arm.groovy new file mode 100644 index 0000000000..21c7b2bb30 --- /dev/null +++ b/ppg/ppg-server-ha-arm.groovy @@ -0,0 +1,237 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ppg-server-ha/ppg-server-ha_builder.sh -O ppg-server-ha_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${PPG_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ppg-server-ha repository', + name: 'GIT_REPO') + string( + defaultValue: '14.4', + description: 'Tag/Branch for ppg-server-ha repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-14.4', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PPG_SERVER_HA source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ppg-server-ha.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/ppg-server-ha.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PPG-SERVER_HA generic source packages') { + parallel { + stage('Build PPG-SERVER_HA generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PPG-SERVER_HA generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PPG-SERVER-HA RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/ppg-server-ha-arm.yml b/ppg/ppg-server-ha-arm.yml new file mode 100644 index 0000000000..58122159ef --- /dev/null +++ b/ppg/ppg-server-ha-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-ppg-server-ha-arm-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ppg-server-ha-arm.groovy + diff --git a/ppg/ppg-server-ha.groovy b/ppg/ppg-server-ha.groovy index e097648111..76047a5ac0 100644 --- a/ppg/ppg-server-ha.groovy +++ b/ppg/ppg-server-ha.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ppg-server-ha/ppg-server-ha_builder.sh -O ppg-server-ha_builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${PPG_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./ppg-server-ha_builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,27 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for ppg-server-ha repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '14.4', - description: 'Tag/Branch for ppg-server-ha repository', + defaultValue: '17.6', + description: 'Tag/Branch for ppg-server-ha packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-14.4', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -71,140 +70,263 @@ pipeline { sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ppg-server-ha.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo "INFO: UPLOAD PATH: ${REPO_UPLOAD_PATH}" echo ${REPO_UPLOAD_PATH} > uploadPath echo ${AWS_STASH_PATH} > awsUploadPath cat test/ppg-server-ha.properties cat uploadPath ''' + script { + def path = sh(script: 'cat uploadPath', returnStdout: true).trim() + currentBuild.description = "UPLOAD_PATH: ${path}" + } + archiveArtifacts artifacts: 'uploadPath', fingerprint: true script { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER_HA generic source packages') { parallel { stage('Build PPG-SERVER_HA generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER_HA generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PPG-SERVER-HA RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -212,14 +334,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/ppg-server-ha.yml b/ppg/ppg-server-ha.yml index 64e0bd6938..2d74c54f08 100644 --- a/ppg/ppg-server-ha.yml +++ b/ppg/ppg-server-ha.yml @@ -1,5 +1,5 @@ - job: - name: ppg-server-ha-autobuild-RELEASE + name: hetzner-ppg-server-ha-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-server-ha.groovy diff --git a/ppg/ppg-server.groovy b/ppg/ppg-server.groovy index 82340a9072..9eb81b7977 100644 --- a/ppg/ppg-server.groovy +++ b/ppg/ppg-server.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ppg-server/ppg-server_builder.sh -O ppg-server_builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${PPG_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./ppg-server_builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,27 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for ppg-server repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '14.4', - description: 'Tag/Branch for ppg-server repository', + defaultValue: '17.6', + description: 'Tag/Branch for ppg-server packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-14.4', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -80,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PPG-SERVER generic source packages') { parallel { stage('Build PPG-SERVER generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PPG-SERVER generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PPG-SERVER RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Ubuntu Jammy(22.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye(11) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -212,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/ppg-server.yml b/ppg/ppg-server.yml index 5d334a1cf2..8dbf177b84 100644 --- a/ppg/ppg-server.yml +++ b/ppg/ppg-server.yml @@ -1,5 +1,5 @@ - job: - name: ppg-server-autobuild-RELEASE + name: hetzner-ppg-server-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ppg-server.groovy diff --git a/ppg/ppg_release.groovy b/ppg/ppg_release.groovy index 3449f0a87f..a75229ac41 100644 --- a/ppg/ppg_release.groovy +++ b/ppg/ppg_release.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@ENG-7_postgresql_rel', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -13,9 +13,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) text(name: 'PACKAGES', defaultValue: '', description: 'put all pathes to all packages') string(name: 'repo_version', defaultValue: '', description: 'Repository Version i.e ppg-15.3 or ppg-15') choice(name: 'component', choices: ['release', 'testing', 'experimental', 'laboratory'], description: 'Component') @@ -62,12 +66,12 @@ pipeline { } stage('Push to public repository') { steps { - sync2ProdPPG(repo_version, component) + sync2ProdPPG(params.CLOUD, repo_version, component) } } stage('Push to WEB') { steps { - sync2web(repo_version, component) + sync2web(params.CLOUD, repo_version, component) } } } diff --git a/ppg/ppg_release.yml b/ppg/ppg_release.yml index 246edb818a..5fcf30c4f6 100644 --- a/ppg/ppg_release.yml +++ b/ppg/ppg_release.yml @@ -1,5 +1,5 @@ - job: - name: Percona-PostgreSQL-Distribution-RELEASE + name: hetzner-Percona-PostgreSQL-Distribution-RELEASE project-type: pipeline description: | Do not edit this job through the web! diff --git a/ppg/proj95.groovy b/ppg/proj95.groovy new file mode 100644 index 0000000000..c2c5272fa7 --- /dev/null +++ b/ppg/proj95.groovy @@ -0,0 +1,165 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_IMAGE, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir -p test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/master/proj95/builder.sh -O builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_IMAGE} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --rpm_release=${RPM_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh "sudo rm -rf ./*" +} + +def AWS_STASH_PATH + +pipeline { + agent none + + parameters { + choice(name: 'CLOUD', choices: ['Hetzner', 'AWS'], description: 'Cloud infra for build') + string(name: 'GIT_REPO', defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for proj95 repository') + string(name: 'GIT_BRANCH', defaultValue: '16.3', description: 'Tag/Branch for postgresql packaging repository') + string(name: 'RPM_RELEASE', defaultValue: '1', description: 'RPM release value') + string(name: 'PPG_REPO', defaultValue: 'ppg-16.3', description: 'PPG repo name') + choice(name: 'COMPONENT', choices: ['laboratory', 'testing', 'experimental'], description: 'Repo component to push packages to') + } + + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + + stages { + stage('Build SRPM (generic)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + buildStage("oraclelinux:8", "--get_src_rpm=1") + pushArtifactFolder(params.CLOUD, "srpm/", "") + popArtifactFolder(params.CLOUD, "srpm/", "") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/proj95.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/proj95.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + stash includes: 'test/**', name: 'srpmContents' + } + } + stage('Build RPMs in Parallel') { + parallel { + stage('Build Oracle Linux 8 x86_64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:8", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Build Oracle Linux 8 aarch64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:8", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Build Oracle Linux 9 x86_64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:9", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + + stage('Build Oracle Linux 9 aarch64') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + unstash 'uploadPath' + unstash 'srpmContents' + script { + buildStage("oraclelinux:9", "--build_rpm=1") + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Sign packages') { + agent { label 'docker' } + steps { + signRPM(params.CLOUD) + } + } + + stage('Push to public repository') { + agent { label 'docker' } + steps { + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + } + + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } + always { + cleanUpWS() + deleteDir() + } + } +} \ No newline at end of file diff --git a/ppg/proj95.yml b/ppg/proj95.yml new file mode 100644 index 0000000000..b0c2e96f76 --- /dev/null +++ b/ppg/proj95.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-proj95-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/proj95.groovy \ No newline at end of file diff --git a/ppg/pysyncobj-arm.groovy b/ppg/pysyncobj-arm.groovy new file mode 100644 index 0000000000..ad076bf852 --- /dev/null +++ b/ppg/pysyncobj-arm.groovy @@ -0,0 +1,254 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pysyncobj/pysyncobj_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for pysyncobj repository', + name: 'GIT_REPO') + string( + defaultValue: '0.3.10', + description: 'Tag/Branch for pysyncobj repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.3', + description: 'Tag/Branch for postgresql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PYSYNCOBJ source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/pysyncobj.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pysyncobj.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PYSYNCOBJ generic source packages') { + parallel { + stage('Build PYSYNCOBJ generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PYSYNCOBJ generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PYSYNCOBJ RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/pysyncobj-arm.yml b/ppg/pysyncobj-arm.yml new file mode 100644 index 0000000000..20b6f7db9e --- /dev/null +++ b/ppg/pysyncobj-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pysyncobj-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/pysyncobj-arm.groovy + diff --git a/ppg/pysyncobj.groovy b/ppg/pysyncobj.groovy index 96d434181e..a53a56994b 100644 --- a/ppg/pysyncobj.groovy +++ b/ppg/pysyncobj.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/pysyncobj/pysyncobj_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for pysyncobj repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '0.3.10', - description: 'Tag/Branch for pysyncobj repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.3', - description: 'Tag/Branch for postgresql packaging repository', + defaultValue: '17.6', + description: 'Tag/Branch for pysyncobj packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.3', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,36 +79,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PYSYNCOBJ generic source packages') { parallel { stage('Build PYSYNCOBJ generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PYSYNCOBJ generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +117,106 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +224,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/pysyncobj.yml b/ppg/pysyncobj.yml index ce7e345525..012934ebd0 100644 --- a/ppg/pysyncobj.yml +++ b/ppg/pysyncobj.yml @@ -1,5 +1,5 @@ - job: - name: pysyncobj-RELEASE + name: hetzner-pysyncobj-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/pysyncobj.groovy diff --git a/ppg/timescaledb-arm.groovy b/ppg/timescaledb-arm.groovy new file mode 100644 index 0000000000..9f8a9f3c81 --- /dev/null +++ b/ppg/timescaledb-arm.groovy @@ -0,0 +1,253 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/timescaledb/timescaledb_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for timescaledb repository', + name: 'GIT_REPO') + string( + defaultValue: '2.15.2', + description: 'Branch/Tag of the timescaledb', + name: 'PG_BRANCH') + string( + defaultValue: '16.3', + description: 'Tag/Branch for packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16-extras', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create timescaledb source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/timescaledb.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/timescaledb.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build timescaledb generic source packages') { + parallel { + stage('Build timescaledb generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build timescaledb generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build timescaledb RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/timescaledb-arm.yml b/ppg/timescaledb-arm.yml new file mode 100644 index 0000000000..52bff7a322 --- /dev/null +++ b/ppg/timescaledb-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-timescaledb-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/timescaledb-arm.groovy diff --git a/ppg/timescaledb.groovy b/ppg/timescaledb.groovy index 6b53b41c09..0aa6a6f47e 100644 --- a/ppg/timescaledb.groovy +++ b/ppg/timescaledb.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', description: 'URL for timescaledb repository', @@ -84,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build timescaledb generic source packages') { parallel { stage('Build timescaledb generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build timescaledb generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -122,93 +126,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +220,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/timescaledb.yml b/ppg/timescaledb.yml index 22fb8cde7b..47970dff7a 100644 --- a/ppg/timescaledb.yml +++ b/ppg/timescaledb.yml @@ -1,5 +1,5 @@ - job: - name: timescaledb-RELEASE + name: hetzner-timescaledb-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/timescaledb.groovy diff --git a/ppg/wal2json-arm.groovy b/ppg/wal2json-arm.groovy new file mode 100644 index 0000000000..ed4da67005 --- /dev/null +++ b/ppg/wal2json-arm.groovy @@ -0,0 +1,240 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/wal2json/wal2json_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for wal2json repository', + name: 'GIT_REPO') + string( + defaultValue: '2.5', + description: 'General version of the product', + name: 'PG_BRANCH') + string( + defaultValue: '16.1', + description: 'Tag/Branch for wal2json repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.1', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create WAL2JSON source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/wal2json.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/wal2json.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build WAL2JSON generic source packages') { + parallel { + stage('Build WAL2JSON generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build wal2json generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build WAL2JSON RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/wal2json-arm.yml b/ppg/wal2json-arm.yml new file mode 100644 index 0000000000..3e345ed16b --- /dev/null +++ b/ppg/wal2json-arm.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-wal2json-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/wal2json-arm.groovy \ No newline at end of file diff --git a/ppg/wal2json.groovy b/ppg/wal2json.groovy index 4367bb2fba..b38079053c 100644 --- a/ppg/wal2json.groovy +++ b/ppg/wal2json.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/wal2json/wal2json_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for wal2json repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: 'ver_1.4.4', - description: 'Tag/Branch for wal2json repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.0', + defaultValue: '17.6', description: 'Tag/Branch for wal2json packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.0', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,248 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build wal2json generic source packages') { parallel { stage('Build wal2json generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build wal2json generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build wal2json RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04) AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian bookworm(12)') { + stage('Debian bookworm(12) ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +328,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/wal2json.yml b/ppg/wal2json.yml index c0b09523a5..c76eff220e 100644 --- a/ppg/wal2json.yml +++ b/ppg/wal2json.yml @@ -1,5 +1,5 @@ - job: - name: wal2json-RELEASE + name: hetzner-wal2json-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/wal2json.groovy diff --git a/ppg/ydiff-arm.groovy b/ppg/ydiff-arm.groovy new file mode 100644 index 0000000000..623e1437e7 --- /dev/null +++ b/ppg/ydiff-arm.groovy @@ -0,0 +1,241 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ydiff/ydiff_builder.sh -O builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/postgres-packaging.git', + description: 'URL for ydiff repository', + name: 'GIT_REPO') + string( + defaultValue: '1.2', + description: 'Tag/Branch for ydiff repository', + name: 'PG_BRANCH') + string( + defaultValue: '16.3', + description: 'Tag/Branch for postgresql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'ppg-16.3', + description: 'PPG repo name', + name: 'PPG_REPO') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create YDIFF source tarball') { + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:8", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/ydiff.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/ydiff.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build YDIFF generic source packages') { + parallel { + stage('Build YDIFF generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build YDIFF generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build YDIFF RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) + } + } + + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/ppg/ydiff-arm.yml b/ppg/ydiff-arm.yml new file mode 100644 index 0000000000..790ef63f45 --- /dev/null +++ b/ppg/ydiff-arm.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-ydiff-arm-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ppg/ydiff-arm.groovy + diff --git a/ppg/ydiff.groovy b/ppg/ydiff.groovy index 759bfa22c3..7a164af763 100644 --- a/ppg/ydiff.groovy +++ b/ppg/ydiff.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,9 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace mkdir test wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/ydiff/ydiff_builder.sh -O builder.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/versions.sh -O versions.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/install-deps.sh -O install-deps.sh + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/common-functions.sh -O common-functions.sh pwd -P ls -laR export build_dir=\$(pwd -P) @@ -15,7 +18,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { set -o xtrace cd \${build_dir} bash -x ./builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./builder.sh --builddir=\${build_dir}/test --branch=${PG_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + bash -x ./builder.sh --builddir=\${build_dir}/test ${STAGE_PARAM}" """ } @@ -29,31 +32,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/postgres-packaging.git', - description: 'URL for ydiff repository', + description: 'URL for packaging repository', name: 'GIT_REPO') string( - defaultValue: '1.2', - description: 'Tag/Branch for ydiff repository', - name: 'PG_BRANCH') - string( - defaultValue: '16.3', - description: 'Tag/Branch for postgresql packaging repository', + defaultValue: '17.6', + description: 'Tag/Branch for ydiff packaging repository', name: 'GIT_BRANCH') string( - defaultValue: '1', - description: 'RPM release value', - name: 'RPM_RELEASE') - string( - defaultValue: '1', - description: 'DEB release value', - name: 'DEB_RELEASE') - string( - defaultValue: 'ppg-16.3', + defaultValue: 'ppg-17.6', description: 'PPG repo name', name: 'PPG_REPO') choice( @@ -84,131 +79,183 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build YDIFF generic source packages') { parallel { stage('Build YDIFF generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build YDIFF generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_src_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build YDIFF RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Oracle Linux 10 AMD') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -216,14 +263,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PPG_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PPG_REPO, COMPONENT) } } diff --git a/ppg/ydiff.yml b/ppg/ydiff.yml index c32229aac9..6074e5d0e5 100644 --- a/ppg/ydiff.yml +++ b/ppg/ydiff.yml @@ -1,5 +1,5 @@ - job: - name: ydiff-RELEASE + name: hetzner-ydiff-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ppg/ydiff.groovy diff --git a/proxysql/build-binary-proxysql b/proxysql/build-binary-proxysql index a751cb0e81..dcf0ccb766 100755 --- a/proxysql/build-binary-proxysql +++ b/proxysql/build-binary-proxysql @@ -24,6 +24,7 @@ if [ -f /usr/bin/yum ]; then sudo cmake --version fi sudo yum -y install wget + sudo yum -y install libuuid-devel if [[ $RHEL -eq 7 ]]; then sudo yum -y install automake bzip2 cmake3 make gcc-c++ gcc git openssl openssl-devel gnutls gnutls-devel libtool patch libuuid libuuid-devel perl-IPC-Cmd if [ -f /usr/bin/cmake3 ]; then @@ -33,6 +34,7 @@ if [ -f /usr/bin/yum ]; then fi elif [ -f /usr/bin/apt ]; then DEBIAN_VERSION=$(lsb_release -sc) + sudo apt-get -y update if [ ${DEBIAN_VERSION} = "stretch" ]; then sudo sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list sudo sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list @@ -44,6 +46,7 @@ elif [ -f /usr/bin/apt ]; then fi if [ ${DEBIAN_VERSION} = "bionic" -o ${DEBIAN_VERSION} = "buster" -o ${DEBIAN_VERSION} = "stretch" ]; then sudo wget https://github.com/Kitware/CMake/releases/download/v3.19.8/cmake-3.19.8.tar.gz + sudo wget --no-check-certificate https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz sudo tar -zxvf cmake-3.19.8.tar.gz cd cmake-3.19.8/ sudo ./bootstrap @@ -56,8 +59,8 @@ fi # We need to use golang version >= 1.21. Downloading golang from tarball to ensure each platform uses the same version. sudo rm -rf /usr/bin/go -sudo wget https://go.dev/dl/go1.22.7.linux-amd64.tar.gz -sudo tar -C /usr/local/ -xzf go1.22.7.linux-amd64.tar.gz +sudo wget https://go.dev/dl/go1.23.11.linux-amd64.tar.gz +sudo tar -C /usr/local/ -xzf go1.23.11.linux-amd64.tar.gz sudo ln -s /usr/local/go/bin/go /usr/bin/go go version diff --git a/proxysql/proxysql.groovy b/proxysql/proxysql.groovy index 388bbd908a..8999e54eb0 100644 --- a/proxysql/proxysql.groovy +++ b/proxysql/proxysql.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -30,9 +30,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/proxysql-packaging.git', description: 'URL for proxysql packaging repository', @@ -85,10 +89,13 @@ pipeline { } stages { stage('Create PROXYSQL source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } steps { // slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("centos:7", "--get_sources=1") + buildStage("oraclelinux:8", "--get_sources=1") sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/proxysql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -101,36 +108,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PROXYSQL generic source packages') { parallel { stage('Build PROXYSQL generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("centos:7", "--build_src_rpm=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PROXYSQL generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -139,229 +146,324 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oracle Linux 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Focal tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } + stage('Ubuntu Noble tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_tarball=1") + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } } } stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PROXYSQL_DEST_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PROXYSQL_DEST_REPO, COMPONENT) } } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + sudo systemctl restart docker + git clone https://github.com/percona/percona-docker + cd percona-docker/proxysql + sed -i "s/ENV PROXYSQL_VERSION.*/ENV PROXYSQL_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/enable proxysql testing/enable proxysql ${COMPONENT}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/proxysql2:${VERSION}-${RPM_RELEASE} . + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/proxysql2:${VERSION}-${RPM_RELEASE} + + PROXYSQL_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PROXYSQL_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PROXYSQL_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + + docker tag perconalab/proxysql2:${VERSION}-${RPM_RELEASE} perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION}.${PROXYSQL_PATCH_VERSION} + docker tag perconalab/proxysql2:${VERSION}-${RPM_RELEASE} perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION} + docker tag perconalab/proxysql2:${VERSION}-${RPM_RELEASE} perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION} + + sudo docker push perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION}.${PROXYSQL_PATCH_VERSION} + sudo docker push perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION} + sudo docker push perconalab/proxysql2:${PROXYSQL_MAJOR_VERSION} + + ''' + } + } + } + } } post { success { diff --git a/proxysql/proxysql.yml b/proxysql/proxysql.yml index 0db3f93ef5..15508eadaa 100644 --- a/proxysql/proxysql.yml +++ b/proxysql/proxysql.yml @@ -1,5 +1,5 @@ - job: - name: proxysql-RELEASE + name: hetzner-proxysql-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: proxysql/proxysql.groovy diff --git a/proxysql/proxysql3.groovy b/proxysql/proxysql3.groovy new file mode 100644 index 0000000000..8d316ed43d --- /dev/null +++ b/proxysql/proxysql3.groovy @@ -0,0 +1,413 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + sh """ + set -o xtrace + mkdir test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/scripts/proxysql_builder.sh -O proxysql_builder.sh + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + sed -i "s/^RPM_RELEASE=.*/RPM_RELEASE=${RPM_RELEASE}/g" proxysql_builder.sh + sed -i "s/^DEB_RELEASE=.*/DEB_RELEASE=${DEB_RELEASE}/g" proxysql_builder.sh + bash -x ./proxysql_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./proxysql_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --proxysql_repo=${PROXYSQL_REPO} --pat_repo=${PAT_REPO} --pat_tag=${PAT_TAG} --proxysql_branch=${PROXYSQL_BRANCH} --proxysql_ver=${VERSION} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/proxysql-packaging.git', + description: 'URL for proxysql packaging repository', + name: 'GIT_REPO') + string( + defaultValue: 'v3.0', + description: 'Tag/Branch for proxysql packaging repository', + name: 'GIT_BRANCH') + string( + defaultValue: 'https://github.com/sysown/proxysql.git', + description: 'URL for proxysql repository', + name: 'PROXYSQL_REPO') + string( + defaultValue: 'v3.0.1', + description: 'Tag/Branch for proxysql repository', + name: 'PROXYSQL_BRANCH') + string( + defaultValue: '3.0.1', + description: 'PROXYSQL release value', + name: 'VERSION') + string( + defaultValue: 'https://github.com/percona/proxysql-admin-tool.git', + description: 'URL for proxysql-admin-tool repository', + name: 'PAT_REPO') + string( + defaultValue: 'v2', + description: 'Tag/Branch for proxysql-admin-tool repository', + name: 'PAT_TAG') + string( + defaultValue: '1.1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1.1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: 'proxysql', + description: 'PROXYSQL repo name', + name: 'PROXYSQL_DEST_REPO') + choice( + choices: 'testing\nexperimental\nlaboratory', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Create PROXYSQL source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + // slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + buildStage("oraclelinux:9", "--get_sources=1") + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/proxysql.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/proxysql.properties + cat uploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PROXYSQL generic source packages') { + parallel { + stage('Build PROXYSQL generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_src_rpm=1") + + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PROXYSQL generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_src_deb=1") + + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PROXYSQL RPMs/DEBs/Binary tarballs') { + parallel { + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9 tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_tarball=1") + + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Ubuntu Jammy tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_tarball=1") + + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Ubuntu Noble tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_tarball=1") + + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + + stage('Sign packages') { + steps { + signRPM(params.CLOUD) + signDEB(params.CLOUD) + } + } + stage('Push to public repository') { + steps { + // sync packages + sync2ProdAutoBuild(params.CLOUD, PROXYSQL_DEST_REPO, COMPONENT) + } + } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + sudo systemctl restart docker + git clone https://github.com/percona/percona-docker + cd percona-docker/proxysql + sed -i "s/ENV PROXYSQL_VERSION.*/ENV PROXYSQL_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/enable proxysql testing/enable proxysql ${COMPONENT}/g" Dockerfile + sed -i "s/proxysql2/proxysql3/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/proxysql3:${VERSION}-${RPM_RELEASE} . + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/proxysql3:${VERSION}-${RPM_RELEASE} + + PROXYSQL_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PROXYSQL_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PROXYSQL_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + + docker tag perconalab/proxysql3:${VERSION}-${RPM_RELEASE} perconalab/proxysql3:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION}.${PROXYSQL_PATCH_VERSION} + docker tag perconalab/proxysql3:${VERSION}-${RPM_RELEASE} perconalab/proxysql3:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION} + docker tag perconalab/proxysql3:${VERSION}-${RPM_RELEASE} perconalab/proxysql3:${PROXYSQL_MAJOR_VERSION} + + sudo docker push perconalab/proxysql3:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION}.${PROXYSQL_PATCH_VERSION} + sudo docker push perconalab/proxysql3:${PROXYSQL_MAJOR_VERSION}.${PROXYSQL_MINOR_VERSION} + sudo docker push perconalab/proxysql3:${PROXYSQL_MAJOR_VERSION} + + ''' + } + } + } + } + } + post { + success { + // slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${PROXYSQL_BRANCH} + ${PAT_TAG} - [${BUILD_URL}]" + } + deleteDir() + } + failure { + // slackNotify("", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${PROXYSQL_BRANCH} + ${PAT_TAG} - [${AWS_STASH_PATH}]" + } + deleteDir() + } + } +} diff --git a/proxysql/proxysql3.yml b/proxysql/proxysql3.yml new file mode 100644 index 0000000000..055d43bb2d --- /dev/null +++ b/proxysql/proxysql3.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-proxysql-v3-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: proxysql/proxysql3.groovy + diff --git a/proxysql/qa-proxysql2-param.yml b/proxysql/qa-proxysql2-param.yml index 43aa3fa09a..97dd6c1e27 100644 --- a/proxysql/qa-proxysql2-param.yml +++ b/proxysql/qa-proxysql2-param.yml @@ -82,12 +82,10 @@ type: user-defined name: DOCKER_OS values: - - centos:7 - centos:8 - ubuntu:jammy - ubuntu:focal - ubuntu:noble - - debian:buster - debian:bullseye - debian:bookworm builders: diff --git a/proxysql/qa-proxysql3-param.yml b/proxysql/qa-proxysql3-param.yml new file mode 100644 index 0000000000..7be11b3205 --- /dev/null +++ b/proxysql/qa-proxysql3-param.yml @@ -0,0 +1,115 @@ +- job: + name: qa-proxysql3-param + project-type: matrix + defaults: global + description: | + Do not edit this job through the web! + disabled: false + concurrent: true + auth-token: proxysql2param + node: micro-amazon + properties: + - build-discarder: + days-to-keep: -1 + num-to-keep: 50 + artifact-days-to-keep: -1 + artifact-num-to-keep: 50 + parameters: + - string: + name: GIT_REPO + default: https://github.com/sysown/proxysql + description: URL to ProxySQL repository + - string: + name: BRANCH + default: "v3.0.1" + description: Tag/Branch for ProxySQL repository + - string: + name: PROXYSQL_PACKAGE_REPO + default: https://github.com/percona/proxysql-packaging + description: URL to ProxySQL pacakge repository + - string: + name: PROXYSQL_PACKAGE_BRANCH + default: v3.0 + description: Tag/Branch for ProxySQL pacakge repository + - string: + name: PXC57_REPO + default: "https://github.com/percona/percona-xtradb-cluster" + description: URL to PXC repository + - string: + name: PXC57_BRANCH + default: "5.7" + description: Tag/Branch for PXC repository + - string: + name: PXB24_REPO + default: https://github.com/percona/percona-xtrabackup + description: URL to PXB24 repository + - string: + name: PXB24_BRANCH + default: percona-xtrabackup-2.4.29 + description: Tag/Branch for PXC repository + - string: + name: PAT_TAG + default: v3 + description: Tag/Branch for ProxySQL-admin-tool repository + - choice: + name: PXC_VERSION + choices: + - PXC57 + - PXC80 + - PXC84 + description: PXC version to test proxysql-admin suite + - choice: + name: JOB_CMAKE + choices: + - /usr/bin/cmake + description: path to cmake binary + - string: + name: CMAKE_OPTS + default: + description: cmake options + - string: + name: MAKE_OPTS + default: + description: make options, like VERBOSE=1 + axes: + - axis: + type: user-defined + name: CMAKE_BUILD_TYPE + values: + - RelWithDebInfo + - Debug + - axis: + type: user-defined + name: DOCKER_OS + values: + - oraclelinux:9 + - ubuntu:jammy + - ubuntu:noble + - debian:bookworm + builders: + - trigger-builds: + - project: qa-proxysql3-pipeline + current-parameters: true + predefined-parameters: | + DOCKER_OS=${DOCKER_OS} + CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + block: true + block-thresholds: + build-step-failure-threshold: FAILURE + unstable-threshold: never + failure-threshold: FAILURE + - shell: | + sudo find . -name "*.xml" -o -name "*.log" -delete + - copyartifact: + project: qa-proxysql3-pipeline + which-build: specific-build + build-number: "${TRIGGERED_BUILD_NUMBERS_qa_proxysql3_pipeline}" + do-not-fingerprint: true + - shell: | + echo "${TRIGGERED_BUILD_NUMBERS_qa_proxysql3_pipeline}" > PIPELINE_BUILD_NUMBER + publishers: + - junit: + results: "**/*.xml" + keep-long-stdio: true + - archive: + artifacts: 'proxysql/sources/proxysql/results/*.tar.gz' diff --git a/proxysql/qa-proxysql3-pipeline.groovy b/proxysql/qa-proxysql3-pipeline.groovy new file mode 100644 index 0000000000..bd3be4dbc8 --- /dev/null +++ b/proxysql/qa-proxysql3-pipeline.groovy @@ -0,0 +1,145 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/percona-lab/jenkins-pipelines.git' +]) + +pipeline_timeout = 10 + +pipeline { + parameters { + choice( + choices: [ 'AWS','Hetzner' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/sysown/proxysql', + description: 'URL to ProxySQL repository', + name: 'GIT_REPO', + trim: true) + string( + defaultValue: 'v3.0.1', + description: 'Tag/Branch for ProxySQL repository', + name: 'BRANCH', + trim: true) + string( + defaultValue: 'https://github.com/percona/proxysql-packaging', + description: 'URL to ProxySQL package repository', + name: 'PROXYSQL_PACKAGE_REPO', + trim: true) + string( + defaultValue: 'v3.0', + description: 'Tag/Branch for ProxySQL package repository', + name: 'PROXYSQL_PACKAGE_BRANCH', + trim: true) + string( + defaultValue: 'v3', + description: 'Tag/Branch for ProxySQL-admin-tool repository', + name: 'PAT_TAG', + trim: true) + choice( + choices: 'PXC84\nPXC80\nPXC57', + description: 'PXC version to test proxysql-admin suite', + name: 'PXC_VERSION') + choice( + choices: 'oraclelinux:9\nubuntu:noble\nubuntu:jammy\ndebian:bookworm', + description: 'OS version for compilation', + name: 'DOCKER_OS') + choice( + choices: '/usr/bin/cmake', + description: 'path to cmake binary', + name: 'JOB_CMAKE') + choice( + choices: 'RelWithDebInfo\nDebug', + description: 'Type of build to produce', + name: 'CMAKE_BUILD_TYPE') + string( + defaultValue: '', + description: 'cmake options', + name: 'CMAKE_OPTS') + string( + defaultValue: '', + description: 'make options, like VERBOSE=1', + name: 'MAKE_OPTS') + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'micro-amazon' + } + options { + skipDefaultCheckout() + skipStagesAfterUnstable() + timeout(time: 6, unit: 'DAYS') + buildDiscarder(logRotator(numToKeepStr: '200', artifactNumToKeepStr: '200')) + } + stages { + stage('Build ProxySQL') { + agent { label 'docker' } + steps { + script { + currentBuild.displayName = "#${BUILD_NUMBER}-${params.BRANCH}-${params.DOCKER_OS}-${params.CMAKE_BUILD_TYPE}-${params.PXC_VERSION}" + } + git branch: 'hetzner', url: 'https://github.com/percona-lab/jenkins-pipelines' + echo 'Checkout ProxySQL sources' + sh ''' + # sudo is needed for better node recovery after compilation failure + # if building failed on compilation stage directory will have files owned by docker user + sudo git reset --hard + sudo git clean -xdf + sudo rm -rf sources + ./proxysql/checkout PROXYSQL + ''' + + echo 'Build ProxySQL' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'c42456e5-c28d-4962-b32c-b75d161bff27', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh ''' + aws ecr-public get-login-password --region us-east-1 | docker login -u AWS --password-stdin public.ecr.aws/e7j3v3n0 + sg docker -c " + if [ \$(docker ps -q | wc -l) -ne 0 ]; then + docker ps -q | xargs docker stop --time 1 || : + fi + ./proxysql/run-build-proxysql ${DOCKER_OS} + " 2>&1 | tee build.log + + if [[ -f \$(ls ./proxysql/sources/proxysql/results/*.tar.gz | head -1) ]]; then + until aws s3 cp --no-progress --acl public-read proxysql/sources/proxysql/results/*.tar.gz s3://pxc-build-cache/${BUILD_TAG}/proxysql-${BRANCH}.tar.gz; do + sleep 5 + done + else + echo cannot find compiled archive + exit 1 + fi + ''' + } + } + } + stage('Test ProxySQL') { + agent { label 'docker' } + steps { + git branch: 'hetzner', url: 'https://github.com/percona-lab/jenkins-pipelines' + echo 'Test ProxySQL' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'c42456e5-c28d-4962-b32c-b75d161bff27', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh ''' + until aws s3 cp --no-progress s3://pxc-build-cache/${BUILD_TAG}/proxysql-${BRANCH}.tar.gz ./proxysql/sources/proxysql/results/proxysql-${BRANCH}.tar.gz; do + sleep 5 + done + aws ecr-public get-login-password --region us-east-1 | docker login -u AWS --password-stdin public.ecr.aws/e7j3v3n0 + sg docker -c " + if [ \$(docker ps -q | wc -l) -ne 0 ]; then + docker ps -q | xargs docker stop --time 1 || : + fi + ./proxysql/run-test-proxysql ${DOCKER_OS} + " + ''' + } + step([$class: 'JUnitResultArchiver', testResults: 'proxysql/sources/proxysql/results/*.xml', healthScaleFactor: 1.0]) + archiveArtifacts 'proxysql/sources/proxysql/results/*.output,proxysql/sources/proxysql/results/*.xml,proxysql/sources/proxysql/results/qa_proxysql_logs.tar.gz' + } + } + } + post { + always { + sh ''' + echo Finish: \$(date -u "+%s") + ''' + } + } +} diff --git a/proxysql/qa-proxysql3-pipeline.yml b/proxysql/qa-proxysql3-pipeline.yml new file mode 100644 index 0000000000..e8c1cbfa40 --- /dev/null +++ b/proxysql/qa-proxysql3-pipeline.yml @@ -0,0 +1,16 @@ +- job: + name: qa-proxysql3-pipeline + project-type: pipeline + description: | + Do not edit this job through the web! + concurrent: true + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'master' + wipe-workspace: false + lightweight-checkout: true + script-path: proxysql/qa-proxysql3-pipeline.groovy + diff --git a/proxysql/test-proxysql b/proxysql/test-proxysql index 9c73fb5f4f..7c23314ce9 100755 --- a/proxysql/test-proxysql +++ b/proxysql/test-proxysql @@ -74,7 +74,7 @@ function get_glibc_version { "oraclelinux:8" | "ol:8") echo "2.28" ;; - "oraclelinux:9" | "ol:9") + "oraclelinux:9" | "ol:9" | "ol:9.6") echo "2.34" ;; "ubuntu:focal") @@ -126,6 +126,7 @@ check_percona_release() { # Function to install percona-release on Debian-based systems install_debian() { echo "Installing percona-release for Debian-based system..." + sudo apt-get update -y sudo curl -O https://repo.percona.com/apt/percona-release_latest.generic_all.deb sudo apt install -y gnupg2 lsb-release ./percona-release_latest.generic_all.deb } @@ -164,21 +165,34 @@ if [ -f /usr/bin/apt ]; then install_package wget gnupg2 lsb-release curl fi -# Installing proxysql 2.0.latest package +# Installing proxysql 3.0.latest package enable_repo proxysql testing -install_package proxysql2 +install_package proxysql3 if [[ "${PXC_VERSION}" == "PXC84" ]]; then + # Fetch PXC versions + LATEST_VERSION_84=$(git ls-remote --refs --sort='version:refname' --tags https://github.com/percona/percona-xtradb-cluster | \ + grep 'Percona-XtraDB-Cluster-8.4' | tail -n1 | cut -d '/' -f3 | cut -d '-' -f4) + VERSION_SUFFIX_84=$(git ls-remote --refs --sort='version:refname' --tags https://github.com/percona/percona-xtradb-cluster | \ + grep 'Percona-XtraDB-Cluster-8.4' | tail -n1 | cut -d '/' -f3 | cut -d '-' -f5) + # Downloading PXC-8.4 tarball package - sudo wget -q -O ${WORKDIR_ABS}/WORKDIR/Percona-XtraDB-Cluster_8.4.0-1.1_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-84/Percona-XtraDB-Cluster-8.4.0/binary/tarball/Percona-XtraDB-Cluster_8.4.0-1.1_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz + sudo wget -q -O ${WORKDIR_ABS}/WORKDIR/Percona-XtraDB-Cluster_${LATEST_VERSION_84}_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz\ + http://downloads.percona.com/downloads/Percona-XtraDB-Cluster-84/Percona-XtraDB-Cluster-${LATEST_VERSION_84}/binary/tarball/Percona-XtraDB-Cluster_${LATEST_VERSION_84}-${VERSION_SUFFIX_84}_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz # Installing PXC-8.4 client package enable_repo pxc-84-lts release install_package percona-xtradb-cluster-client elif [[ "${PXC_VERSION}" == "PXC80" ]]; then + # Fetch PXC versions + LATEST_VERSION_80=$(git ls-remote --refs --sort='version:refname' --tags https://github.com/percona/percona-xtradb-cluster | \ + grep 'Percona-XtraDB-Cluster-8.0' | tail -n1 | cut -d '/' -f3 | cut -d '-' -f4) + VERSION_SUFFIX_80=$(git ls-remote --refs --sort='version:refname' --tags https://github.com/percona/percona-xtradb-cluster | \ + grep 'Percona-XtraDB-Cluster-8.0' | tail -n1 | cut -d '/' -f3 | cut -d '-' -f5) # Downloading PXC-8.0 tarball package - sudo wget -q -O ${WORKDIR_ABS}/WORKDIR/Percona-XtraDB-Cluster_8.0.36-28.1_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz https://downloads.percona.com/downloads/Percona-XtraDB-Cluster-80/Percona-XtraDB-Cluster-8.0.36/binary/tarball/Percona-XtraDB-Cluster_8.0.36-28.1_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz + sudo wget -q -O ${WORKDIR_ABS}/WORKDIR/Percona-XtraDB-Cluster_${LATEST_VERSION_80}_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz \ + http://downloads.percona.com/downloads/Percona-XtraDB-Cluster-80/Percona-XtraDB-Cluster-${LATEST_VERSION_80}/binary/tarball/Percona-XtraDB-Cluster_${LATEST_VERSION_80}-${VERSION_SUFFIX_80}_Linux.x86_64.glibc${glibc_version}-minimal.tar.gz # Installing PXC-8.0 client package enable_repo pxc-80 release @@ -191,9 +205,8 @@ elif [[ "${PXC_VERSION}" == "PXC57" ]]; then # Installing PXC-5.7 client enable_repo pxc-57 release install_package percona-xtradb-cluster-client-5.7 - # Installing PXB-2.4 (only required in case of PXC-5.7) - enable_repo pxb-24 release + enable_repo pxb-24 testing install_package percona-xtrabackup-24 fi @@ -217,14 +230,14 @@ for X in $(seq 0 $COPY_TIMEOUT); do sleep 1 done -for file in proxysql-admin proxysql-admin-common proxysql-login-file proxysql-common percona-scheduler-admin; do +for file in proxysql-admin proxysql-admin-common proxysql-login-file proxysql-common percona-scheduler-admin proxysql; do sudo ln -s $WORKDIR_ABS/PROXY_ADMIN/proxysql*/usr/bin/$file ${WORKDIR_ABS}/WORKDIR/$file done; sudo ln -s $WORKDIR_ABS/PROXY_ADMIN/proxysql*/etc/proxysql-admin.cnf ${WORKDIR_ABS}/WORKDIR/proxysql-2.0/etc/proxysql-admin.cnf sudo ln -s $WORKDIR_ABS/PROXY_ADMIN/proxysql*/usr/bin/pxc_scheduler_handler ${WORKDIR_ABS}/WORKDIR/pxc_scheduler_handler sudo ln -s $WORKDIR_ABS/PROXY_ADMIN/proxysql*/etc/config.toml ${WORKDIR_ABS}/WORKDIR/proxysql-2.0/etc/config.toml sudo ln -s $WORKDIR_ABS/PROXY_ADMIN/proxysql*/tests/testsuite.toml ${WORKDIR_ABS}/WORKDIR/testsuite.toml -sudo ln -s /usr/bin/proxysql $WORKDIR_ABS/WORKDIR/proxysql-2.0/usr/bin/proxysql +sudo ln -s $WORKDIR_ABS/PROXY_ADMIN/proxysql*/usr/bin/proxysql ${WORKDIR_ABS}/WORKDIR/proxysql-2.0/usr/bin/proxysql status=0 # Running ProxySQL QA script diff --git a/ps/jenkins/get-ps-branches-9.x.groovy b/ps/jenkins/get-ps-branches-9.x.groovy index ee1e62ff28..91196a7674 100644 --- a/ps/jenkins/get-ps-branches-9.x.groovy +++ b/ps/jenkins/get-ps-branches-9.x.groovy @@ -81,7 +81,7 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'ps9.0-RELEASE', parameters: [string(name: 'BRANCH', value: BRANCH_NAME), string(name: 'COMPONENT', value: 'testing')] + build job: 'ps9.0-RELEASE', parameters: [string(name: 'BRANCH', value: BRANCH_NAME), string(name: 'COMPONENT', value: 'experimental')] } } diff --git a/ps/jenkins/jemalloc.groovy b/ps/jenkins/jemalloc.groovy index 84386328a2..1e4a319cf6 100644 --- a/ps/jenkins/jemalloc.groovy +++ b/ps/jenkins/jemalloc.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -112,10 +112,12 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { DEBIAN_FRONTEND=noninteractive apt-get -y purge eatmydata || true if [ \\\$DEBIAN_VERSION = focal -o \\\$DEBIAN_VERSION = bullseye -o \\\$DEBIAN_VERSION = jammy -o \\\$DEBIAN_VERSION = noble ]; then PKGLIST=\\"gcc-9\\" + elif [ \\\$DEBIAN_VERSION = trixie ]; then + PKGLIST=\\"gcc-13\\" else PKGLIST=\\"gcc-11\\" fi - if [ \\\$DEBIAN_VERSION = focal -o \\\$DEBIAN_VERSION = bullseye -o \\\$DEBIAN_VERSION = jammy -o \\\$DEBIAN_VERSION = bookworm -o \\\$DEBIAN_VERSION = noble ]; then + if [ \\\$DEBIAN_VERSION = focal -o \\\$DEBIAN_VERSION = bullseye -o \\\$DEBIAN_VERSION = jammy -o \\\$DEBIAN_VERSION = bookworm -o \\\$DEBIAN_VERSION = noble -o \\\$DEBIAN_VERSION = trixie ]; then PKGLIST=\\"\\\${PKGLIST} python3-mysqldb\\" else PKGLIST=\\"\\\${PKGLIST} python-mysqldb\\" @@ -139,8 +141,10 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { dpkg-source -x jemalloc_3.6.0-2.dsc cd jemalloc-3.6.0 sed -i 's/@EXTRA_LDFLAGS@/@EXTRA_LDFLAGS@ -Wl,--allow-multiple-definition/g' Makefile.in - if [ \\\$DEBIAN_VERSION = bookworm -a \\\$ARCH = aarch64 ]; then - sed -i 's/make check/#make check/g' debian/rules + if [ \\\$DEBIAN_VERSION = bookworm -o \\\$DEBIAN_VERSION = trixie ]; then + if [ \\\$ARCH = aarch64 ]; then + sed -i 's/make check/#make check/g' debian/rules + fi fi sed -i 's/override_dh_auto_test:/override_dh_builddeb:\\n\\tdh_builddeb -- -Zgzip\n\noverride_dh_auto_test:/g' debian/rules cat debian/rules @@ -166,9 +170,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) /* string( defaultValue: 'https://github.com/percona-lab/jemalloc-packaging.git', @@ -226,19 +234,19 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build jemalloc packages') { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) sh ''' echo "============>" ''' @@ -250,19 +258,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -271,19 +279,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "RPM") sh ''' pwd @@ -292,19 +300,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "RPM") sh ''' pwd @@ -313,19 +321,61 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "RPM") + sh ''' + pwd + ls -la test/rpm + cp -r test/srpm . + cp -r test/rpm . + ''' + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "RPM") + sh ''' + pwd + ls -la test/rpm + cp -r test/srpm . + cp -r test/rpm . + ''' + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "DEB") sh ''' pwd @@ -333,17 +383,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "DEB") sh ''' pwd @@ -351,17 +401,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "DEB") sh ''' pwd @@ -369,17 +419,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "DEB") sh ''' pwd @@ -387,17 +437,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "DEB") sh ''' pwd @@ -405,17 +455,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "DEB") sh ''' pwd @@ -423,17 +473,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "DEB") sh ''' pwd @@ -441,17 +491,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "DEB") sh ''' pwd @@ -459,17 +509,53 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie (13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:trixie", "DEB") + sh ''' + pwd + ls -la test/deb + cp -r test/deb . + ''' + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie (13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:trixie", "DEB") + sh ''' + pwd + ls -la test/deb + cp -r test/deb . + ''' + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "DEB") sh ''' pwd @@ -477,17 +563,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "DEB") sh ''' pwd @@ -495,8 +581,8 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -510,7 +596,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild('tools', COMPONENT) + sync2ProdAutoBuild(params.CLOUD, 'tools', COMPONENT) } } diff --git a/ps/jenkins/jemalloc.yml b/ps/jenkins/jemalloc.yml index 1650207e9c..09fd8f6f31 100644 --- a/ps/jenkins/jemalloc.yml +++ b/ps/jenkins/jemalloc.yml @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/jemalloc.groovy diff --git a/ps/jenkins/mysql-shell.groovy b/ps/jenkins/mysql-shell.groovy index 1c827f30ee..a588cef073 100644 --- a/ps/jenkins/mysql-shell.groovy +++ b/ps/jenkins/mysql-shell.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,21 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + environment { + PS_MAJOR_RELEASE = sh( + returnStdout: true, + script: ''' + echo ${PS_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}' + ''' + ).trim() } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/mysql-shell-packaging.git', description: 'URL for mysql-shell packaging repository', @@ -98,36 +110,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build MYSQL-SHELL generic source packages') { parallel { stage('Build MYSQL-SHELL generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build MYSQL-SHELL generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -137,182 +149,212 @@ pipeline { /* stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } */ stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + when { + expression { true } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + when { + expression { true } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) ARM') { + when { + expression { env.PS_MAJOR_RELEASE == "80" } + } agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { - PS_MAJOR_RELEASE = sh(returnStdout: true, script: ''' echo ${PS_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}' ''').trim() - if ("${PS_MAJOR_RELEASE}" == "80") { - cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - } else { - echo "The step is skipped." - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy (22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { PS_MAJOR_RELEASE = sh(returnStdout: true, script: ''' echo ${PS_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}' ''').trim() if ("${PS_MAJOR_RELEASE}" == "80") { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } else { echo "The step is skipped." } @@ -321,67 +363,67 @@ pipeline { } stage('Debian Bookworm (12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Oracle Linux 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Debian Bullseye (11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } @@ -396,7 +438,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild('mysql-shell', COMPONENT) + sync2ProdAutoBuild(params.CLOUD, 'mysql-shell', COMPONENT) } } @@ -404,9 +446,6 @@ pipeline { post { success { // slackNotify("", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") - script { - currentBuild.description = "Built on ${SHELL_BRANCH}, path to packages: experimental/${AWS_STASH_PATH}" - } deleteDir() } failure { @@ -417,6 +456,9 @@ pipeline { sh ''' sudo rm -rf ./* ''' + script { + currentBuild.description = "Built on SHELL ${SHELL_BRANCH}, PS ${PS_BRANCH} path to packages: experimental/${AWS_STASH_PATH}" + } deleteDir() } } diff --git a/ps/jenkins/mysql-shell.yml b/ps/jenkins/mysql-shell.yml index 1789900126..8167530dcd 100644 --- a/ps/jenkins/mysql-shell.yml +++ b/ps/jenkins/mysql-shell.yml @@ -1,5 +1,5 @@ - job: - name: mysql-shell-8.0-RELEASE + name: hetzner-mysql-shell-8.0-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/mysql-shell.groovy diff --git a/ps/jenkins/percona-server-for-mysql-5.7.groovy b/ps/jenkins/percona-server-for-mysql-5.7.groovy index e83ed7c1e3..d0779c7669 100644 --- a/ps/jenkins/percona-server-for-mysql-5.7.groovy +++ b/ps/jenkins/percona-server-for-mysql-5.7.groovy @@ -1,24 +1,34 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void installCli(String PLATFORM) { sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip - elif [ ${PLATFORM} = "rpm" ]; then - sudo yum -y install wget curl unzip - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi + fi + sudo yum -y install wget curl unzip + fi + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi """ } @@ -43,16 +53,28 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { mkdir -p test wget --header="Authorization: token ${TOKEN}" --header="Accept: application/vnd.github.v3.raw" -O ps_builder.sh \$(echo ${GIT_REPO} | sed -re 's|github.com|api.github.com/repos|; s|\\.git\$||')/contents/build-ps/percona-server-5.7_builder.sh?ref=${BRANCH} sed -i "s|git clone \\\"\\\$REPO\\\"|git clone \$(echo ${GIT_REPO}| sed -re 's|github.com|${TOKEN}@github.com|') percona-server|g" ps_builder.sh - grep "git clone" ps_builder.sh pwd -P export build_dir=\$(pwd -P) set -o xtrace - cd \${build_dir} - if [ -f ./test/percona-server-5.7.properties ]; then - . ./test/percona-server-5.7.properties + if [ "$DOCKER_OS" = "none" ]; then + cd \${build_dir} + if [ -f ./test/percona-server-5.7.properties ]; then + . ./test/percona-server-5.7.properties + fi + sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + else + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + export build_dir=\$(pwd -P) + set -o xtrace + cd \${build_dir} + if [ -f ./test/percona-server-5.7.properties ]; then + . ./test/percona-server-5.7.properties + fi + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + " fi - sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} """ } } @@ -67,9 +89,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server-private.git', description: 'github repository for build', name: 'GIT_REPO') string(defaultValue: 'release-5.7.44-52', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '0', description: 'PerconaFT repository', name: 'PERCONAFT_REPO') @@ -96,7 +122,7 @@ parameters { stages { stage('Create PS source tarball') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -117,40 +143,40 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-5.7.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { parallel { stage('Build PS generic source rpm') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PS generic source deb') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -159,91 +185,91 @@ parameters { parallel { stage('Centos 7') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-8-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-9-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Bionic(18.04)') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-focal-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-noble-x64' } steps { script { @@ -253,304 +279,304 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Buster(10)') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-buster-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bullseye-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bookworm-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Centos 7 debug tarball') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-centos-7-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Centos 8 tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-8-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:8", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Centos 8 debug tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-8-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:8", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-9-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 debug tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-ol-9-x64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Bionic tarball') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Bionic debug tarball') { agent { - label 'min-bionic-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bionic-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-focal-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal debug tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-focal-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy debug tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Buster tarball') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-buster-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Buster debug tarball') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-buster-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bullseye tarball') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bullseye-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bullseye debug tarball') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bullseye-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm tarball') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bookworm-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm debug tarball') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-bookworm-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--debug=1 --build_tarball=1 ") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Upload packages and tarballs from S3') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'min-jammy-x64' } steps { cleanUpWS() installCli("deb") unstash 'properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Sign packages') { @@ -559,9 +585,15 @@ parameters { signDEB() } } + stage('Push to public repository') { + steps { + // sync packages + sync2PrivateProdAutoBuild(params.CLOUD, "ps-57-eol", COMPONENT) + } + } stage('Build docker container') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'min-buster-x64' } steps { script { @@ -579,11 +611,23 @@ parameters { PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 4)}') PS_MAJOR_MINOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 7)}' | sed "s/-//g") + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common - sudo apt-get install -y docker.io - sudo systemctl status docker - sudo apt-get install -y qemu binfmt-support qemu-user-static + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo lscpu | grep -q 'sse4_2' && grep -q 'popcnt' /proc/cpuinfo && echo "Supports x86-64-v2" || echo "Does NOT support x86-64-v2" sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker cd percona-docker/percona-server-5.7 mv /tmp/*.rpm . @@ -595,6 +639,8 @@ parameters { sudo docker tag percona/percona-server:${PS_RELEASE}.${RPM_RELEASE} percona/percona-server:${PS_MAJOR_MINOR_RELEASE} sudo docker images sudo docker save -o percona-server-${PS_RELEASE}-${RPM_RELEASE}.docker.tar percona/percona-server:${PS_RELEASE}.${RPM_RELEASE} percona/percona-server:${PS_RELEASE} percona/percona-server:${PS_MAJOR_RELEASE} percona/percona-server:${PS_MAJOR_MINOR_RELEASE} + sudo addgroup admin + sudo useradd -m -s /bin/bash -g admin -G admin admin sudo chown admin:admin percona-server-${PS_RELEASE}-${RPM_RELEASE}.docker.tar sudo chmod a+r percona-server-${PS_RELEASE}-${RPM_RELEASE}.docker.tar ls -la @@ -609,18 +655,11 @@ parameters { } } } - stage('Push to public repository') { - steps { - // sync packages - //sync2ProdAutoBuild('ps-57', COMPONENT) - sync2PrivateProdAutoBuild("ps-57-eol", COMPONENT) - } - } stage('Push Tarballs to TESTING download area') { steps { script { try { - uploadTarballToDownloadsTesting("ps-gated", "${BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "ps-gated", "${BRANCH}") } catch (err) { echo "Caught: ${err}" @@ -632,11 +671,8 @@ parameters { } post { success { - slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") unstash 'properties' - script { - currentBuild.description = "Built on ${BRANCH}; path to packages: ${COMPONENT}/${AWS_STASH_PATH}" - } deleteDir() } failure { @@ -647,6 +683,9 @@ parameters { sh ''' sudo rm -rf ./* ''' + script { + currentBuild.description = "Built on ${BRANCH}; path to packages: ${COMPONENT}/${AWS_STASH_PATH}" + } deleteDir() } } diff --git a/ps/jenkins/percona-server-for-mysql-5.7.yml b/ps/jenkins/percona-server-for-mysql-5.7.yml index 833a8546c1..4b36362e39 100644 --- a/ps/jenkins/percona-server-for-mysql-5.7.yml +++ b/ps/jenkins/percona-server-for-mysql-5.7.yml @@ -1,5 +1,5 @@ - job: - name: ps5.7-autobuild-RELEASE + name: hetzner-ps5.7-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-5.7.groovy diff --git a/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy b/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy index fb686b0b3d..4799f1bc2b 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy +++ b/ps/jenkins/percona-server-for-mysql-8.0-arm.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -191,9 +191,13 @@ def install_mysql_shell = 'no' pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server.git', description: 'github repository for build', name: 'GIT_REPO') string(defaultValue: 'release-8.0.37-29', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') @@ -254,8 +258,8 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-8.0.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { @@ -278,8 +282,8 @@ parameters { } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } */ @@ -300,8 +304,8 @@ parameters { } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -324,7 +328,7 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("none", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -344,14 +348,14 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("none", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -364,7 +368,7 @@ parameters { popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -386,12 +390,12 @@ parameters { } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -406,13 +410,13 @@ parameters { } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } */ stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -425,14 +429,14 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -447,12 +451,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -467,12 +471,12 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -485,14 +489,14 @@ parameters { popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() @@ -507,7 +511,7 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } /* @@ -526,7 +530,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -546,7 +550,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -566,7 +570,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -586,7 +590,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -608,7 +612,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ZenFS tarball') { @@ -625,7 +629,7 @@ parameters { echo "The step is skipped" } else { buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -647,7 +651,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) tarball') { @@ -665,7 +669,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -685,7 +689,7 @@ parameters { popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -707,7 +711,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ZenFS tarball') { @@ -724,7 +728,7 @@ parameters { echo "The step is skipped" } else { buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -746,7 +750,7 @@ parameters { } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } */ @@ -761,9 +765,9 @@ parameters { installCli("deb") unstash 'properties' - // uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - // uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + // uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + // uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } diff --git a/ps/jenkins/percona-server-for-mysql-8.0-arm.yml b/ps/jenkins/percona-server-for-mysql-8.0-arm.yml index c20c049292..69ee6dfff3 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0-arm.yml +++ b/ps/jenkins/percona-server-for-mysql-8.0-arm.yml @@ -1,5 +1,5 @@ - job: - name: ps8.0-arm-RELEASE + name: hetzner-ps8.0-arm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-8.0-arm.groovy diff --git a/ps/jenkins/percona-server-for-mysql-8.0-docker.groovy b/ps/jenkins/percona-server-for-mysql-8.0-docker.groovy new file mode 100644 index 0000000000..7a5e25671f --- /dev/null +++ b/ps/jenkins/percona-server-for-mysql-8.0-docker.groovy @@ -0,0 +1,439 @@ +/* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +import groovy.transform.Field + +void installCli(String PLATFORM) { + sh """ + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi + fi + sudo yum -y install wget curl unzip + fi + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi + """ +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def installDependencies(def nodeName) { + def aptNodes = ['min-bullseye-x64', 'min-bookworm-x64', 'min-focal-x64', 'min-jammy-x64', 'min-noble-x64'] + def yumNodes = ['min-ol-8-x64', 'min-centos-7-x64', 'min-ol-9-x64', 'min-amazon-2-x64'] + try{ + if (aptNodes.contains(nodeName)) { + if(nodeName == "min-bullseye-x64" || nodeName == "min-bookworm-x64"){ + sh ''' + sudo apt-get update + sudo apt-get install -y ansible git wget + ''' + }else if(nodeName == "min-focal-x64" || nodeName == "min-jammy-x64" || nodeName == "min-noble-x64"){ + sh ''' + sudo apt-get update + sudo apt-get install -y software-properties-common + sudo apt-add-repository --yes --update ppa:ansible/ansible + sudo apt-get install -y ansible git wget + ''' + }else { + error "Node Not Listed in APT" + } + } else if (yumNodes.contains(nodeName)) { + + if(nodeName == "min-centos-7-x64" || nodeName == "min-ol-9-x64"){ + sh ''' + sudo yum install -y epel-release + sudo yum -y update + sudo yum install -y ansible git wget tar + ''' + }else if(nodeName == "min-ol-8-x64"){ + sh ''' + sudo yum install -y epel-release + sudo yum -y update + sudo yum install -y ansible-2.9.27 git wget tar + ''' + }else if(nodeName == "min-amazon-2-x64"){ + sh ''' + sudo amazon-linux-extras install epel + sudo yum -y update + sudo yum install -y ansible git wget + ''' + } + else { + error "Node Not Listed in YUM" + } + } else { + echo "Unexpected node name: ${nodeName}" + } + } catch (Exception e) { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: Server Provision for Mini Package Testing for ${nodeName} at ${BRANCH} FAILED !!") + } + +} + +def runPlaybook(def nodeName) { + + try { + def playbook = "ps_lts_innovation.yml" + def playbook_path = "package-testing/playbooks/${playbook}" + + sh ''' + set -xe + git clone --depth 1 https://github.com/Percona-QA/package-testing + ''' + sh """ + set -xe + export install_repo="\${install_repo}" + export client_to_test="ps80" + export check_warning="\${check_warnings}" + export install_mysql_shell="\${install_mysql_shell}" + ansible-playbook \ + --connection=local \ + --inventory 127.0.0.1, \ + --limit 127.0.0.1 \ + ${playbook_path} + """ + } catch (Exception e) { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: Mini Package Testing for ${nodeName} at ${BRANCH} FAILED !!!") + mini_test_error="True" + } +} + +def minitestNodes = [ "min-bullseye-x64", + "min-bookworm-x64", + "min-centos-7-x64", + "min-ol-8-x64", + "min-focal-x64", + "min-amazon-2-x64", + "min-jammy-x64", + "min-noble-x64", + "min-ol-9-x64" ] + +@Field def mini_test_error = "False" +def AWS_STASH_PATH +def PS8_RELEASE_VERSION +def product_to_test = 'innovation-lts' +def install_repo = 'testing' +def action_to_test = 'install' +def check_warnings = 'yes' +def install_mysql_shell = 'no' + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } +parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + choice( + choices: 'perconalab\npercona', + description: 'Organization on hub.docker.com', + name: 'ORGANIZATION') + string(defaultValue: 'https://github.com/percona/percona-docker', description: 'Dockerfiles source', name: 'REPO_DOCKER') + string(defaultValue: 'main', description: 'Tag/Branch for percona-docker repository', name: 'REPO_DOCKER_BRANCH') + string(defaultValue: 'release-8.0.43-34', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') + string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') + string(defaultValue: '1', description: 'DEB version', name: 'DEB_RELEASE') + choice( + choices: 'NO\nYES', + description: 'Enable fipsmode', + name: 'FIPSMODE') + choice( + choices: 'percona\nmysql', + description: 'Which mysql-shell version have to be used in images.', + name: 'MYSQLSHELL') + choice( + choices: 'testing\nexperimental\nrelease', + description: 'Repo component to push packages to', + name: 'COMPONENT') + choice( + choices: '#releases-ci\n#releases', + description: 'Channel for notifications', + name: 'SLACKNOTIFY') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + + stage('Build docker containers') { + agent { + label 'launcher-x64' + } + steps { + script { + sh ''' + if [ "${MYSQLSHELL}" = "percona" ]; then + Dockerfile="Dockerfile" + else + Dockerfile="Dockerfile-mysqlsh-upstream" + fi + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + MYSQL_SHELL_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') + else + MYSQL_SHELL_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') + fi + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo lscpu | grep -q 'sse4_2' && grep -q 'popcnt' /proc/cpuinfo && echo "Supports x86-64-v2" || echo "Does NOT support x86-64-v2" + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + rm -rf percona-docker + git clone ${REPO_DOCKER} + cd percona-docker + git checkout ${REPO_DOCKER_BRANCH} + if [ ${PS_MAJOR_RELEASE} = "80" ]; then + cd percona-server-8.0 + else + cd percona-server-8.4 + fi + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" ${Dockerfile} + sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" ${Dockerfile} + #sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" ${Dockerfile} + sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" ${Dockerfile} + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + if [ ${PS_MAJOR_RELEASE} = "84" ]; then + sed -i "s/percona-release enable ps-80/percona-release enable ps-84-lts/g" ${Dockerfile} + else + sed -i "s/percona-release enable ps-80/percona-release enable ps-8x-innovation/g" ${Dockerfile} + fi + sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" ${Dockerfile} + sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" ${Dockerfile}.aarch64 + fi + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" ${Dockerfile}.aarch64 + sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" ${Dockerfile}.aarch64 + sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" ${Dockerfile}.aarch64 + sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" ${Dockerfile}.aarch64 + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + if [ ${PS_MAJOR_RELEASE} = "84" ]; then + sed -i "s/percona-release enable ps-80/percona-release enable ps-84-lts/g" ${Dockerfile}.aarch64 + else + sed -i "s/percona-release enable ps-80/percona-release enable ps-8x-innovation/g" ${Dockerfile}.aarch64 + fi + sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" ${Dockerfile}.aarch64 + fi + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 --progress plain --platform="linux/amd64" -f ${Dockerfile} . + sudo docker buildx build --platform linux/arm64 -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --load -f ${Dockerfile}.aarch64 . + else + sudo docker pull perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 + sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 percona/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 + sudo docker pull perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 percona/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + fi + cd ../mysql-router + sed -i "s/ENV ROUTE_VERSION.*/ENV ROUTE_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + if [ ${PS_MAJOR_RELEASE} = "84" ]; then + sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-84-lts testing/g" Dockerfile + else + sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-8x-innovation testing/g" Dockerfile + fi + fi + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 --platform="linux/amd64" . + sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 --platform="linux/arm64" . + sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} + else + sudo docker pull perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 + sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 percona/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 + sudo docker pull perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 + sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 percona/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 + sudo docker tag percona/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 percona/percona-mysql-router:${MYSQL_ROUTER_RELEASE} + fi + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') + if [ ${PS_MAJOR_RELEASE} != "80" ]; then + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') + else + MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') + fi + sudo docker tag ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 ${ORGANIZATION}/percona-server:${PS_RELEASE}-amd64 + sudo docker push ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 + sudo docker push ${ORGANIZATION}/percona-server:${PS_RELEASE}-amd64 + sudo docker tag ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 ${ORGANIZATION}/percona-server:${PS_RELEASE}-arm64 + sudo docker push ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + sudo docker push ${ORGANIZATION}/percona-server:${PS_RELEASE}-arm64 + sudo docker tag ${ORGANIZATION}/percona-mysql-router:${MYSQL_ROUTER_RELEASE} ${ORGANIZATION}/percona-mysql-router:${PS_MAJOR_RELEASE} + sudo docker push ${ORGANIZATION}/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-amd64 + sudo docker push ${ORGANIZATION}/percona-mysql-router:${MYSQL_ROUTER_RELEASE}-arm64 + sudo docker push ${ORGANIZATION}/percona-mysql-router:${MYSQL_ROUTER_RELEASE} + sudo docker push ${ORGANIZATION}/percona-mysql-router:${PS_MAJOR_RELEASE} + ''' + } + sh ''' + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + sudo docker manifest create --amend ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} \ + ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 \ + ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') + PS_MAJOR_FULL_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/-.*//g") + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker manifest push ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-server:${PS_RELEASE} ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-server:${PS_MAJOR_FULL_RELEASE} ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-server:${PS_MAJOR_RELEASE} ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} + PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') + if [ ${PS_MAJOR_RELEASE} = "80" ]; then + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-server:latest ${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE} + fi + ''' + } + } + } + } + stage('Check by Trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + environment { + TRIVY_LOG = "trivy-high-junit.xml" + } + steps { + script { + try { + // 🔹 Install Trivy if not present + sh ''' + if ! command -v trivy &> /dev/null; then + echo "🔄 Installing Trivy..." + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + else + echo "✅ Trivy is already installed." + fi + ''' + + // 🔹 Define the image tags + def PS_RELEASE = "${BRANCH}".replace('release-', '') + def imageList = [ + "${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64", + "${ORGANIZATION}/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64" + ] + + // 🔹 Scan images and store logs + imageList.each { image -> + echo "🔍 Scanning ${image}..." + def result = sh(script: """#!/bin/bash + set -e + sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 1 \ + --scanners vuln \ + --severity HIGH,CRITICAL ${image} + echo "TRIVY_EXIT_CODE=\$?" + """, returnStatus: true) + echo "Actual Trivy exit code: ${result}" + + // 🔴 Fail the build if vulnerabilities are found + if (result != 0) { + sh """ + sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 0 \ + --scanners vuln \ + --severity HIGH,CRITICAL ${image} | tee -a ${TRIVY_LOG} + """ + error "❌ Trivy detected vulnerabilities in ${image}. See ${TRIVY_LOG} for details." + } else { + echo "✅ No critical vulnerabilities found in ${image}." + } + } + } catch (Exception e) { + error "❌ Trivy scan failed: ${e.message}" + } // try + } // script + } // steps + } // stage + } + post { + success { + script { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: (${ORGANIZATION}) build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + } + deleteDir() + } + failure { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: (${ORGANIZATION})build failed for ${BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${BRANCH} for ${ORGANIZATION} organization" + } + deleteDir() + } + } +} diff --git a/ps/jenkins/percona-server-for-mysql-8.0-docker.yml b/ps/jenkins/percona-server-for-mysql-8.0-docker.yml new file mode 100644 index 0000000000..ee817071b1 --- /dev/null +++ b/ps/jenkins/percona-server-for-mysql-8.0-docker.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-ps8.0-docker-build + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/adivinho/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: ps/jenkins/percona-server-for-mysql-8.0-docker.groovy diff --git a/ps/jenkins/percona-server-for-mysql-8.0.groovy b/ps/jenkins/percona-server-for-mysql-8.0.groovy index fe4ee6c4f9..ee2f6ebee3 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0.groovy +++ b/ps/jenkins/percona-server-for-mysql-8.0.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,68 +8,93 @@ import groovy.transform.Field void installCli(String PLATFORM) { sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip - elif [ ${PLATFORM} = "rpm" ]; then - export RHVER=\$(rpm --eval %rhel) - if [ \${RHVER} = "7" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true - sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true - if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then - cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi fi + sudo yum -y install wget curl unzip fi - sudo yum -y install wget curl unzip - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi """ } void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -O ps_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -o ps_builder.sh - if [ ${FIPSMODE} = "YES" ]; then - sed -i 's|percona-server-server/usr|percona-server-server-pro/usr|g' ps_builder.sh - sed -i 's|dbg-package=percona-server-dbg|dbg-package=percona-server-pro-dbg|g' ps_builder.sh - fi - grep "percona-server-server" ps_builder.sh - export build_dir=\$(pwd -P) - if [ "$DOCKER_OS" = "none" ]; then - set -o xtrace - cd \${build_dir} - if [ -f ./test/percona-server-8.0.properties ]; then - . ./test/percona-server-8.0.properties - fi - sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 - if [ ${BUILD_TOKUDB_TOKUBACKUP} = "ON" ]; then - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - else - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - fi - else - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " - set -o xtrace - cd \${build_dir} - if [ -f ./test/percona-server-8.0.properties ]; then - . ./test/percona-server-8.0.properties - fi - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 - if [ ${BUILD_TOKUDB_TOKUBACKUP} = \"ON\" ]; then - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - else - bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} - fi" - fi - """ + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + mkdir -p test + if [ \${FIPSMODE} = "YES" ]; then + MYSQL_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${BRANCH}/MYSQL_VERSION && grep MYSQL_VERSION_MINOR MYSQL_VERSION | awk -F= '{print \$2}') + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${MYSQL_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o ps_builder.sh \ + "https://api.github.com/repos/percona/percona-server-private-build/contents/build-ps/percona-server-8.0_builder.sh?ref=\${PRO_BRANCH}" + sed -i 's|percona-server-server/usr|percona-server-server-pro/usr|g' ps_builder.sh + sed -i 's|dbg-package=percona-server-dbg|dbg-package=percona-server-pro-dbg|g' ps_builder.sh + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -O ps_builder.sh || curl \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/build-ps/percona-server-8.0_builder.sh -o ps_builder.sh + fi + grep "percona-server-server" ps_builder.sh + export build_dir=\$(pwd -P) + if [ "$DOCKER_OS" = "none" ]; then + set -o xtrace + cd \${build_dir} + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-server-private-build.git percona-server-private-build + mv -f \${build_dir}/percona-server-private-build/build-ps \${build_dir}/test/. + fi + if [ -f ./test/percona-server-8.0.properties ]; then + . ./test/percona-server-8.0.properties + fi + sudo bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ ${BUILD_TOKUDB_TOKUBACKUP} = "ON" ]; then + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + else + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + fi + else + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-server-private-build.git percona-server-private-build + mv -f \${build_dir}/percona-server-private-build/build-ps \${build_dir}/test/. + fi + if [ -f ./test/percona-server-8.0.properties ]; then + . ./test/percona-server-8.0.properties + fi + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ ${BUILD_TOKUDB_TOKUBACKUP} = \"ON\" ]; then + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --build_tokudb_tokubackup=1 --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + else + bash -x ./ps_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --perconaft_branch=${PERCONAFT_BRANCH} --tokubackup_branch=${TOKUBACKUP_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM} + fi" + fi + """ + } } void cleanUpWS() { @@ -161,7 +186,6 @@ def runPlaybook(def nodeName) { def minitestNodes = [ "min-bullseye-x64", "min-bookworm-x64", - "min-centos-7-x64", "min-ol-8-x64", "min-focal-x64", "min-amazon-2-x64", @@ -196,11 +220,15 @@ def install_mysql_shell = 'no' pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server.git', description: 'github repository for build', name: 'GIT_REPO') - string(defaultValue: 'release-8.0.28-19', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') + string(defaultValue: 'release-8.0.43-34', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') string(defaultValue: '1', description: 'DEB version', name: 'DEB_RELEASE') choice( @@ -211,10 +239,6 @@ parameters { string(defaultValue: 'Percona-Server-8.0.27-18', description: 'Tag/Branch for PerconaFT repository', name: 'PERCONAFT_BRANCH') string(defaultValue: '0', description: 'TokuBackup repository', name: 'TOKUBACKUP_REPO') string(defaultValue: 'Percona-Server-8.0.27-18', description: 'Tag/Branch for TokuBackup repository', name: 'TOKUBACKUP_BRANCH') - choice( - choices: 'NO\nYES', - description: 'Prepare packages and tarballs for Centos 7', - name: 'ENABLE_EL7') choice( choices: 'ON\nOFF', description: 'Compile with ZenFS support?, only affects Ubuntu Hirsute', @@ -242,13 +266,19 @@ parameters { stage('Create PS source tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' } steps { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") cleanUpWS() installCli("deb") - buildStage("none", "--get_sources=1") + script { + if (env.FIPSMODE == 'YES') { + buildStage("none", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("none", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-8.0.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -263,307 +293,360 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-8.0.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { parallel { stage('Build PS generic source rpm') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_src_rpm=1 --enable_fipsmode=1") + buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_src_rpm=1") + buildStage("centos:7", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PS generic source deb') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_source_deb=1 --enable_fipsmode=1") + buildStage("ubuntu:focal", "--build_source_deb=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_source_deb=1") + buildStage("ubuntu:focal", "--build_source_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PS RPMs/DEBs/Binary tarballs') { parallel { - stage('Centos 7') { + stage('Oracle Linux 8') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES' || env.ENABLE_EL7 == 'NO') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("none", "--build_rpm=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } - stage('Oracle Linux 8') { + stage('Centos 8 ARM') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("none", "--build_rpm=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } - stage('Centos 8 ARM') { + stage('Oracle Linux 9') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - echo "The step is skipped" + buildStage("oraclelinux:9", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("centos:8", "--build_rpm=1") - - pushArtifactFolder("rpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1 --with_zenfs=1") } } + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_rpm=1 --with_zenfs=1") + buildStage("oraclelinux:9", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9 ARM') { + stage('Oracle Linux 10') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + buildStage("oraclelinux:10", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("oraclelinux:9", "--build_rpm=1") + buildStage("oraclelinux:10", "--build_rpm=1 --with_zenfs=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } -/* stage('Amazon Linux 2023') { + stage('Oracle Linux 10 ARM') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") } else { - buildStage("amazonlinux:2023", "--build_rpm=1 --with_zenfs=1") + buildStage("oraclelinux:10", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } - }*/ + } + stage('Amazon Linux 2023') { + when { + expression { env.FIPSMODE == 'YES' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + stage('Amazon Linux 2023 ARM') { + when { + expression { env.FIPSMODE == 'YES' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } stage('Ubuntu Focal(20.04)') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("none", "--build_deb=1 --with_zenfs=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1") } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1") } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("none", "--build_deb=1 --with_zenfs=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bookworm(12)') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1") } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04) ARM') { +/* stage('Debian Trixie(13)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - echo "The step is skipped" + buildStage("debian:trixie", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") - - pushArtifactFolder("deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1 --with_zenfs=1") } } + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + }*/ + stage('Ubuntu Focal(20.04) ARM') { + when { + expression { env.FIPSMODE == 'NO' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -572,18 +655,18 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -592,38 +675,37 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -632,261 +714,234 @@ parameters { } } - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Centos 7 binary tarball') { +/* stage('Debian Trixie(13) ARM') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'YES' || env.ENABLE_EL7 == 'NO') { - echo "The step is skipped" + if (env.FIPSMODE == 'YES') { + buildStage("debian:trixie", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") - - pushArtifactFolder("tarball/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1 --with_zenfs=1") } } - } - } - stage('Centos 7 debug tarball') { - agent { - label 'min-centos-7-x64' - } - steps { - script { - if (env.FIPSMODE == 'YES' || env.ENABLE_EL7 == 'NO') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } - } + }*/ stage('Oracle Linux 8 binary tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 8 debug tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("rpm") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_tarball=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_tarball=1") + buildStage("oraclelinux:9", "--build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ZenFS tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } + buildStage("oraclelinux:9", "--build_tarball=1 --with_zenfs=1") + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 debug tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--debug=1 --build_tarball=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--debug=1 --build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--debug=1 --build_tarball=1") + buildStage("oraclelinux:9", "--debug=1 --build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Focal(20.04) debug tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - installCli("deb") - unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--debug=1 --build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_tarball=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_tarball=1") + buildStage("ubuntu:jammy", "--build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ZenFS tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - buildStage("none", "--build_tarball=1 --with_zenfs=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - } - } + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_tarball=1 --with_zenfs=1") + + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) debug tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--debug=1 --build_tarball=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--debug=1 --build_tarball=1") + buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1") } } - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Upload packages and tarballs from S3') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - installCli("deb") + installCli("rpm") unstash 'properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } @@ -905,22 +960,22 @@ parameters { // sync packages if ("${MYSQL_VERSION_MINOR}" == "0") { if (env.FIPSMODE == 'YES') { - sync2PrivateProdAutoBuild("ps-80-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-80-pro", COMPONENT) } else { - sync2ProdAutoBuild("ps-80", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-80", COMPONENT) } } else { if (env.FIPSMODE == 'YES') { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2PrivateProdAutoBuild("ps-84-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-84-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("ps-8x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-8x-innovation-pro", COMPONENT) } } else { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2ProdAutoBuild("ps-84-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-84-lts", COMPONENT) } else { - sync2ProdAutoBuild("ps-8x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-8x-innovation", COMPONENT) } } } @@ -932,9 +987,9 @@ parameters { script { try { if (env.FIPSMODE == 'YES') { - uploadTarballToDownloadsTesting("ps-gated", "${BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "ps-gated", "${BRANCH}") } else { - uploadTarballToDownloadsTesting("ps", "${BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "ps", "${BRANCH}") } } catch (err) { @@ -945,219 +1000,55 @@ parameters { } } stage('Build docker containers') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-focal-x64' + label 'launcher-x64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - echo "====> Build docker container" - cleanUpWS() - installCli("deb") - sh ''' - sleep 1200 - ''' - unstash 'properties' - sh ''' - PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') - PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - MYSQL_SHELL_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') - MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') - else - MYSQL_SHELL_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') - MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') - fi - sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common - sudo apt-get install -y docker.io - sudo systemctl status docker - sudo apt-get install -y qemu binfmt-support qemu-user-static - sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - git clone https://github.com/percona/percona-docker - cd percona-docker/percona-server-8.0 - sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile - sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" Dockerfile - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile - fi - sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" Dockerfile - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - if [ ${PS_MAJOR_RELEASE} = "84" ]; then - sed -i "s/percona-release enable ps-80/percona-release enable ps-84-lts/g" Dockerfile - else - sed -i "s/percona-release enable ps-80/percona-release enable ps-8x-innovation/g" Dockerfile - fi - sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" Dockerfile - fi - sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile.aarch64 - sed -i "s/ENV PS_TELEMETRY_VERSION.*/ENV PS_TELEMETRY_VERSION ${PS_RELEASE}-${RPM_RELEASE}/g" Dockerfile.aarch64 - sed -i "s/ENV PS_REPO .*/ENV PS_REPO testing/g" Dockerfile.aarch64 - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - if [ ${PS_MAJOR_RELEASE} = "84" ]; then - sed -i "s/percona-release enable ps-80/percona-release enable ps-84-lts/g" Dockerfile.aarch64 - else - sed -i "s/percona-release enable ps-80/percona-release enable ps-8x-innovation/g" Dockerfile.aarch64 - fi - sed -i "s/percona-release enable mysql-shell/PS_REPO=\"testing\";percona-release enable mysql-shell/g" Dockerfile.aarch64 - fi - sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 . - sudo docker build -t perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . - cd ../mysql-router - sed -i "s/ENV ROUTE_VERSION.*/ENV ROUTE_VERSION ${PS_RELEASE}.${RPM_RELEASE}/g" Dockerfile - sed -i "s/ENV MYSQL_SHELL_VERSION.*/ENV MYSQL_SHELL_VERSION ${MYSQL_SHELL_RELEASE}-${RPM_RELEASE}/g" Dockerfile - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - if [ ${PS_MAJOR_RELEASE} = "84" ]; then - sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-84-lts testing/g" Dockerfile - else - sed -i "s/percona-release enable ps-80 testing/percona-release enable ps-8x-innovation testing/g" Dockerfile - fi - fi - sudo docker build -t perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} . - sudo docker images - ''' - withCredentials([ - usernamePassword(credentialsId: 'hub.docker.com', - passwordVariable: 'PASS', - usernameVariable: 'USER' - )]) { - sh ''' - echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin - PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') - PS_MAJOR_RELEASE=$(echo ${BRANCH} | sed "s/release-//g" | awk '{print substr($0, 0, 3)}') - if [ ${PS_MAJOR_RELEASE} != "80" ]; then - MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 6)}' | sed 's/-//g') - else - MYSQL_ROUTER_RELEASE=$(echo ${BRANCH} | sed 's/release-//g' | awk '{print substr($0, 0, 7)}' | sed 's/-//g') - fi - sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} perconalab/percona-server:${PS_RELEASE} - sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} - sudo docker push perconalab/percona-server:${PS_RELEASE} - sudo docker tag perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-aarch64 perconalab/percona-server:${PS_RELEASE}-aarch64 - sudo docker push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-aarch64 - sudo docker push perconalab/percona-server:${PS_RELEASE}-aarch64 - sudo docker tag perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} perconalab/percona-mysql-router:${PS_MAJOR_RELEASE} - sudo docker push perconalab/percona-mysql-router:${MYSQL_ROUTER_RELEASE} - sudo docker push perconalab/percona-mysql-router:${PS_MAJOR_RELEASE} - ''' - } - sh ''' - PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') - sudo docker manifest create perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} \ - perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 \ - perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 - sudo docker manifest annotate perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 - sudo docker manifest annotate perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE}-amd64 --os linux --arch amd64 - sudo docker manifest inspect perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} - ''' - withCredentials([ - usernamePassword(credentialsId: 'hub.docker.com', - passwordVariable: 'PASS', - usernameVariable: 'USER' - )]) { - sh ''' - PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') - echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin - PS_RELEASE=$(echo ${BRANCH} | sed 's/release-//g') - sudo docker manifest push perconalab/percona-server:${PS_RELEASE}.${RPM_RELEASE} - ''' - } - } + build job: 'hetzner-ps8.0-docker-build', + parameters: [ + string(name: 'CLOUD', value: 'Hetzner'), + string(name: 'ORGANIZATION', value: 'perconalab'), + string(name: 'BRANCH', value: "${BRANCH}"), + string(name: 'RPM_RELEASE', value: '1'), + string(name: 'DEB_RELEASE', value: '1'), + string(name: 'FIPSMODE', value: 'NO'), + booleanParam(name: 'RUN_FAST', value: true) + ], + wait: false } } - } + } } post { success { script { if (env.FIPSMODE == 'YES') { - slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO -> build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") } else { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]") } } slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: Triggering Builds for Package Testing for ${BRANCH} - [${BUILD_URL}]") - unstash 'properties' - script { - currentBuild.description = "Built on ${BRANCH}; path to packages: ${COMPONENT}/${AWS_STASH_PATH}" - REVISION = sh(returnStdout: true, script: "grep REVISION test/percona-server-8.0.properties | awk -F '=' '{ print\$2 }'").trim() - PS_RELEASE = sh(returnStdout: true, script: "echo ${BRANCH} | sed 's/release-//g'").trim() - PS8_RELEASE_VERSION = sh(returnStdout: true, script: """ echo ${BRANCH} | sed -nE '/release-(8\\.[0-9]{1})\\..*/s//\\1/p' """).trim() - - if("${PS8_RELEASE_VERSION}"){ - echo "Executing MINITESTS as VALID VALUES FOR PS8_RELEASE_VERSION:${PS8_RELEASE_VERSION}" - echo "Checking for the Github Repo VERSIONS file changes..." - withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { - sh """ - set -x - git clone https://jenkins-pxc-cd:$TOKEN@github.com/Percona-QA/package-testing.git - cd package-testing - git config user.name "jenkins-pxc-cd" - git config user.email "it+jenkins-pxc-cd@percona.com" - echo "${PS8_RELEASE_VERSION} is the VALUE!!@!" - export RELEASE_VER_VAL="${PS8_RELEASE_VERSION}" - if [[ "\$RELEASE_VER_VAL" =~ ^8.[0-9]{1}\$ ]]; then - echo "\$RELEASE_VER_VAL is a valid version" - OLD_REV=\$(cat VERSIONS | grep PS_INN_LTS_REV | cut -d '=' -f2- ) - OLD_VER=\$(cat VERSIONS | grep PS_INN_LTS_VER | cut -d '=' -f2- ) - sed -i s/PS_INN_LTS_REV=\$OLD_REV/PS_INN_LTS_REV='"'${REVISION}'"'/g VERSIONS - sed -i s/PS_INN_LTS_VER=\$OLD_VER/PS_INN_LTS_VER='"'${PS_RELEASE}'"'/g VERSIONS - - else - echo "INVALID PS8_RELEASE_VERSION VALUE: ${PS8_RELEASE_VERSION}" - fi - git diff - if [[ -z \$(git diff) ]]; then - echo "No changes" - else - echo "There are changes" - git add -A - git commit -m "Autocommit: add ${REVISION} and ${PS_RELEASE} for ${PS8_RELEASE_VERSION} package testing VERSIONS file." - git push - fi - """ - } - echo "Start Minitests for PS" - package_tests_ps80(minitestNodes) - if("${mini_test_error}" == "True"){ - error "NOT TRIGGERING PACKAGE TESTS AND INTEGRATION TESTS DUE TO MINITEST FAILURE !!" - }else{ - echo "TRIGGERING THE PACKAGE TESTING JOB!!!" - build job: 'package-testing-ps-innovation-lts', propagate: false, wait: false, parameters: [string(name: 'product_to_test', value: "${product_to_test}"),string(name: 'install_repo', value: "testing"),string(name: 'node_to_test', value: "all"),string(name: 'action_to_test', value: "all"),string(name: 'check_warnings', value: "yes"),string(name: 'install_mysql_shell', value: "no")] - - echo "Trigger PMM_PS Github Actions Workflow" - - withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'GITHUB_API_TOKEN')]) { - sh """ - curl -i -v -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${GITHUB_API_TOKEN}" \ - "https://api.github.com/repos/Percona-Lab/qa-integration/actions/workflows/PMM_PS.yaml/dispatches" \ - -d '{"ref":"main","inputs":{"ps_version":"${PS_RELEASE}"}}' - """ - } - - } - } - else{ - error "Skipping MINITESTS and Other Triggers as invalid RELEASE VERSION FOR THIS JOB" - slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: Skipping MINITESTS and Other Triggers as invalid RELEASE VERSION FOR THIS JOB ${BRANCH} - [${BUILD_URL}]") - } - } deleteDir() } failure { - slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${BRANCH} - [${BUILD_URL}]") - script { - currentBuild.description = "Built on ${BRANCH}" - } deleteDir() } always { sh ''' sudo rm -rf ./* ''' + script { + if (env.FIPSMODE == 'YES') { + currentBuild.description = "Pro -> Build on ${BRANCH}" + } else { + currentBuild.description = "Build on ${BRANCH}" + } + } deleteDir() } } diff --git a/ps/jenkins/percona-server-for-mysql-8.0.yml b/ps/jenkins/percona-server-for-mysql-8.0.yml index d334d9a4de..49d5ad3dee 100644 --- a/ps/jenkins/percona-server-for-mysql-8.0.yml +++ b/ps/jenkins/percona-server-for-mysql-8.0.yml @@ -1,5 +1,5 @@ - job: - name: ps8.0-autobuild-RELEASE + name: hetzner-ps8.0-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-8.0.groovy diff --git a/ps/jenkins/percona-server-for-mysql-9.0.groovy b/ps/jenkins/percona-server-for-mysql-9.0.groovy index 000b9fcaaf..51395070fe 100644 --- a/ps/jenkins/percona-server-for-mysql-9.0.groovy +++ b/ps/jenkins/percona-server-for-mysql-9.0.groovy @@ -1,5 +1,5 @@ /* groovylint-disable DuplicateStringLiteral, GStringExpressionWithinString, LineLength */ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,27 +8,29 @@ import groovy.transform.Field void installCli(String PLATFORM) { sh """ - set -o xtrace - if [ -d aws ]; then - rm -rf aws - fi - if [ ${PLATFORM} = "deb" ]; then - sudo apt-get update - sudo apt-get -y install wget curl unzip - elif [ ${PLATFORM} = "rpm" ]; then - export RHVER=\$(rpm --eval %rhel) - if [ \${RHVER} = "7" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true - sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true - if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then - cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + if [ \${CLOUD} = "AWS" ]; then + set -o xtrace + if [ -d aws ]; then + rm -rf aws + fi + if [ ${PLATFORM} = "deb" ]; then + sudo apt-get update + sudo apt-get -y install wget curl unzip + elif [ ${PLATFORM} = "rpm" ]; then + export RHVER=\$(rpm --eval %rhel) + if [ \${RHVER} = "7" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true + sudo sed -i 's|#\\s*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true + if [ -e "/etc/yum.repos.d/CentOS-SCLo-scl.repo" ]; then + cat /etc/yum.repos.d/CentOS-SCLo-scl.repo + fi fi + sudo yum -y install wget curl unzip fi - sudo yum -y install wget curl unzip - fi - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip - unzip awscliv2.zip - sudo ./aws/install || true + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install || true + fi """ } @@ -183,11 +185,15 @@ def install_mysql_shell = 'no' pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string(defaultValue: 'https://github.com/percona/percona-server.git', description: 'github repository for build', name: 'GIT_REPO') - string(defaultValue: 'release-9.0.1-1', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') + string(defaultValue: 'release-9.3.0-1', description: 'Tag/Branch for percona-server repository', name: 'BRANCH') string(defaultValue: '1', description: 'RPM version', name: 'RPM_RELEASE') string(defaultValue: '1', description: 'DEB version', name: 'DEB_RELEASE') choice( @@ -221,7 +227,7 @@ parameters { stage('Create PS source tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' } steps { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -242,51 +248,51 @@ parameters { } stash includes: 'uploadPath', name: 'uploadPath' stash includes: 'test/percona-server-9.0.properties', name: 'properties' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PS generic source packages') { parallel { stage('Build PS generic source rpm') { agent { - label 'min-centos-7-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_src_rpm=1 --enable_fipsmode=1") + buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_src_rpm=1") + buildStage("centos:7", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PS generic source deb') { agent { - label 'min-buster-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_source_deb=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_source_deb=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_source_deb=1") + buildStage("ubuntu:jammy", "--build_source_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -295,7 +301,7 @@ parameters { parallel { stage('Oracle Linux 8') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -305,11 +311,11 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("none", "--build_rpm=1") + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_rpm=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -317,7 +323,7 @@ parameters { } stage('Centos 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -327,10 +333,10 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } @@ -338,34 +344,34 @@ parameters { } stage('Oracle Linux 9') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_rpm=1 --with_zenfs=1") + buildStage("oraclelinux:9", "--build_rpm=1 --with_zenfs=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") @@ -373,14 +379,57 @@ parameters { buildStage("oraclelinux:9", "--build_rpm=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:10", "--build_rpm=1 --with_zenfs=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:10", "--build_rpm=1 --with_zenfs=1") + } + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + installCli("rpm") + unstash 'properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:10", "--build_rpm=1") + } + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } +/* stage('Ubuntu Focal(20.04)') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -390,59 +439,60 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("none", "--build_deb=1 --with_zenfs=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } } +*/ stage('Ubuntu Jammy(22.04)') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1") + } + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'min-noble-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") - } - if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1") } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } +/* stage('Debian Bullseye(11)') { agent { - label 'min-bullseye-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -452,39 +502,41 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("none", "--build_deb=1 --with_zenfs=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } } +*/ stage('Debian Bookworm(12)') { agent { - label 'min-bookworm-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") + buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_deb=1 --with_zenfs=1") + buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } +/* stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -494,59 +546,60 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } } +*/ stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_deb=1 --with_zenfs=1") } + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_deb=1 --with_zenfs=1") } - if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) - } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } +/* stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -556,25 +609,26 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } } +*/ stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --with_zenfs=1 --enable_fipsmode=1") @@ -583,14 +637,14 @@ parameters { } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Oracle Linux 8 binary tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -600,10 +654,10 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -611,7 +665,7 @@ parameters { } stage('Oracle Linux 8 debug tarball') { agent { - label 'min-ol-8-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -621,10 +675,10 @@ parameters { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--debug=1 --build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -632,41 +686,41 @@ parameters { } stage('Oracle Linux 9 tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_tarball=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_tarball=1") + buildStage("oraclelinux:9", "--build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9 ZenFS tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { echo "The step is skipped" } else { - buildStage("none", "--build_tarball=1 --with_zenfs=1") + buildStage("oraclelinux:9", "--build_tarball=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -674,28 +728,29 @@ parameters { } stage('Oracle Linux 9 debug tarball') { agent { - label 'min-ol-9-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("rpm") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--debug=1 --build_tarball=1 --enable_fipsmode=1") + buildStage("oraclelinux:9", "--debug=1 --build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--debug=1 --build_tarball=1") + buildStage("oraclelinux:9", "--debug=1 --build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } +/* stage('Ubuntu Focal(20.04) tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -705,10 +760,10 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -716,7 +771,7 @@ parameters { } stage('Ubuntu Focal(20.04) debug tarball') { agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -726,52 +781,53 @@ parameters { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("none", "--debug=1 --build_tarball=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--debug=1 --build_tarball=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } +*/ stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--build_tarball=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--build_tarball=1") + buildStage("ubuntu:jammy", "--build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04) ZenFS tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { echo "The step is skipped" } else { - buildStage("none", "--build_tarball=1 --with_zenfs=1") + buildStage("ubuntu:jammy", "--build_tarball=1 --with_zenfs=1") if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -779,21 +835,21 @@ parameters { } stage('Ubuntu Jammy(22.04) debug tarball') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("none", "--debug=1 --build_tarball=1 --enable_fipsmode=1") + buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1 --enable_fipsmode=1") } else { - buildStage("none", "--debug=1 --build_tarball=1") + buildStage("ubuntu:jammy", "--debug=1 --build_tarball=1") } if (env.EXPERIMENTALMODE == 'NO') { - pushArtifactFolder("tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } @@ -802,24 +858,23 @@ parameters { } stage('Upload packages and tarballs from S3') { agent { - label 'min-jammy-x64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() installCli("deb") unstash 'properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) script { if (env.EXPERIMENTALMODE == 'NO') { - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } } } - stage('Sign packages') { steps { script { @@ -839,15 +894,15 @@ parameters { // sync packages if (env.FIPSMODE == 'YES') { if ("${MYSQL_VERSION_MINOR}" == "7") { - sync2PrivateProdAutoBuild("ps-97-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-97-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("ps-9x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "ps-9x-innovation-pro", COMPONENT) } } else { if ("${MYSQL_VERSION_MINOR}" == "7") { - sync2ProdAutoBuild("ps-97-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-97-lts", COMPONENT) } else { - sync2ProdAutoBuild("ps-9x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "ps-9x-innovation", COMPONENT) } } } diff --git a/ps/jenkins/percona-server-for-mysql-9.0.yml b/ps/jenkins/percona-server-for-mysql-9.0.yml index 48674ec333..fd7f00b932 100644 --- a/ps/jenkins/percona-server-for-mysql-9.0.yml +++ b/ps/jenkins/percona-server-for-mysql-9.0.yml @@ -1,5 +1,5 @@ - job: - name: ps9.0-RELEASE + name: hetzner-ps9.0-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/percona-server-for-mysql-9.0.groovy diff --git a/ps/jenkins/qpress.groovy b/ps/jenkins/qpress.groovy index 1c2b1b7a18..59e79c4ce4 100644 --- a/ps/jenkins/qpress.groovy +++ b/ps/jenkins/qpress.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -106,7 +106,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { PKGLIST=\\"\\\${PKGLIST} libldap2-dev libnuma-dev libjemalloc-dev libc6-dbg valgrind libjson-perl\\" PKGLIST=\\"\\\${PKGLIST} libmecab2 mecab mecab-ipadic zip unzip wget\\" PKGLIST=\\"\\\${PKGLIST} build-essential debhelper devscripts lintian diffutils patch patchutils\\" - if [ \\\$DEBIAN_VERSION = focal -o \\\$DEBIAN_VERSION = bullseye -o \\\$DEBIAN_VERSION = jammy -o \\\$DEBIAN_VERSION = bookworm -o \\\$DEBIAN_VERSION = noble ]; then + if [ \\\$DEBIAN_VERSION = focal -o \\\$DEBIAN_VERSION = bullseye -o \\\$DEBIAN_VERSION = jammy -o \\\$DEBIAN_VERSION = bookworm -o \\\$DEBIAN_VERSION = trixie -o \\\$DEBIAN_VERSION = noble ]; then PKGLIST=\\"\\\${PKGLIST} python3-mysqldb\\" else PKGLIST=\\"\\\${PKGLIST} python-mysqldb\\" @@ -142,9 +142,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona-lab/qpress-packaging.git', description: 'URL for qpress packaging repository', @@ -196,19 +200,19 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build qpress packages') { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -217,19 +221,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "RPM") sh ''' pwd @@ -238,19 +242,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "RPM") sh ''' pwd @@ -259,19 +263,19 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "RPM") sh ''' pwd @@ -280,19 +284,61 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "RPM") + sh ''' + pwd + ls -la test/rpm + cp -r test/srpm . + cp -r test/rpm . + ''' + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "RPM") + sh ''' + pwd + ls -la test/rpm + cp -r test/srpm . + cp -r test/rpm . + ''' + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "RPM") sh ''' pwd @@ -301,19 +347,40 @@ pipeline { cp -r test/rpm . ''' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "RPM") + sh ''' + pwd + ls -la test/rpm + cp -r test/srpm . + cp -r test/rpm . + ''' + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "DEB") sh ''' pwd @@ -321,17 +388,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "DEB") sh ''' pwd @@ -339,17 +406,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "DEB") sh ''' pwd @@ -357,17 +424,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "DEB") sh ''' pwd @@ -375,17 +442,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "DEB") sh ''' pwd @@ -393,17 +460,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy (22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "DEB") sh ''' pwd @@ -411,17 +478,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "DEB") sh ''' pwd @@ -429,17 +496,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "DEB") sh ''' pwd @@ -447,17 +514,53 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie (13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:trixie", "DEB") + sh ''' + pwd + ls -la test/deb + cp -r test/deb . + ''' + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie (13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:trixie", "DEB") + sh ''' + pwd + ls -la test/deb + cp -r test/deb . + ''' + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "DEB") sh ''' pwd @@ -465,17 +568,17 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble (24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "DEB") sh ''' pwd @@ -483,8 +586,8 @@ pipeline { cp -r test/deb . ''' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } @@ -498,7 +601,7 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild('tools', COMPONENT) + sync2ProdAutoBuild(params.CLOUD, 'tools', COMPONENT) } } diff --git a/ps/jenkins/qpress.yml b/ps/jenkins/qpress.yml index 2171795e63..a164ecf24f 100644 --- a/ps/jenkins/qpress.yml +++ b/ps/jenkins/qpress.yml @@ -1,5 +1,5 @@ - job: - name: qpress-RELEASE + name: hetzner-qpress-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: ps/jenkins/qpress.groovy diff --git a/psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.groovy b/psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.groovy new file mode 100644 index 0000000000..c5df02b96f --- /dev/null +++ b/psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.groovy @@ -0,0 +1,178 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + ls -laR ./ + rm -rf test/* + mkdir -p test + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + """ + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') + string( + defaultValue: 'https://github.com/percona/percona-server-mongodb.git', + description: 'URL for percona-server-mongodb repository', + name: 'GIT_REPO') + string( + defaultValue: 'release-7.0.22-12', + description: 'Tag/Branch for percona-server-mongodb repository', + name: 'GIT_BRANCH') + string( + defaultValue: 'https://github.com/percona/percona-server-mongodb.git', + description: 'URL for CUSTOM percona-server-mongodb repository to take BuildScript from', + name: 'BS_GIT_REPO') + string( + defaultValue: 'release-7.0.22-12', + description: 'Tag/Branch for CUSTOM percona-server-mongodb repository to take BuildScript from', + name: 'BS_GIT_BRANCH') + string( + defaultValue: '7.0.22', + description: 'PSMDB release value', + name: 'PSMDB_VERSION') + string( + defaultValue: '12', + description: 'PSMDB release value', + name: 'PSMDB_RELEASE') + string( + defaultValue: '100.12.1', + description: 'https://docs.mongodb.com/database-tools/installation/', + name: 'MONGO_TOOLS_TAG') + string( + defaultValue: 'CUSTOM199', + description: 'Jira task name without dash(CUSTOM-199 -> CUSTOM199)', + name: 'JIRA_TASK') + choice( + choices: 'laboratory\ntesting\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Create PSMDB source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } + sh ''' + REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-70.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/percona-server-mongodb-70.properties + cat uploadPath + cat awsUploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Centos 7 binary tarball(glibc2.17)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("centos:7", "--build_tarball=1") + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + } + } + stage('Upload packages and tarballs from S3') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + + stage('Push Tarballs to TESTING download area') { + steps { + script { + try { + uploadTarballToDownloadsTesting(params.CLOUD, "issue", "${JIRA_TASK}") + } + catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } + } + } + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.yml b/psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.yml new file mode 100644 index 0000000000..ec6a635dbc --- /dev/null +++ b/psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-custom-psmdb70-ol7-tgz + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/vorsel/jenkins-pipelines.git + branches: + - 'CUSTOM-199_psmdb70_ol7_tgz' + wipe-workspace: false + lightweight-checkout: true + script-path: psmdb/jenkins/custom/custom-psmdb70-ol7-tgz.groovy + diff --git a/psmdb/jenkins/get-psmdb-branches-6.0.groovy b/psmdb/jenkins/get-psmdb-branches-6.0.groovy index 365bf6e30a..73ec4be7d1 100644 --- a/psmdb/jenkins/get-psmdb-branches-6.0.groovy +++ b/psmdb/jenkins/get-psmdb-branches-6.0.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-for-mongodb repository', @@ -22,10 +26,13 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh """ EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties || EC=\$? + aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties ${S3_ENDPOINT} || EC=\$? if [ \${EC} = 1 ]; then LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-6.0\\* | tail -1) @@ -38,10 +45,10 @@ pipeline { echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_60.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_60.properties - aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} echo "START_NEW_BUILD=NO" > startBuild else - aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties . + aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_60.properties . ${S3_ENDPOINT} source branch_commit_id_60.properties LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-6.0\\* | tail -1) @@ -59,11 +66,10 @@ pipeline { echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_60.properties echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_60.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_60.properties - aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_60.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} fi """ } - script { START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id_60.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id_60.properties; echo \${COMMIT_ID}").trim() @@ -86,9 +92,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'psmdb60-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb60-aarch64-build', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] + build job: 'hetzner-psmdb60-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] } } stage('Build skipped') { diff --git a/psmdb/jenkins/get-psmdb-branches-6.0.yml b/psmdb/jenkins/get-psmdb-branches-6.0.yml index b08d9537da..0f2b467683 100644 --- a/psmdb/jenkins/get-psmdb-branches-6.0.yml +++ b/psmdb/jenkins/get-psmdb-branches-6.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb60-autobuild + name: hetzner-psmdb60-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/get-psmdb-branches-6.0.groovy diff --git a/psmdb/jenkins/get-psmdb-branches-7.0.groovy b/psmdb/jenkins/get-psmdb-branches-7.0.groovy index 5896399b45..b5345cb733 100644 --- a/psmdb/jenkins/get-psmdb-branches-7.0.groovy +++ b/psmdb/jenkins/get-psmdb-branches-7.0.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-for-mongodb repository', @@ -22,10 +26,13 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh """ EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties || EC=\$? + aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties ${S3_ENDPOINT} || EC=\$? if [ \${EC} = 1 ]; then LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-7.0\\* | tail -1) @@ -38,10 +45,10 @@ pipeline { echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_70.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_70.properties - aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} echo "START_NEW_BUILD=NO" > startBuild else - aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties . + aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_70.properties . ${S3_ENDPOINT} source branch_commit_id_70.properties LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-7.0\\* | tail -1) @@ -59,11 +66,10 @@ pipeline { echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_70.properties echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_70.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_70.properties - aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + aws s3 cp branch_commit_id_70.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} fi """ } - script { START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id_70.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id_70.properties; echo \${COMMIT_ID}").trim() @@ -86,9 +92,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'psmdb70-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb70-aarch64-build', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] + build job: 'hetzner-psmdb70-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] } } stage('Build skipped') { diff --git a/psmdb/jenkins/get-psmdb-branches-7.0.yml b/psmdb/jenkins/get-psmdb-branches-7.0.yml index b903634d6a..df9c78f535 100644 --- a/psmdb/jenkins/get-psmdb-branches-7.0.yml +++ b/psmdb/jenkins/get-psmdb-branches-7.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb70-autobuild + name: hetzner-psmdb70-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/get-psmdb-branches-7.0.groovy diff --git a/psmdb/jenkins/get-psmdb-branches-8.0.groovy b/psmdb/jenkins/get-psmdb-branches-8.0.groovy index 0445905748..c631df0e0f 100644 --- a/psmdb/jenkins/get-psmdb-branches-8.0.groovy +++ b/psmdb/jenkins/get-psmdb-branches-8.0.groovy @@ -1,13 +1,17 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-for-mongodb repository', @@ -22,10 +26,13 @@ pipeline { stages { stage('Get release branches') { steps { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + script { + String S3_STASH = (params.CLOUD == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (params.CLOUD == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh """ EC=0 - aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties || EC=\$? + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 ls s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || EC=\$? if [ \${EC} = 1 ]; then LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-8.0\\* | tail -1) @@ -38,10 +45,10 @@ pipeline { echo "COMMIT_ID=\${COMMIT_ID}" >> branch_commit_id_80.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_80.properties - aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 echo "START_NEW_BUILD=NO" > startBuild else - aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties . + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp s3://percona-jenkins-artifactory/percona-server-mongodb/branch_commit_id_80.properties . ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 source branch_commit_id_80.properties LATEST_RELEASE_BRANCH=\$(git -c 'versionsort.suffix=-' ls-remote --heads --sort='v:refname' ${GIT_REPO} release-8.0\\* | tail -1) @@ -59,11 +66,10 @@ pipeline { echo "BRANCH_NAME=\${LATEST_BRANCH_NAME}" > branch_commit_id_80.properties echo "COMMIT_ID=\${LATEST_COMMIT_ID}" >> branch_commit_id_80.properties echo "MONGO_TOOLS_TAG=\${MONGO_TOOLS_TAG}" >> branch_commit_id_80.properties - aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp branch_commit_id_80.properties s3://percona-jenkins-artifactory/percona-server-mongodb/ ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 fi """ } - script { START_NEW_BUILD = sh(returnStdout: true, script: "source startBuild; echo \${START_NEW_BUILD}").trim() BRANCH_NAME = sh(returnStdout: true, script: "source branch_commit_id_80.properties; echo \${BRANCH_NAME}").trim() COMMIT_ID = sh(returnStdout: true, script: "source branch_commit_id_80.properties; echo \${COMMIT_ID}").trim() @@ -86,9 +92,8 @@ pipeline { """ } slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: new changes for branch ${BRANCH_NAME}[commit id: ${COMMIT_ID}] were detected, build will be started soon") - build job: 'psmdb80-autobuild-RELEASE', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb80-aarch64-build', parameters: [string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] - build job: 'psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] + build job: 'hetzner-psmdb80-autobuild-RELEASE', parameters: [string(name: 'CLOUD', value: CLOUD), string(name: 'GIT_BRANCH', value: BRANCH_NAME), string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE), string(name: 'MONGO_TOOLS_TAG', value: MONGO_TOOLS_TAG), string(name: 'COMPONENT', value: 'testing')] + build job: 'hetzner-psmdb-multijob-testing', propagate: false, wait: false, parameters: [string(name: 'PSMDB_VERSION', value: VERSION), string(name: 'PSMDB_RELEASE', value: RELEASE)] } } stage('Build skipped') { diff --git a/psmdb/jenkins/get-psmdb-branches-8.0.yml b/psmdb/jenkins/get-psmdb-branches-8.0.yml index 7c1ad0e1fb..e173237f61 100644 --- a/psmdb/jenkins/get-psmdb-branches-8.0.yml +++ b/psmdb/jenkins/get-psmdb-branches-8.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb80-autobuild + name: hetzner-psmdb80-autobuild project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/get-psmdb-branches-8.0.groovy diff --git a/psmdb/jenkins/percona-mongodb-mongosh.groovy b/psmdb/jenkins/percona-mongodb-mongosh.groovy index 7b3069404e..6969c51a8c 100644 --- a/psmdb/jenkins/percona-mongodb-mongosh.groovy +++ b/psmdb/jenkins/percona-mongodb-mongosh.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -30,9 +30,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/mongodb-js/mongosh.git', description: 'URL for percona-mongodb-backup repository', @@ -54,7 +58,7 @@ pipeline { description: 'VERSION value', name: 'VERSION') string( - defaultValue: 'psmdb-70', + defaultValue: 'psmdb-80', description: 'PSMDB repo name', name: 'PSMDB_REPO') choice( @@ -70,6 +74,9 @@ pipeline { } stages { stage('Create MongoDB Shell source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${MONGOSH_GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() @@ -86,144 +93,218 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build MongoDB Shell RPMs/DEBs/Binary tarballs') { parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_mongosh=1 --build_variant=rpm-x64") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:8", "--build_mongosh=1 --build_variant=rpm-arm64") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_mongosh=1 --build_variant=rpm-x64") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_mongosh=1 --build_variant=rpm-arm64") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + /* stage('Oracle Linux 10(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_mongosh=1 --build_variant=rpm-x64") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } */ // Commented out in scope of PKG-1083 + /* stage('Oracle Linux 10(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_mongosh=1 --build_variant=rpm-arm64") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } */ // Commented out in scope of PKG-1083 + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_mongosh=1 --build_variant=rpm-x64") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_mongosh=1 --build_variant=deb-x64") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_mongosh=1 --build_variant=rpm-arm64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy (22.04)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy (22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_mongosh=1 --build_variant=deb-arm64") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble (24.04)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble (24.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:noble", "--build_mongosh=1 --build_variant=deb-arm64") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Debian Bullseye (11)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bookworm(12)') { + stage('Debian Bookworm (12)(x86_64)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_mongosh=1 --build_variant=deb-x64") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oraclelinux 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_mongosh=1 --build_variant=linux-x64") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } + stage('Upload packages and tarballs from S3') { + agent { + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } @@ -232,7 +313,7 @@ pipeline { success { slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${MONGOSH_GIT_BRANCH} - [${BUILD_URL}]") script { - currentBuild.description = "Built on ${MONGOSH_GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + currentBuild.description = "[${CLOUD}]: Built on ${MONGOSH_GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } deleteDir() } diff --git a/psmdb/jenkins/percona-mongodb-mongosh.yml b/psmdb/jenkins/percona-mongodb-mongosh.yml index 29b08536db..97a124b76c 100644 --- a/psmdb/jenkins/percona-mongodb-mongosh.yml +++ b/psmdb/jenkins/percona-mongodb-mongosh.yml @@ -1,5 +1,5 @@ - job: - name: mongosh-autobuild-RELEASE + name: hetzner-mongosh-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-mongodb-mongosh.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy b/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy index ef63e85f23..a47b8eb7ba 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-3.6.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -90,36 +94,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -128,105 +132,105 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Xenial(16.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Stretch(9)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:stretch", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-3.6.yml b/psmdb/jenkins/percona-server-for-mongodb-3.6.yml index 68520d2e73..4d0a89e195 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-3.6.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-3.6.yml @@ -1,5 +1,5 @@ - job: - name: psmdb36-autobuild-RELEASE + name: hetzner-psmdb36-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-3.6.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy index 4aa3de0be0..822526d8b7 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-4.0.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -86,36 +90,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -124,92 +128,92 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.0.yml b/psmdb/jenkins/percona-server-for-mongodb-4.0.yml index ea731cc559..57251e7b4b 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-4.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb40-autobuild-RELEASE + name: hetzner-psmdb40-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-4.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy b/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy index efce86b7e2..e67429d901 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-4.2.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -86,36 +90,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -124,130 +128,130 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug binary tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Jammy(22.04) debug binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.2.yml b/psmdb/jenkins/percona-server-for-mongodb-4.2.yml index c6d928738d..7d3aa10dd6 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.2.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-4.2.yml @@ -1,5 +1,5 @@ - job: - name: psmdb42-autobuild-RELEASE + name: hetzner-psmdb42-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-4.2.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy b/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy index a0d87bb204..3be21d9b09 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-4.4.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -70,7 +74,7 @@ pipeline { stages { stage('Create PSMDB source tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -89,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -127,130 +131,130 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 binary tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Centos 7 debug binary tarball(glibc2.17)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Jammy(22.04) debug binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--debug=1") - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-4.4.yml b/psmdb/jenkins/percona-server-for-mongodb-4.4.yml index 9057a22cd7..2177c434a8 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-4.4.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-4.4.yml @@ -1,5 +1,5 @@ - job: - name: psmdb44-autobuild-RELEASE + name: hetzner-psmdb44-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-4.4.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy index e584b3f4c3..f12609d1a0 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-5.0.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -70,7 +74,7 @@ pipeline { stages { stage('Create PSMDB source tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") @@ -89,36 +93,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -127,132 +131,132 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oraclelinux 8 binary tarball(glibc2.28)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } stage('Debian Bullseye(11) binary tarball(glibc2.31)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') } } } diff --git a/psmdb/jenkins/percona-server-for-mongodb-5.0.yml b/psmdb/jenkins/percona-server-for-mongodb-5.0.yml index fb9a3b3ddc..f5977e245a 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-5.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-5.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb50-autobuild-RELEASE + name: hetzner-psmdb50-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-5.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy index 05fa4f368d..ed0a684dae 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-6.0.groovy @@ -1,22 +1,39 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ set -o xtrace - cd \${build_dir} - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + ls -laR ./ + rm -rf test/* + mkdir -p test + if [ \${FULL_FEATURED} = "yes" ]; then + PRO_BRANCH="v6.0" + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o psmdb_builder.sh \ + "https://api.github.com/repos/percona/mongo-build-scripts/contents/scripts/psmdb_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" """ + } } void cleanUpWS() { @@ -29,9 +46,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -58,12 +79,36 @@ pipeline { name: 'PSMDB_REPO') choice( choices: 'no\nyes', - description: 'Enable fipsmode', - name: 'FIPSMODE') + description: 'Enable all pro features', + name: 'FULL_FEATURED') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PSMDB_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') + choice( + name: 'DEBUG', + choices: ['no','yes'], + description: 'Additionally build debug image') + choice( + name: 'TESTS', + choices: ['yes','no'], + description: 'Run tests after building') } options { skipDefaultCheckout() @@ -73,10 +118,22 @@ pipeline { } stages { stage('Create PSMDB source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("oraclelinux:8", "--get_sources=1") + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-60.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -90,314 +147,398 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_src_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_src_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_src_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_src_deb=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_src_deb=1") + buildStage("ubuntu:jammy", "--build_src_deb=1") } } - - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PSMDB RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 8(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Amazon Linux 2023') { + stage('Amazon Linux 2023(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { buildStage("amazonlinux:2023", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Focal(20.04)') { + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_deb=1") + buildStage("amazonlinux:2023", "--build_rpm=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Noble(24.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bullseye", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bullseye", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bullseye", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Centos 8 binary tarball(glibc2.28)') { + stage('Oracle Linux 8 binary tarball(glibc2.28)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } - stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { - when { - expression { env.FIPSMODE != 'yes' } - } + stage('Amazon Linux 2023 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_tarball=1 --full_featured=1") + } else { + buildStage("amazonlinux:2023", "--build_tarball=1") + } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + } } } stage('Debian Bullseye(11) binary tarball(glibc2.31)') { when { - expression { env.FIPSMODE != 'yes' } + expression { env.FULL_FEATURED != 'yes' } } agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } + + stage('Upload packages and tarballs from S3') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { // Replace by a new procedure when it's ready - sync2PrivateProdAutoBuild(PSMDB_REPO+"-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, PSMDB_REPO+"-pro", COMPONENT) } else { - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } } } stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { try { - uploadTarballToDownloadsTesting("psmdb-gated", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb-gated", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -405,7 +546,7 @@ pipeline { } } else { try { - uploadTarballToDownloadsTesting("psmdb", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -415,13 +556,288 @@ pipeline { } } } + stage ('Build docker container for aws ecr') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && sudo apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | sudo docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-6.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sudo docker tag percona-server-mongodb public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + fi + """ + } + } + } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-6.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-6.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug percona/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage ('Run testing job') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TESTS', value: 'yes' + } + } + steps { + script { + def psmdb_image = 'percona/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + if ( params.PSMDB_REPO_TYPE == 'testing' ) { + psmdb_image = 'perconalab/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + } + if ( params.PSMDB_REPO_TYPE == 'experimental' ) { + psmdb_image = 'public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-' + params.PSMDB_VERSION + '-amd64' + } + def pbm_branch = sh(returnStdout: true, script: """ + git clone https://github.com/percona/percona-backup-mongodb.git >/dev/null 2>/dev/null + PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) + echo \$PBM_RELEASE + """).trim() + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + } + } + } } post { success { slackNotify("#releases", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + if (env.FULL_FEATURED == 'yes') { + currentBuild.description = "!!! PRO Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } else { + currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" + } } deleteDir() } diff --git a/psmdb/jenkins/percona-server-for-mongodb-6.0.yml b/psmdb/jenkins/percona-server-for-mongodb-6.0.yml index 043bffb44e..f0e2742c76 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-6.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-6.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb60-autobuild-RELEASE + name: hetzner-psmdb60-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-6.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy index 7968083597..33440f4cda 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-7.0.groovy @@ -1,22 +1,40 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ set -o xtrace - cd \${build_dir} - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" - """ + ls -laR ./ + rm -rf test/* + mkdir -p test + if [ \${FULL_FEATURED} = "yes" ]; then + PRO_BRANCH="v7.0" + curl -H "Authorization: token ${TOKEN}" https://api.github.com/user + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o psmdb_builder.sh \ + "https://api.github.com/repos/percona/mongo-build-scripts/contents/scripts/psmdb_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + """ + } } void cleanUpWS() { @@ -29,9 +47,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -58,12 +80,36 @@ pipeline { name: 'PSMDB_REPO') choice( choices: 'no\nyes', - description: 'Enable fipsmode', - name: 'FIPSMODE') + description: 'Enable all pro features', + name: 'FULL_FEATURED') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PSMDB_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') + choice( + name: 'DEBUG', + choices: ['no','yes'], + description: 'Additionally build debug image') + choice( + name: 'TESTS', + choices: ['yes','no'], + description: 'Run tests after building') } options { skipDefaultCheckout() @@ -73,13 +119,22 @@ pipeline { } stages { stage('Create PSMDB source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("oraclelinux:8", "--get_sources=1") + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-70.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -93,351 +148,431 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_src_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_src_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_src_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_src_deb=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_src_deb=1") + buildStage("ubuntu:jammy", "--build_src_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PSMDB RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - /* stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { buildStage("amazonlinux:2023", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } - }*/ - stage('Ubuntu Focal(20.04)') { + } + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_deb=1") + buildStage("amazonlinux:2023", "--build_rpm=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04)(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bullseye", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Bullseye(11)(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bullseye", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bullseye", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Debian Bookworm(12)') { + stage('Debian Bookworm(12)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oracle Linux 8 binary tarball(glibc2.28)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } - stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { - when { - expression { env.FIPSMODE != 'yes' } - } + stage('Amazon Linux 2023 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_tarball=1 --full_featured=1") + } else { + buildStage("amazonlinux:2023", "--build_tarball=1") + } + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) + } } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Debian Bullseye(11) binary tarball(glibc2.31)') { when { - expression { env.FIPSMODE != 'yes' } + expression { env.FULL_FEATURED != 'yes' } } agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) binary tarball(glibc2.36)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_tarball=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } + stage('Upload packages and tarballs from S3') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages script { - if (env.FIPSMODE == 'yes') { - sync2PrivateProdAutoBuild(PSMDB_REPO+"-pro", COMPONENT) + if (env.FULL_FEATURED == 'yes') { + sync2PrivateProdAutoBuild(params.CLOUD, PSMDB_REPO+"-pro", COMPONENT) } else { - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } } } stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { try { - uploadTarballToDownloadsTesting("psmdb-gated", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb-gated", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -445,7 +580,7 @@ pipeline { } } else { try { - uploadTarballToDownloadsTesting("psmdb", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -455,13 +590,284 @@ pipeline { } } } + stage ('Build docker containers for aws ecr') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && sudo apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | sudo docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-7.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sudo docker tag percona-server-mongodb public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + fi + """ + } + } + } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-7.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-7.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux + --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux + --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug percona/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage ('Run testing job') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TESTS', value: 'yes' + } + } + steps { + script { + def psmdb_image = 'percona/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + if ( params.PSMDB_REPO_TYPE == 'testing' ) { + psmdb_image = 'perconalab/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + } + if ( params.PSMDB_REPO_TYPE == 'experimental' ) { + psmdb_image = 'public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-' + params.PSMDB_VERSION + '-amd64' + } + def pbm_branch = sh(returnStdout: true, script: """ + git clone https://github.com/percona/percona-backup-mongodb.git >/dev/null 2>/dev/null + PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) + echo \$PBM_RELEASE + """).trim() + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + } + } + } } post { success { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { currentBuild.description = "!!! PRO Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } else { currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" diff --git a/psmdb/jenkins/percona-server-for-mongodb-7.0.yml b/psmdb/jenkins/percona-server-for-mongodb-7.0.yml index bb8bbcadb9..d8e1958974 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-7.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-7.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb70-autobuild-RELEASE + name: hetzner-psmdb70-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-7.0.groovy diff --git a/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy b/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy index 490334a8a1..a0cba0520c 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy +++ b/psmdb/jenkins/percona-server-for-mongodb-8.0.groovy @@ -1,22 +1,37 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ set -o xtrace - cd \${build_dir} - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" - """ + mkdir test + if [ \${FULL_FEATURED} = "yes" ]; then + PRO_BRANCH="v8.0" + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o psmdb_builder.sh \ + "https://api.github.com/repos/percona/mongo-build-scripts/contents/scripts/psmdb_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/scripts/psmdb_builder.sh -O psmdb_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FULL_FEATURED} = "yes" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/mongo-build-scripts.git percona-packaging + mv -f \${build_dir}/percona-packaging \${build_dir}/test/. + ls -la \${build_dir}/percona-packaging + fi + bash -x ./psmdb_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --psm_ver=${PSMDB_VERSION} --psm_release=${PSMDB_RELEASE} --mongo_tools_tag=${MONGO_TOOLS_TAG} ${STAGE_PARAM}" + """ + } } void cleanUpWS() { @@ -29,9 +44,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'micro-amazon' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'micro-amazon' } parameters { + choice( + choices: ['Hetzner','AWS'], + description: 'Cloud infra for build', + name: 'CLOUD') string( defaultValue: 'https://github.com/percona/percona-server-mongodb.git', description: 'URL for percona-server-mongodb repository', @@ -58,12 +77,36 @@ pipeline { name: 'PSMDB_REPO') choice( choices: 'no\nyes', - description: 'Enable fipsmode', - name: 'FIPSMODE') + description: 'Enable all pro features', + name: 'FULL_FEATURED') choice( choices: 'laboratory\ntesting\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') + choice( + name: 'BUILD_DOCKER', + choices: ['true', 'false'], + description: 'Build and push Docker images (default: true)') + choice( + name: 'BUILD_PACKAGES', + choices: ['true', 'false'], + description: 'Build packages (default: true)') + choice( + name: 'TARGET_REPO', + choices: ['PerconaLab','DockerHub'], + description: 'Target repo for docker image, use DockerHub for release only') + choice( + name: 'PSMDB_REPO_TYPE', + choices: ['testing','release','experimental'], + description: 'Packages repo for docker images') + choice( + name: 'DEBUG', + choices: ['no','yes'], + description: 'Additionally build debug image') + choice( + name: 'TESTS', + choices: ['yes','no'], + description: 'Run tests after building') } options { skipDefaultCheckout() @@ -73,13 +116,22 @@ pipeline { } stages { stage('Create PSMDB source tarball') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("oraclelinux:8", "--get_sources=1") + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--get_sources=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-server-mongodb-80.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -93,360 +145,400 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PSMDB generic source packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { stage('Build PSMDB generic source rpm') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_src_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_src_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_src_rpm=1") } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PSMDB generic source deb') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_src_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_src_deb=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_src_deb=1") + buildStage("ubuntu:jammy", "--build_src_deb=1") } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PSMDB RPMs/DEBs/Binary tarballs') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } parallel { - stage('Oracle Linux 8') { + stage('Oracle Linux 8(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 8(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:8", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 9(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } } - - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - /* stage('Amazon Linux 2023') { + stage('Oracle Linux 9(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_rpm=1 --full_featured=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + } + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Amazon Linux 2023(x86_64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { buildStage("amazonlinux:2023", "--build_rpm=1") } } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } - }*/ - stage('Ubuntu Focal(20.04)') { + } + stage('Amazon Linux 2023(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_rpm=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_deb=1") + buildStage("amazonlinux:2023", "--build_rpm=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Ubuntu Jammy(22.04)') { + stage('Ubuntu Jammy(22.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Ubuntu Jammy(22.04)(aarch64)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + script { + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_deb=1 --full_featured=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + } + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } - stage('Ubuntu Noble(24.04)') { + stage('Ubuntu Noble(24.04)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } -/* - stage('Debian Bullseye(11)') { + stage('Ubuntu Noble(24.04)(aarch64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bullseye", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_deb=1 --full_featured=1") } else { - buildStage("debian:bullseye", "--build_deb=1") + buildStage("ubuntu:noble", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } -*/ - stage('Debian Bookworm(12)') { + stage('Debian Bookworm(12)(x86_64)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_deb=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_deb=1") } } - - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Oracle Linux 8 binary tarball(glibc2.28)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:8", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:8", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:8", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("oraclelinux:9", "--build_tarball=1 --full_featured=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } - stage('Ubuntu Focal(20.04) binary tarball(glibc2.31)') { + stage('Amazon Linux 2023 binary tarball(glibc2.34)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:focal", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("amazonlinux:2023", "--build_tarball=1 --full_featured=1") } else { - buildStage("ubuntu:focal", "--build_tarball=1") + buildStage("amazonlinux:2023", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04) binary tarball(glibc2.35)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:jammy", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) binary tarball(glibc2.39)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("ubuntu:noble", "--build_tarball=1 --full_featured=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } -/* - stage('Ubuntu Jammy(22.04) debug binary tarball(glibc2.35)') { - agent { - label 'docker-64gb' - } - steps { - cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - script { - if (env.FIPSMODE == 'yes') { - buildStage("ubuntu:jammy", "--debug=1 --enable_fipsmode=1") - } else { - buildStage("ubuntu:jammy", "--debug=1") - } - pushArtifactFolder("debug/", AWS_STASH_PATH) - } - } - } -*/ stage('Debian Bookworm(12) binary tarball(glibc2.36)') { agent { - label 'docker-64gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { - if (env.FIPSMODE == 'yes') { - buildStage("debian:bookworm", "--build_tarball=1 --enable_fipsmode=1") + if (env.FULL_FEATURED == 'yes') { + buildStage("debian:bookworm", "--build_tarball=1 --full_featured=1") } else { buildStage("debian:bookworm", "--build_tarball=1") } - pushArtifactFolder("tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "tarball/", AWS_STASH_PATH) } } } } } + stage('Upload packages and tarballs from S3') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-64gb' + } + steps { + cleanUpWS() + + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "tarball/", AWS_STASH_PATH, 'binary') + } + } + stage('Sign packages') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { signRPM() signDEB() } } stage('Push to public repository') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { // sync packages script { - if (env.FIPSMODE == 'yes') { - sync2PrivateProdAutoBuild(PSMDB_REPO+"-pro", COMPONENT) + if (env.FULL_FEATURED == 'yes') { + sync2PrivateProdAutoBuild(params.CLOUD, PSMDB_REPO+"-pro", COMPONENT) } else { - sync2ProdAutoBuild(PSMDB_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PSMDB_REPO, COMPONENT) } } } } stage('Push Tarballs to TESTING download area') { + when { + expression { return params.BUILD_PACKAGES == 'true' } + } steps { script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { try { - uploadTarballToDownloadsTesting("psmdb-gated", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb-gated", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -454,7 +546,7 @@ pipeline { } } else { try { - uploadTarballToDownloadsTesting("psmdb", "${PSMDB_VERSION}") + uploadTarballToDownloadsTesting(params.CLOUD, "psmdb", "${PSMDB_VERSION}") } catch (err) { echo "Caught: ${err}" @@ -464,13 +556,282 @@ pipeline { } } } + stage ('Build docker container for aws ecr') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'AWS_ECR' + } + } + steps { + withCredentials([aws(accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '8468e4e0-5371-4741-a9bb-7c143140acea', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) { + sh """ + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + if [ -f "/usr/bin/yum" ] ; then sudo yum install -y unzip ; else sudo apt-get update && sudo apt-get -y install unzip ; fi + unzip -o awscliv2.zip + sudo ./aws/install + aws ecr-public get-login-password --region us-east-1 | sudo docker login --username AWS --password-stdin public.ecr.aws/e7j3v3n0 + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-8.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + sudo docker tag percona-server-mongodb public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-amd64 + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + sudo docker push public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-${PSMDB_VERSION}-debug + fi + """ + } + } + } + stage('Build docker containers for PerconaLab') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'PerconaLab' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-8.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + echo "Building ARM image" + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} perconalab/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push perconalab/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push perconalab/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + ''' + } + } + } + } + } + stage('Build docker containers for DockerHub registry') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TARGET_REPO', value: 'DockerHub' + } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FULL_FEATURED == 'yes') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1500 + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-server-mongodb-8.0 + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t percona-server-mongodb-amd64 -f Dockerfile . + sudo docker tag percona-server-mongodb-amd64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + + if [ ${DEBUG} = "yes" ]; then + sed -E "s/FROM percona(.+)/FROM percona-server-mongodb/" -i Dockerfile.debug + sudo docker build . -f Dockerfile.debug --no-cache --platform "linux/amd64" -t percona-server-mongodb-debug + fi + + sed -i "s/ENV PSMDB_VERSION.*/ENV PSMDB_VERSION ${PSMDB_VERSION}-${PSMDB_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PSMDB_REPO.*/ENV PSMDB_REPO ${PSMDB_REPO_TYPE}/g" Dockerfile.aarch64 + sudo docker build --no-cache --platform "linux/arm64" -t percona-server-mongodb-arm64 -f Dockerfile.aarch64 . + sudo docker tag percona-server-mongodb-arm64 percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + + PSMDB_MAJOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f1) + PSMDB_MINOR_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f2) + PSMDB_PATCH_VERSION=$(echo $PSMDB_VERSION | cut -d'.' -f3) + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + + sudo docker manifest create --amend percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 \ + percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} percona/percona-server-mongodb:${PSMDB_VERSION}-${PSMDB_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION}.${PSMDB_PATCH_VERSION} + sudo docker manifest push percona/percona-server-mongodb:${PSMDB_MAJOR_VERSION}.${PSMDB_MINOR_VERSION} + + if [ ${DEBUG} = "yes" ]; then + sudo docker tag percona-server-mongodb-debug percona/percona-server-mongodb:${PSMDB_VERSION}-debug + sudo docker push percona/percona-server-mongodb:${PSMDB_VERSION}-debug + fi + + ''' + } + } + } + } + } + stage ('Run testing job') { + when { + allOf { + expression { return params.BUILD_DOCKER == 'true' } + environment name: 'TESTS', value: 'yes' + } + } + steps { + script { + def psmdb_image = 'percona/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + if ( params.PSMDB_REPO_TYPE == 'testing' ) { + psmdb_image = 'perconalab/percona-server-mongodb:' + params.PSMDB_VERSION + '-amd64' + } + if ( params.PSMDB_REPO_TYPE == 'experimental' ) { + psmdb_image = 'public.ecr.aws/e7j3v3n0/psmdb-build:psmdb-' + params.PSMDB_VERSION + '-amd64' + } + def pbm_branch = sh(returnStdout: true, script: """ + git clone https://github.com/percona/percona-backup-mongodb.git >/dev/null 2>/dev/null + PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) + echo \$PBM_RELEASE + """).trim() + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + } + } + } } post { success { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") script { - if (env.FIPSMODE == 'yes') { + if (env.FULL_FEATURED == 'yes') { currentBuild.description = "!!! PRO Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" } else { currentBuild.description = "Built on ${GIT_BRANCH}. Path to packages: experimental/${AWS_STASH_PATH}" diff --git a/psmdb/jenkins/percona-server-for-mongodb-8.0.yml b/psmdb/jenkins/percona-server-for-mongodb-8.0.yml index a72222a31b..60a92cc4d0 100644 --- a/psmdb/jenkins/percona-server-for-mongodb-8.0.yml +++ b/psmdb/jenkins/percona-server-for-mongodb-8.0.yml @@ -1,5 +1,5 @@ - job: - name: psmdb80-autobuild-RELEASE + name: hetzner-psmdb80-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/jenkins/percona-server-for-mongodb-8.0.groovy diff --git a/psmdb/jenkins/psmdb_generate_sbom.groovy b/psmdb/jenkins/psmdb_generate_sbom.groovy new file mode 100644 index 0000000000..be242e7a80 --- /dev/null +++ b/psmdb/jenkins/psmdb_generate_sbom.groovy @@ -0,0 +1,412 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([ + string(credentialsId: 'SNYK_TOKEN', variable: 'SNYK_TOKEN'), + string(credentialsId: 'SNYK_ORG_TOKEN', variable: 'SNYK_ORG_TOKEN') + ]){ + sh """ + echo "Starting Build Stage" + set -o xtrace + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/percona-packaging/psmdb_sbom/psmdb_generate_sbom.sh -O psmdb_generate_sbom.sh + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} -e SNYK_TOKEN=${SNYK_TOKEN} -e SNYK_ORG_TOKEN=${SNYK_ORG_TOKEN} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + ls -laR + uname -a + bash -x ./psmdb_generate_sbom.sh --builddir=\${build_dir} --psmdb_version=${PSMDB_VERSION} --repo_type=${REPO_TYPE} --git_repo=${GIT_REPO} --git_branch=${GIT_BRANCH} ${STAGE_PARAM}" + curl -fsSL https://raw.githubusercontent.com/EvgeniyPatlan/sbom_verifier/main/install_sbom_verifier.sh | bash + bash sbom_verifier.sh psmdb_sbom/*.json + """ + } +} + +void uploadPSMDBSBOMToTestingDownloadServer(String productName, String packageVersion, String SBOMType) { + + script { + try { + uploadSBOMToDownloadsTesting(params.CLOUD, productName, packageVersion, SBOMType) + } catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def TIMESTAMP +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-server-mongodb.git', + description: 'URL for psmdb_sbom repository', + name: 'GIT_REPO') + string( + defaultValue: '7.0.18-11', + description: 'Version of Percona Server MongoDB', + name: 'PSMDB_VERSION') + string( + defaultValue: 'main', + description: 'Tag/Branch for percona-server-mongodb repository', + name: 'GIT_BRANCH') + choice( + choices: 'laboratory\ntesting\nexperimental\nrelease', + description: 'Packaging repository type', + name: 'REPO_TYPE') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + + stage('Create timestamp') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + TIMESTAMP = sh(script: 'date +%Y%m%d%H%M%S', returnStdout: true).trim() + sh """ + echo ${TIMESTAMP} > timestamp + cat timestamp + """ + TIMESTAMP = sh(returnStdout: true, script: "cat timestamp").trim() + } + stash includes: 'timestamp', name: 'timestamp' + } + } + + stage('Generate SBOM') { + parallel { + stage('Generate PSMDB SBOM OL/8 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + echo "Ran stash includes" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM OL/8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("oraclelinux:8", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM OL/9 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM OL/9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("oraclelinux:9", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM AL 2023 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("amazonlinux:2023", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM AL 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("amazonlinux:2023", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } +/* stage('Generate PSMDB SBOM OL/10 AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM OL/10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("oraclelinux:10", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } */ + stage('Generate PSMDB SBOM Jammy AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM Jammy ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("ubuntu:jammy", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM Noble AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM Noble ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("ubuntu:noble", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + stage('Generate PSMDB SBOM Bullseye AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + if (params.PSMDB_VERSION?.startsWith("7.0")) { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("debian:bullseye", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + } + stage('Generate PSMDB SBOM Bookworm AMD') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + script { + unstash 'timestamp' + AWS_STASH_PATH="/srv/UPLOAD/${REPO_TYPE}/BUILDS/PSMDB_SBOM/${PSMDB_VERSION}/${TIMESTAMP}" + sh """ + echo ${AWS_STASH_PATH} > uploadPath-${PSMDB_VERSION} + cat uploadPath-${PSMDB_VERSION} + """ + stash includes: "uploadPath-${PSMDB_VERSION}", name: "uploadPath-${PSMDB_VERSION}" + buildStage("debian:bookworm", "") + pushArtifactFolder(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH) + } + } + } + } //parallel + } // stage + stage('Upload SBOMS from AWS') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker' + } + steps { + cleanUpWS() + uploadSBOMfromAWS(params.CLOUD, "psmdb_sbom/", AWS_STASH_PATH, "json", "${PSMDB_VERSION}") + } + } + stage('Push SBOMS to TESTING downloads area') { + steps { + cleanUpWS() + uploadPSMDBSBOMToTestingDownloadServer("psmdb_sbom", "${PSMDB_VERSION}", "json") + } + } + } + post { + success { + slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + script { + currentBuild.description = "Built on ${GIT_BRANCH}" + } + deleteDir() + } + failure { + slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + deleteDir() + } + } +} diff --git a/psmdb/jenkins/psmdb_generate_sbom.yml b/psmdb/jenkins/psmdb_generate_sbom.yml new file mode 100644 index 0000000000..c570dd7573 --- /dev/null +++ b/psmdb/jenkins/psmdb_generate_sbom.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-psmdb_generate_sbom + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: psmdb/jenkins/psmdb_generate_sbom.groovy + diff --git a/psmdb/psmdb-docker-arm.groovy b/psmdb/psmdb-docker-arm.groovy index a35ffdae21..81d70f5748 100644 --- a/psmdb/psmdb-docker-arm.groovy +++ b/psmdb/psmdb-docker-arm.groovy @@ -5,12 +5,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker-64gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-64gb-aarch64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PSMDB_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PSMDB_VERSION', defaultValue: '6.0.2-1', description: 'PSMDB version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') @@ -234,7 +235,7 @@ pipeline { PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) echo \$PBM_RELEASE """).trim() - build job: 'pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'instance', value: 'docker-64gb-aarch64'), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'instance', value: 'docker-aarch64'), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}"), booleanParam(name: 'ADD_JENKINS_MARKED_TESTS', value: true)] } } } diff --git a/psmdb/psmdb-docker-arm.yml b/psmdb/psmdb-docker-arm.yml index 1c2c82a79d..ae1c4ac69c 100644 --- a/psmdb/psmdb-docker-arm.yml +++ b/psmdb/psmdb-docker-arm.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker-arm + name: hetzner-psmdb-docker-arm project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker-arm.groovy diff --git a/psmdb/psmdb-docker-cve.groovy b/psmdb/psmdb-docker-cve.groovy index 477ff63785..a51eb7a2a7 100644 --- a/psmdb/psmdb-docker-cve.groovy +++ b/psmdb/psmdb-docker-cve.groovy @@ -5,7 +5,7 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'master' + label 'launcher-x64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' diff --git a/psmdb/psmdb-docker-cve.yml b/psmdb/psmdb-docker-cve.yml index 003b98c628..501b6e11b3 100644 --- a/psmdb/psmdb-docker-cve.yml +++ b/psmdb/psmdb-docker-cve.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker-cve + name: hetzner-psmdb-docker-cve project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker-cve.groovy diff --git a/psmdb/psmdb-docker-pro.groovy b/psmdb/psmdb-docker-pro.groovy index 8acfc40e4a..a630e184b4 100644 --- a/psmdb/psmdb-docker-pro.groovy +++ b/psmdb/psmdb-docker-pro.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) @@ -7,16 +7,21 @@ def REPO_CI_PATH pipeline { agent { - label params.ARCH == 'x86_64' ? 'docker' : 'docker-64gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PSMDB_REPO', choices: ['release','testing','experimental'], description: 'percona-release repo to take packages from') string(name: 'PSMDB_VERSION', defaultValue: '6.0.18-15', description: 'PSMDB version, for example: 6.0.18-15 or 7.0.14-8') - string(name: 'TICKET_NAME', defaultValue: '', description: 'Target folder for docker image') - choice(name: 'ARCH', choices: ['x86_64','aarch64'], description: 'Docker arch') + choice(name: 'TARGET_REPO', choices: ['DockerHub','REPO.CI'], description: 'Target destination for docker image, use Percona for release only') + string(name: 'TICKET_NAME', defaultValue: '', description: 'Target folder on repo.ci for docker image if TARGET_REPO is repo.ci') + string(name: 'PD_BRANCH', defaultValue: '', description: 'Branch/commit from percona-docker repository to be used for building images') + booleanParam(name: 'UPDATE_MAJ_REL_TAG', defaultValue: false, description: 'Update major release tag (6.0 for 6.0.20) in DockerHub') + booleanParam(name: 'UPDATE_LATEST_TAG', defaultValue: false, description: 'Update \'latest\' tag in DockerHub') + booleanParam(name: 'IGNORE_TRIVY', defaultValue: false, description: 'Push images despite failed trivy check, use with caution') } options { disableConcurrentBuilds() @@ -29,40 +34,77 @@ pipeline { } } } + stage('Prepare environment') { + steps { + script { + if (params.CLOUD == 'Hetzner') { + sh ''' + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common apparmor apparmor-utils + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo mkdir -p /usr/libexec/docker/cli-plugins + LATEST=\$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\\1/') + sudo curl -L "https://github.com/docker/buildx/releases/download/v\${LATEST}/buildx-v\${LATEST}.linux-amd64" \\ + -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ''' + } else { + sh ''' + sudo yum install -y ca-certificates curl gnupg2 git qemu qemu-user-static || true + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ''' + } + } + } + } stage ('Build image') { steps { withCredentials([usernamePassword(credentialsId: 'PSMDB_PRIVATE_REPO_ACCESS', passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { sh """ - MAJ_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "." '{print \$1"."\$2}') + MAJ_VER=\$(echo "${params.PSMDB_VERSION}" | awk -F "." '{print \$1"."\$2}') echo \$MAJ_VER - MIN_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "-" '{print \$1}') + MIN_VER=\$(echo "${params.PSMDB_VERSION}" | awk -F "-" '{print \$1}') echo \$MIN_VER git clone https://github.com/percona/percona-docker cd percona-docker/percona-server-mongodb-\$MAJ_VER - sed -E "s|ENV PSMDB_VERSION (.+)|ENV PSMDB_VERSION ${params.PSMDB_VERSION}|" -i Dockerfile - sed -E "s|ENV PSMDB_REPO (.+)|ENV PSMDB_REPO ${params.PSMDB_REPO}|" -i Dockerfile - REPO_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "." '{print \$1\$2}') - OS_VER=\$(grep -P "ENV OS_VER" Dockerfile | cut -d " " -f 3) - echo \$OS_VER - OS_VER_NUM=\$(echo \$OS_VER | sed "s|el||") - echo \$OS_VER_NUM - PKG_URL="http://repo.percona.com/private/${USERNAME}-${PASSWORD}/psmdb-\$REPO_VER-pro/yum/${params.PSMDB_REPO}/\${OS_VER_NUM}/RPMS/${ARCH}/" - echo \$PKG_URL - sed -E "/curl -Lf -o \\/tmp\\/Percona-Server-MongoDB-server.rpm/d" -i Dockerfile - sed -E "/percona-server-mongodb-mongos-\${FULL_PERCONA_VERSION}/d" -i Dockerfile - sed -E "/percona-server-mongodb-tools-\${FULL_PERCONA_VERSION}/d" -i Dockerfile - sed -E "s|rpmkeys --checksig /tmp/Percona-Server-MongoDB-server.rpm|rpmkeys --checksig /tmp/*.rpm|" -i Dockerfile - sed -E "s|rpm -iv /tmp/Percona-Server-MongoDB-server.rpm --nodeps;|dnf -y install /tmp/Percona-Server-MongoDB-mongos-pro.rpm /tmp/Percona-Server-MongoDB-tools.rpm; rpm -iv /tmp/Percona-Server-MongoDB-server-pro.rpm --nodeps;|" -i Dockerfile - sed -E "s|rm -rf /tmp/Percona-Server-MongoDB-server.rpm|rm -rf /tmp/*.rpm|" -i Dockerfile - sed -E "s|server/LICENSE|server-pro/LICENSE|" -i Dockerfile - - mkdir -p packages - curl -Lf -o ./packages/Percona-Server-MongoDB-server-pro.rpm \$PKG_URL/percona-server-mongodb-server-pro-${params.PSMDB_VERSION}.\$OS_VER.${ARCH}.rpm - curl -Lf -o ./packages/Percona-Server-MongoDB-mongos-pro.rpm \$PKG_URL/percona-server-mongodb-mongos-pro-${params.PSMDB_VERSION}.\$OS_VER.${ARCH}.rpm - curl -Lf -o ./packages/Percona-Server-MongoDB-tools.rpm \$PKG_URL/percona-server-mongodb-tools-${params.PSMDB_VERSION}.\$OS_VER.${ARCH}.rpm - sed -E '/^ARG PERCONA_TELEMETRY_DISABLE/a \\\nCOPY packages/*.rpm /tmp/' -i Dockerfile - cat Dockerfile - docker build . -t percona-server-mongodb-pro:${params.PSMDB_VERSION} + if [ -n "${params.PD_BRANCH}" ]; then + git checkout ${params.PD_BRANCH} + fi + for ARCH in x86_64 aarch64; do + FILENAME=\$(if [ \"\$ARCH\" = \"x86_64\" ]; then echo \"Dockerfile\"; else echo \"Dockerfile.aarch64\"; fi) + sed -E "s|ENV PSMDB_VERSION (.+)|ENV PSMDB_VERSION ${params.PSMDB_VERSION}|" -i "\$FILENAME" + sed -E "s|ENV PSMDB_REPO (.+)|ENV PSMDB_REPO ${params.PSMDB_REPO}|" -i "\$FILENAME" + REPO_VER=\$(echo "${params.PSMDB_VERSION}" | awk -F "." '{print \$1\$2}') + OS_VER=\$(grep -P "ENV OS_VER" Dockerfile | cut -d " " -f 3) + echo \$OS_VER + OS_VER_NUM=\$(echo \$OS_VER | sed "s|el||") + echo \$OS_VER_NUM + PKG_URL="http://repo.percona.com/private/${USERNAME}-${PASSWORD}/psmdb-\$REPO_VER-pro/yum/${params.PSMDB_REPO}/\${OS_VER_NUM}/RPMS/\${ARCH}/" + echo \$PKG_URL + sed -E "/curl -Lf -o \\/tmp\\/Percona-Server-MongoDB-server.rpm/d" -i "\$FILENAME" + sed -E "/percona-server-mongodb-mongos-\${FULL_PERCONA_VERSION}/d" -i "\$FILENAME" + sed -E "/percona-server-mongodb-tools-\${FULL_PERCONA_VERSION}/d" -i "\$FILENAME" + sed -E "s|mongosh |mongosh percona-telemetry-agent |" -i "\$FILENAME" + sed -E "s|rpmkeys --checksig /tmp/Percona-Server-MongoDB-server.rpm|rpmkeys --checksig /tmp/*.rpm|" -i "\$FILENAME" + sed -E "s|rpm -iv /tmp/Percona-Server-MongoDB-server.rpm --nodeps;|rpm -iv /tmp/Percona-Server-MongoDB-mongos-pro.rpm /tmp/Percona-Server-MongoDB-tools.rpm; rpm -iv /tmp/Percona-Server-MongoDB-server-pro.rpm --nodeps;|" -i "\$FILENAME" + sed -E "s|rm -rf /tmp/Percona-Server-MongoDB-server.rpm|rm -rf /tmp/*.rpm|" -i "\$FILENAME" + sed -E "s|server/LICENSE|server-pro/LICENSE|" -i "\$FILENAME" + mkdir -p packages + curl -Lf -o ./packages/Percona-Server-MongoDB-server-pro.rpm \$PKG_URL/percona-server-mongodb-server-pro-${params.PSMDB_VERSION}.\$OS_VER.\${ARCH}.rpm + curl -Lf -o ./packages/Percona-Server-MongoDB-mongos-pro.rpm \$PKG_URL/percona-server-mongodb-mongos-pro-${params.PSMDB_VERSION}.\$OS_VER.\${ARCH}.rpm + curl -Lf -o ./packages/Percona-Server-MongoDB-tools.rpm \$PKG_URL/percona-server-mongodb-tools-${params.PSMDB_VERSION}.\$OS_VER.\${ARCH}.rpm + sed -E '/^ARG PERCONA_TELEMETRY_DISABLE/a \\\nCOPY packages/*.rpm /tmp/' -i "\$FILENAME" + cat "\$FILENAME" + if [ "\$ARCH" = "x86_64" ]; then + docker build --no-cache -t percona-server-mongodb-pro-amd -f "\$FILENAME" . + else + docker build --no-cache -t percona-server-mongodb-pro-arm --platform="linux/arm64" -f "\$FILENAME" . + fi + done rm -rf packages """ } @@ -75,22 +117,16 @@ pipeline { try { sh """ TRIVY_VERSION=\$(curl --silent 'https://api.github.com/repos/aquasecurity/trivy/releases/latest' | grep '"tag_name":' | tr -d '"' | sed -E 's/.*v(.+),.*/\\1/') - if [[ ${ARCH} == "x86_64" ]]; then - TRIVY_ARCH="64bit" - elif [[ ${ARCH} == "aarch64" ]]; then - TRIVY_ARCH="ARM64" - fi - wget https://github.com/aquasecurity/trivy/releases/download/v\${TRIVY_VERSION}/trivy_\${TRIVY_VERSION}_Linux-\${TRIVY_ARCH}.tar.gz - sudo tar zxvf trivy_\${TRIVY_VERSION}_Linux-\${TRIVY_ARCH}.tar.gz -C /usr/local/bin/ - rm trivy_\${TRIVY_VERSION}_Linux-\${TRIVY_ARCH}.tar.gz + wget https://github.com/aquasecurity/trivy/releases/download/v\${TRIVY_VERSION}/trivy_\${TRIVY_VERSION}_Linux-64bit.tar.gz + sudo tar zxvf trivy_\${TRIVY_VERSION}_Linux-64bit.tar.gz -C /usr/local/bin/ wget https://raw.githubusercontent.com/aquasecurity/trivy/v\${TRIVY_VERSION}/contrib/junit.tpl curl https://raw.githubusercontent.com/Percona-QA/psmdb-testing/main/docker/trivyignore -o ".trivyignore" - if [ ${params.PSMDB_REPO} = "release" ]; then - /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-server-mongodb-pro:${params.PSMDB_VERSION} + if [ "${params.IGNORE_TRIVY}" = "true" ]; then + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-server-mongodb-pro-arm + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-server-mongodb-pro-amd else - /usr/local/bin/trivy -q image --format template --template @junit.tpl -o trivy-hight-junit.xml \ - --timeout 10m0s --ignore-unfixed --exit-code 0 --severity HIGH,CRITICAL percona-server-mongodb-pro:${params.PSMDB_VERSION} + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-server-mongodb-pro-arm + /usr/local/bin/trivy -q image --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL percona-server-mongodb-pro-amd fi """ } catch (Exception e) { @@ -101,17 +137,17 @@ pipeline { } } } - post { - always { - junit testResults: "*-junit.xml", keepLongStdio: true, allowEmptyResults: true, skipPublishingChecks: true - } - } } stage ('Push image to repo.ci') { + when { + environment name: 'TARGET_REPO', value: 'REPO.CI' + } steps { sh """ - docker save -o percona-server-mongodb-pro-${params.PSMDB_VERSION}.tar percona-server-mongodb-pro:${params.PSMDB_VERSION} - gzip percona-server-mongodb-pro-${params.PSMDB_VERSION}.tar + docker save -o percona-server-mongodb-pro-${params.PSMDB_VERSION}-amd64.tar percona-server-mongodb-pro-amd + docker save -o percona-server-mongodb-pro-${params.PSMDB_VERSION}-arm64.tar percona-server-mongodb-pro-arm + gzip percona-server-mongodb-pro-${params.PSMDB_VERSION}-amd64.tar + gzip percona-server-mongodb-pro-${params.PSMDB_VERSION}-arm64.tar echo "UPLOAD/experimental/CUSTOM/${TICKET_NAME}/${JOB_NAME}/${BUILD_NUMBER}" > uploadPath """ script { @@ -122,6 +158,79 @@ pipeline { uploadTarball('docker') } } + stage ('Push images to private percona docker registry') { + when { + environment name: 'TARGET_REPO', value: 'DockerHub' + } + steps { + withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { + sh """ + docker login -u '${USER}' -p '${PASS}' + MAJ_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "." '{print \$1"."\$2}') + MIN_VER=\$(echo ${params.PSMDB_VERSION} | awk -F "-" '{print \$1}') + + docker tag percona-server-mongodb-pro-amd percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 + docker tag percona-server-mongodb-pro-arm percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 + docker push percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 + docker push percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 + + docker tag percona-server-mongodb-pro-amd percona/percona-server-mongodb-pro:\$MIN_VER-amd64 + docker tag percona-server-mongodb-pro-arm percona/percona-server-mongodb-pro:\$MIN_VER-arm64 + docker push percona/percona-server-mongodb-pro:\$MIN_VER-amd64 + docker push percona/percona-server-mongodb-pro:\$MIN_VER-arm64 + + docker manifest create percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} \ + percona/percona-server-mongodb-pro:${params.PSMDB_VERSION}-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} + docker manifest push percona/percona-server-mongodb-pro:${params.PSMDB_VERSION} + + docker manifest create percona/percona-server-mongodb-pro:\$MIN_VER \ + percona/percona-server-mongodb-pro:\$MIN_VER-amd64 \ + percona/percona-server-mongodb-pro:\$MIN_VER-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:\$MIN_VER \ + percona/percona-server-mongodb-pro:\$MIN_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:\$MIN_VER \ + percona/percona-server-mongodb-pro:\$MIN_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:\$MIN_VER + docker manifest push percona/percona-server-mongodb-pro:\$MIN_VER + + if [ ${params.UPDATE_MAJ_REL_TAG} = "true" ]; then + docker tag percona-server-mongodb-pro-amd percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 + docker tag percona-server-mongodb-pro-arm percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + docker push percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 + docker push percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + + docker manifest create percona/percona-server-mongodb-pro:\$MAJ_VER \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:\$MAJ_VER \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:\$MAJ_VER \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:\$MAJ_VER + docker manifest push percona/percona-server-mongodb-pro:\$MAJ_VER + fi + + if [ "${params.UPDATE_MAJ_REL_TAG}" = "true" ] && [ "${params.UPDATE_LATEST_TAG}" = "true" ]; then + docker manifest create percona/percona-server-mongodb-pro:latest \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 + docker manifest annotate percona/percona-server-mongodb-pro:latest \ + percona/percona-server-mongodb-pro:\$MAJ_VER-arm64 --os linux --arch arm64 --variant v8 + docker manifest annotate percona/percona-server-mongodb-pro:latest \ + percona/percona-server-mongodb-pro:\$MAJ_VER-amd64 --os linux --arch amd64 + docker manifest inspect percona/percona-server-mongodb-pro:latest + docker manifest push percona/percona-server-mongodb-pro:latest + fi + """ + } + } + } } post { always { @@ -132,22 +241,13 @@ pipeline { deleteDir() } success { - slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: Building of PSMDB ${PSMDB_VERSION} repo ${PSMDB_REPO} succeed") - script { - currentBuild.description = "Built on ${PSMDB_VERSION}. Path to packages: ${REPO_CI_PATH}" - } + slackNotify("#mongodb_autofeed", "#00FF00", "[${JOB_NAME}]: Building of PSMDB PRO ${PSMDB_VERSION}, repo ${PSMDB_REPO} succeed") } unstable { - slackNotify("#releases-ci", "#F6F930", "[${JOB_NAME}]: Building of PSMDB ${PSMDB_VERSION} repo ${PSMDB_REPO} unstable - [${BUILD_URL}testReport/]") - script { - currentBuild.description = "Built on ${PSMDB_VERSION}. Path to packages: ${REPO_CI_PATH}" - } + slackNotify("#mongodb_autofeed", "#F6F930", "[${JOB_NAME}]: Building of PSMDB PRO ${PSMDB_VERSION}, repo ${PSMDB_REPO} unstable - [${BUILD_URL}testReport/]") } failure { - slackNotify("#releases-ci", "#FF0000", "[${JOB_NAME}]: Building of PSMDB ${PSMDB_VERSION} repo ${PSMDB_REPO} failed - [${BUILD_URL}]") - script { - currentBuild.description = "Built on ${PSMDB_VERSION}. Path to packages: ${REPO_CI_PATH}" - } + slackNotify("#mongodb_autofeed", "#FF0000", "[${JOB_NAME}]: Building of PSMDB PRO ${PSMDB_VERSION}, repo ${PSMDB_REPO} failed - [${BUILD_URL}]") } - } +} } diff --git a/psmdb/psmdb-docker-pro.yml b/psmdb/psmdb-docker-pro.yml index 3378230517..ac4cb1a4f6 100644 --- a/psmdb/psmdb-docker-pro.yml +++ b/psmdb/psmdb-docker-pro.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker-pro + name: hetzner-psmdb-docker-pro project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker-pro.groovy diff --git a/psmdb/psmdb-docker.groovy b/psmdb/psmdb-docker.groovy index cd545bda8a..2e77ff8a0c 100644 --- a/psmdb/psmdb-docker.groovy +++ b/psmdb/psmdb-docker.groovy @@ -5,12 +5,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') choice(name: 'PSMDB_REPO', choices: ['testing','release','experimental'], description: 'Percona-release repo') string(name: 'PSMDB_VERSION', defaultValue: '6.0.2-1', description: 'PSMDB version') choice(name: 'TARGET_REPO', choices: ['PerconaLab','AWS_ECR','DockerHub'], description: 'Target repo for docker image, use DockerHub for release only') @@ -168,7 +169,7 @@ pipeline { PBM_RELEASE=\$(cd percona-backup-mongodb && git branch -r | grep release | sed 's|origin/||' | sort --version-sort | tail -1) echo \$PBM_RELEASE """).trim() - build job: 'pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}")] + build job: 'hetzner-pbm-functional-tests', propagate: false, wait: false, parameters: [string(name: 'PBM_BRANCH', value: pbm_branch ), string(name: 'PSMDB', value: psmdb_image ), string(name: 'TESTING_BRANCH', value: "pbm-${pbm_branch}"), booleanParam(name: 'ADD_JENKINS_MARKED_TESTS', value: true)] } } } diff --git a/psmdb/psmdb-docker.yml b/psmdb/psmdb-docker.yml index ed4f6d22eb..79ae959506 100644 --- a/psmdb/psmdb-docker.yml +++ b/psmdb/psmdb-docker.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-docker + name: hetzner-psmdb-docker project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-docker.groovy diff --git a/psmdb/psmdb-integration.groovy b/psmdb/psmdb-integration.groovy index a3549e276e..d0fe888ace 100644 --- a/psmdb/psmdb-integration.groovy +++ b/psmdb/psmdb-integration.groovy @@ -5,12 +5,14 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') + string(name: 'TEST_VERSION', defaultValue: 'main', description: 'Integration tests version: main, v3') string(name: 'PSMDB_VERSION', defaultValue: 'latest', description: 'PSMDB version') string(name: 'PBM_VERSION', defaultValue: 'latest', description: 'PBM version') string(name: 'PMM_VERSION', defaultValue: 'latest', description: 'PMM2 agent version') @@ -29,7 +31,7 @@ pipeline { -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token ${GITHUB_API_TOKEN}" \ "https://api.github.com/repos/Percona-Lab/qa-integration/actions/workflows/PMM_PSMDB_PBM.yml/dispatches" \ - -d '{"ref":"main","inputs":{"psmdb_version":"${params.PSMDB_VERSION}","pbm_version":"${params.PBM_VERSION}","pmm_version":"${params.PMM_VERSION}","pmm_repo":"${params.PMM_REPO}","pmm_image":"${params.PMM_IMAGE}"}}' + -d '{"ref":"${params.TEST_VERSION}","inputs":{"psmdb_version":"${params.PSMDB_VERSION}","pbm_version":"${params.PBM_VERSION}","pmm_version":"${params.PMM_VERSION}","pmm_repo":"${params.PMM_REPO}","pmm_image":"${params.PMM_IMAGE}"}}' """ } } diff --git a/psmdb/psmdb-integration.yml b/psmdb/psmdb-integration.yml index dcc0aee5a6..4e61b72568 100644 --- a/psmdb/psmdb-integration.yml +++ b/psmdb/psmdb-integration.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-integration + name: hetzner-psmdb-integration project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-integration.groovy diff --git a/psmdb/psmdb-multijob-testing.groovy b/psmdb/psmdb-multijob-testing.groovy index 98e94bd052..fc5acc2f66 100644 --- a/psmdb/psmdb-multijob-testing.groovy +++ b/psmdb/psmdb-multijob-testing.groovy @@ -5,12 +5,13 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'master' + label params.CLOUD == 'Hetzner' ? 'launcher-x64' : 'master' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice(name: 'CLOUD', choices: [ 'Hetzner','AWS' ], description: 'Cloud infra for build') string(name: 'PSMDB_VERSION', defaultValue: '4.4.17', description: 'PSMDB Version') string(name: 'PSMDB_RELEASE', defaultValue: '17', description: 'PSMDB Release') } @@ -32,12 +33,12 @@ pipeline { build job: 'psmdb-parallel', parameters: [ string(name: 'REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: params.PSMDB_VERSION ), string(name: 'ENABLE_TOOLKIT', value: "false"), string(name: 'TESTING_BRANCH', value: "main") ] } } - stage ('Build docker images and check for vulnerabilities') { - steps { + stage ('Build docker images and check for vulnerabilities') { + steps { script { - def version = params.PSMDB_VERSION + '-' + params.PSMDB_RELEASE - build job: 'psmdb-docker', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] - build job: 'psmdb-docker-arm', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] + def version = params.PSMDB_VERSION + '-' + params.PSMDB_RELEASE + build job: 'hetzner-psmdb-docker', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] + build job: 'hetzner-psmdb-docker-arm', parameters: [string(name: 'PSMDB_REPO', value: "testing"), string(name: 'PSMDB_VERSION', value: version ), string(name: 'TARGET_REPO', value: "PerconaLab") ] } } } @@ -45,8 +46,9 @@ pipeline { steps { script { def version = params.PSMDB_VERSION + '-' + params.PSMDB_RELEASE - build job: 'psmdb-integration', parameters: [string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "release"), string(name: 'PMM_IMAGE', value: "percona/pmm-server:latest") ] - build job: 'psmdb-integration', parameters: [string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "experimental"), string(name: 'PMM_IMAGE', value: "perconalab/pmm-server:dev-latest") ] + build job: 'hetzner-psmdb-integration', parameters: [string(name: 'TEST_VERSION', value: "main"), string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "release"), string(name: 'PMM_IMAGE', value: "percona/pmm-server:2") ] + build job: 'hetzner-psmdb-integration', parameters: [string(name: 'TEST_VERSION', value: "v3"), string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "release"), string(name: 'PMM_IMAGE', value: "percona/pmm-server:latest") ] + build job: 'hetzner-psmdb-integration', parameters: [string(name: 'TEST_VERSION', value: "v3"), string(name: 'PSMDB_VERSION', value: version), string(name: 'PBM_VERSION', value: "latest" ), string(name: 'PMM_VERSION', value: "latest"), string(name: 'PMM_REPO', value: "experimental"), string(name: 'PMM_IMAGE', value: "perconalab/pmm-server:3-dev-latest") ] } } } diff --git a/psmdb/psmdb-multijob-testing.yml b/psmdb/psmdb-multijob-testing.yml index 8e38ceaf9f..c2c79ee26c 100644 --- a/psmdb/psmdb-multijob-testing.yml +++ b/psmdb/psmdb-multijob-testing.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-multijob-testing + name: hetzner-psmdb-multijob-testing project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-multijob-testing.groovy diff --git a/psmdb/psmdb-regression.groovy b/psmdb/psmdb-regression.groovy index 387fba2469..467e016de3 100644 --- a/psmdb/psmdb-regression.groovy +++ b/psmdb/psmdb-regression.groovy @@ -1,11 +1,11 @@ -library changelog: false, identifier: "lib@master", retriever: modernSCM([ +library changelog: false, identifier: "lib@hetzner", retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) pipeline { agent { - label 'master' + label 'launcher-x64' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' @@ -22,13 +22,13 @@ pipeline { string(name: 'parallelexecutors', defaultValue: '1', description: 'Number of parallel executors') string(name: 'testsuites', defaultValue: 'core,unittests,dbtest', description: 'Comma-separated list of testuites') string(name: 'listsuites', defaultValue: '', description: 'URL with list of testuites') - choice(name: 'instance', choices: ['docker-64gb','docker-64gb-aarch64'], description: 'Ec2 instance type for running suites') + choice(name: 'instance', choices: ['docker-x64','docker-aarch64'], description: 'Hetzner instance type for running suites') string(name: 'paralleljobs', defaultValue: '4', description: 'Number of parallel jobs passed to resmoke.py') booleanParam(name: 'unittests',defaultValue: true, description: 'Check if list of suites contains unittests') booleanParam(name: 'integrationtests',defaultValue: false, description: 'Check if list of suites contains integration tests') booleanParam(name: 'benchmarktests',defaultValue: false, description: 'Check if list of suites contains benchmark tests') string(name: 'OS', defaultValue: 'debian:12', description: 'Base OS, can be changed to build the image for PBM tests') - string(name: 'scons_params', defaultValue: '--variables-files=gcc.vars --disable-warnings-as-errors --ssl --opt=size -j8 --use-sasl-client --wiredtiger --audit --inmemory --hotbackup ', description: 'Parameters for scons') + string(name: 'scons_params', defaultValue: '--variables-files=gcc.vars --disable-warnings-as-errors --ssl --link-model=dynamic --linker=gold --dbg=off --opt=off -j8 --use-sasl-client --wiredtiger --audit --inmemory --hotbackup ', description: 'Parameters for scons') string(name: 'resmoke_params', defaultValue: '--excludeWithAnyTags=featureFlagColumnstoreIndexes,featureFlagUpdateOneWithoutShardKey,featureFlagGlobalIndexesShardingCatalog,featureFlagGlobalIndexes,featureFlagTelemetry,featureFlagAuditConfigClusterParameter,serverless,does_not_support_config_fuzzer,featureFlagDeprioritizeLowPriorityOperations,featureFlagSbeFull,featureFlagQueryStats,featureFlagTransitionToCatalogShard,requires_latch_analyzer', description: 'Extra params passed to resmoke.py') } options { @@ -157,7 +157,7 @@ pipeline { sh """ echo "start suite ${suiteName}" docker run -v `pwd`/test_results:/work -w /work --rm -i ${image} bash -c 'rm -rf *' - docker run -v `pwd`/test_results:/work --rm ${image} bash -c "${script} && python buildscripts/resmoke.py run --suite ${suite} ${params.resmoke_params} --reportFile=/work/resmoke_${suiteName}_s.json > /work/resmoke_${suiteName}_s.log 2>&1" || true + docker run --ulimit memlock=-1 -v `pwd`/test_results:/work --rm ${image} bash -c "${script} && python buildscripts/resmoke.py run --suite ${suite} ${params.resmoke_params} --reportFile=/work/resmoke_${suiteName}_s.json > /work/resmoke_${suiteName}_s.log 2>&1" || true docker run -v `pwd`/test_results:/work -w /work --rm ${image} bash -c 'python /opt/percona-server-mongodb/resmoke2junit.py && chmod -R 777 /work' echo "finish suite ${suiteName}" """ @@ -166,7 +166,7 @@ pipeline { sh """ echo "start suite ${suiteName}" docker run -v `pwd`/test_results:/work -w /work --rm -i ${image} bash -c 'rm -rf *' - docker run -v `pwd`/test_results:/work --rm ${image} bash -c "python buildscripts/resmoke.py run --suite $suite ${params.resmoke_params} --reportFile=/work/resmoke_${suiteName}_s.json > /work/resmoke_${suiteName}_s.log 2>&1" || true + docker run --ulimit memlock=-1 -v `pwd`/test_results:/work --rm ${image} bash -c "python buildscripts/resmoke.py run --suite $suite ${params.resmoke_params} --reportFile=/work/resmoke_${suiteName}_s.json > /work/resmoke_${suiteName}_s.log 2>&1" || true docker run -v `pwd`/test_results:/work -w /work --rm ${image} bash -c 'python /opt/percona-server-mongodb/resmoke2junit.py && chmod -R 777 /work' echo "finish suite ${suiteName}" """ diff --git a/psmdb/psmdb-regression.yml b/psmdb/psmdb-regression.yml index 16682a50c4..c8098e9e41 100644 --- a/psmdb/psmdb-regression.yml +++ b/psmdb/psmdb-regression.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-regression + name: hetzner-psmdb-regression project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-regression.groovy diff --git a/psmdb/psmdb-site-check.groovy b/psmdb/psmdb-site-check.groovy index 6f27c7dfae..fe5d4ccbcf 100644 --- a/psmdb/psmdb-site-check.groovy +++ b/psmdb/psmdb-site-check.groovy @@ -5,12 +5,16 @@ library changelog: false, identifier: "lib@master", retriever: modernSCM([ pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } environment { PATH = '/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin' } parameters { + choice( + name: 'CLOUD', + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build') string( defaultValue: '7.0.2-1', description: 'PSMDB Version for tests', diff --git a/psmdb/psmdb-site-check.yml b/psmdb/psmdb-site-check.yml index e866533389..844ae07295 100644 --- a/psmdb/psmdb-site-check.yml +++ b/psmdb/psmdb-site-check.yml @@ -1,5 +1,5 @@ - job: - name: psmdb-site-check + name: hetzner-psmdb-site-check project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: psmdb/psmdb-site-check.groovy diff --git a/pt/jenkins/percona-toolkit.groovy b/pt/jenkins/percona-toolkit.groovy index 1d474c2636..057f6da2c4 100644 --- a/pt/jenkins/percona-toolkit.groovy +++ b/pt/jenkins/percona-toolkit.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -12,6 +12,7 @@ void buildStage(String DOCKER_OS, String STAGE_PARAM) { export build_dir=\$(pwd -P) docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " set -o xtrace + sed -i "s:archive.ubuntu.com:mirrors.edge.kernel.org:g" /etc/apt/sources.list cd \${build_dir} bash -x ./pt_builder.sh --builddir=\${build_dir}/test --install_deps=1 bash -x ./pt_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --version=${VERSION} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" @@ -28,19 +29,23 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-toolkit.git', description: 'URL for percona-toolkit repository', name: 'GIT_REPO') string( - defaultValue: 'v2.1', + defaultValue: 'release-3.7.0-1', description: 'Tag/Branch for percona toolkit repository', name: 'GIT_BRANCH') string( - defaultValue: '2.5.1', + defaultValue: '3.7.0', description: 'Percona Toolkit release value', name: 'VERSION') string( @@ -83,36 +88,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build Percona Toolkit generic source packages') { parallel { stage('Build Percona Toolkit generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build Percona Toolkit generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_src_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -121,197 +126,223 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Xenial(16.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } @@ -327,10 +358,92 @@ pipeline { stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(PT_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, PT_REPO, COMPONENT) } } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-toolkit + sed -i "s/ENV PT_VERSION.*/ENV PT_VERSION ${VERSION}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/pt release/pt ${COMPONENT}/g" Dockerfile + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} . + + sudo docker build --no-cache -t perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --platform="linux/arm64" -f Dockerfile . + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + sudo docker push perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} + sudo docker push perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + + PT_MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) + PT_MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) + PT_PATCH_VERSION=$(echo $VERSION | cut -d'.' -f3) + sudo docker manifest create perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} + sudo docker manifest create perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} + + sudo docker manifest create perconalab/percona-toolkit:${PT_MAJOR_VERSION} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} \ + perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-toolkit:${PT_MAJOR_VERSION} perconalab/percona-toolkit:${VERSION}-${RPM_RELEASE} --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-toolkit:${PT_MAJOR_VERSION} + + sudo docker manifest push perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION}.${PT_PATCH_VERSION} + sudo docker manifest push perconalab/percona-toolkit:${PT_MAJOR_VERSION}.${PT_MINOR_VERSION} + sudo docker manifest push perconalab/percona-toolkit:${PT_MAJOR_VERSION} + ''' + } + } + } + } } post { success { diff --git a/pt/jenkins/percona-toolkit.yml b/pt/jenkins/percona-toolkit.yml index 11407be6c7..b1dcb17f49 100644 --- a/pt/jenkins/percona-toolkit.yml +++ b/pt/jenkins/percona-toolkit.yml @@ -1,5 +1,5 @@ - job: - name: percona-toolkit-RELEASE + name: hetzner-percona-toolkit-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pt/jenkins/percona-toolkit.groovy diff --git a/pxb/jenkins/pxb-24-arm.groovy b/pxb/jenkins/pxb-24-arm.groovy index 30a83a4704..1d10e8513c 100644 --- a/pxb/jenkins/pxb-24-arm.groovy +++ b/pxb/jenkins/pxb-24-arm.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -61,7 +65,7 @@ pipeline { stages { stage('Create PXB source tarball') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -83,8 +87,8 @@ pipeline { XB_VERSION_EXTRA = sh(returnStdout: true, script: "grep 'XB_VERSION_EXTRA' ./test/percona-xtrabackup-2.4.properties | cut -d = -f 2 | sed 's/-//g'").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } @@ -92,15 +96,15 @@ pipeline { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } } //parallel @@ -110,28 +114,28 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } diff --git a/pxb/jenkins/pxb-24-arm.yml b/pxb/jenkins/pxb-24-arm.yml index ef14c24cb8..7e25a4e08d 100644 --- a/pxb/jenkins/pxb-24-arm.yml +++ b/pxb/jenkins/pxb-24-arm.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-2.4-arm-RELEASE + name: hetzner-percona-xtrabackup-2.4-arm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-24-arm.groovy diff --git a/pxb/jenkins/pxb-24.groovy b/pxb/jenkins/pxb-24.groovy index 153050250a..367fa548f8 100644 --- a/pxb/jenkins/pxb-24.groovy +++ b/pxb/jenkins/pxb-24.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -79,36 +83,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PXB generic source packages') { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXB generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -117,145 +121,145 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } /* stage('Amazon Linux 2023') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("amazonlinux:2023", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } }*/ stage('Ubuntu Bionic(18.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('OracleLinux 8 tarball') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } diff --git a/pxb/jenkins/pxb-24.yml b/pxb/jenkins/pxb-24.yml index e4bb139e2f..0f277a75f1 100644 --- a/pxb/jenkins/pxb-24.yml +++ b/pxb/jenkins/pxb-24.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-2.4-RELEASE + name: hetzner-percona-xtrabackup-2.4-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-24.groovy diff --git a/pxb/jenkins/pxb-80-arm.groovy b/pxb/jenkins/pxb-80-arm.groovy index a9da8a92d1..689ed997e9 100644 --- a/pxb/jenkins/pxb-80-arm.groovy +++ b/pxb/jenkins/pxb-80-arm.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -29,9 +29,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -61,7 +65,7 @@ pipeline { stages { stage('Create PXB source tarball') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { slackNotify("#releases-ci", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") @@ -83,8 +87,8 @@ pipeline { XB_VERSION_EXTRA = sh(returnStdout: true, script: "grep 'XB_VERSION_EXTRA' ./test/percona-xtrabackup-8.0.properties | cut -d = -f 2 | sed 's/-//g'").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } @@ -92,28 +96,28 @@ pipeline { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXB generic source deb') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -123,93 +127,93 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() popArtifactFolder("source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } diff --git a/pxb/jenkins/pxb-80-arm.yml b/pxb/jenkins/pxb-80-arm.yml index 552a406cf5..964bb5f4c1 100644 --- a/pxb/jenkins/pxb-80-arm.yml +++ b/pxb/jenkins/pxb-80-arm.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-8.0-arm-RELEASE + name: hetzner-percona-xtrabackup-8.0-arm-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-80-arm.groovy diff --git a/pxb/jenkins/pxb-80-docker.groovy b/pxb/jenkins/pxb-80-docker.groovy new file mode 100644 index 0000000000..0e1dcbd62c --- /dev/null +++ b/pxb/jenkins/pxb-80-docker.groovy @@ -0,0 +1,288 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +import groovy.transform.Field + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def minitestNodes = [ "min-bullseye-x64", + "min-bookworm-x64", + "min-centos-7-x64", + "min-ol-8-x64", + "min-focal-x64", + "min-jammy-x64", + "min-ol-9-x64" ] + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + choice( + choices: 'perconalab\npercona', + description: 'Organization on hub.docker.com', + name: 'ORGANIZATION') + string( + defaultValue: 'https://github.com/percona/percona-xtrabackup.git', + description: 'URL for PXB git repository', + name: 'GIT_REPO') + string( + defaultValue: '8.0', + description: 'Tag/Branch for PXB repository', + name: 'BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + choice( + choices: 'testing\nlaboratory\nexperimental', + description: 'Repo component to push packages to', + name: 'COMPONENT') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + } + stages { + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + echo "====> Build docker containers" + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + curl -O https://raw.githubusercontent.com/percona/percona-server/refs/heads/${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}/MYSQL_VERSION + . ./MYSQL_VERSION + rm -rf percona-docker + git clone https://github.com/percona/percona-docker + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + cd percona-docker/percona-xtrabackup-8.0 + else + cd percona-docker/percona-xtrabackup-8.x + fi + sed -i "s/ENV XTRABACKUP_VERSION.*/ENV XTRABACKUP_VERSION ${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.1/g" Dockerfile + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + sed -i "s/pxb-80 testing/pxb-80 ${COMPONENT}/g" Dockerfile + sed -i "s/ps-80;/ps-80 ${COMPONENT};/g" Dockerfile + else + sed -i "s/pxb-84-lts testing/pxb-84-lts ${COMPONENT}/g" Dockerfile + sed -i "s/ps-84-lts release/ps-84-lts ${COMPONENT}/g" Dockerfile + fi + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --platform "linux/amd64" . + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" . + else + sudo docker pull perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker tag perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker pull perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker tag perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + fi + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + + sudo docker manifest create perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker manifest push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:latest perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + else + sudo docker push percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + + sudo docker manifest create percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} \ + percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker manifest push percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + + sudo docker buildx imagetools create -t percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA} percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH} percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR} percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + if [ ${XB_VERSION_MINOR} = "0" ]; then + sudo docker buildx imagetools create -t percona/percona-xtrabackup:latest percona/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + fi + fi + ''' + } + } // scripts + } // steps + } // stage +stage('Check by Trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + environment { + TRIVY_LOG = "trivy-high-junit.xml" + } + steps { + script { + try { + // 🔹 Fetch XtraBackup Version + echo "🔄 Fetching XtraBackup version..." + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + cat XB_VERSION + ''' + + // ✅ Read the version file and parse it manually (CPS-compatible way) + def versionEnv = [:] + def versionContent = readFile('XB_VERSION') + + // 🔹 Manually split lines and process them + versionContent.split("\n").each { line -> + def parts = line.split('=') + if (parts.length == 2) { + versionEnv[parts[0].trim()] = parts[1].trim() + } + } + + // 🔹 Extract version components + def XB_VERSION_MAJOR = versionEnv['XB_VERSION_MAJOR'] + def XB_VERSION_MINOR = versionEnv['XB_VERSION_MINOR'] + def XB_VERSION_PATCH = versionEnv['XB_VERSION_PATCH'] + def XB_VERSION_EXTRA = versionEnv['XB_VERSION_EXTRA'] + + // 🔹 Install Trivy if not already installed + sh ''' + if ! command -v trivy &> /dev/null; then + echo "🔄 Installing Trivy..." + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + else + echo "✅ Trivy is already installed." + fi + ''' + + // 🔹 Define the image tags + def imageList = [ + "perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64", + "perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64" + ] + + // 🔹 Scan images and store logs + imageList.each { image -> + echo "🔍 Scanning ${image}..." + def result = sh(script: """#!/bin/bash + LANG=C.UTF-8 sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 1 \ + --severity HIGH,CRITICAL ${image} | tee -a ${TRIVY_LOG} + """, returnStatus: true) + + if (result != 0) { + error "❌ Trivy detected vulnerabilities in ${image}. See ${TRIVY_LOG} for details." + } else { + echo "✅ No critical vulnerabilities found in ${image}." + } + } + } catch (Exception e) { + error "❌ Trivy scan failed: ${e.message}" + } + } + } + post { + always { + // 🔹 Archive the report + archiveArtifacts artifacts: "${TRIVY_LOG}", allowEmptyArchive: true + } + } +} +/* + stage('Check by trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + catchError { + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + sudo trivy -q image --format table \ + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 | tee -a trivy-hight-junit.xml + sudo trivy -q image --format table \ + --timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 | tee -a trivy-hight-junit.xml + ''' + } + } + } +*/ + } + post { + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${BRANCH}" + } + deleteDir() + } + } +} diff --git a/pxb/jenkins/pxb-80-docker.yml b/pxb/jenkins/pxb-80-docker.yml new file mode 100644 index 0000000000..5021470d0b --- /dev/null +++ b/pxb/jenkins/pxb-80-docker.yml @@ -0,0 +1,15 @@ +- job: + name: hetzner-pxb8.0-docker-build + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pxb/jenkins/pxb-80-docker.groovy + diff --git a/pxb/jenkins/pxb-80.groovy b/pxb/jenkins/pxb-80.groovy index afa44d1b7e..25d488b4cf 100644 --- a/pxb/jenkins/pxb-80.groovy +++ b/pxb/jenkins/pxb-80.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -6,18 +6,40 @@ library changelog: false, identifier: 'lib@master', retriever: modernSCM([ import groovy.transform.Field void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/storage/innobase/xtrabackup/utils/percona-xtrabackup-8.0_builder.sh -O percona-xtrabackup-8.0_builder.sh - pwd -P - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " - set -o xtrace - cd \${build_dir} - bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" - """ + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + mkdir test + if [ \${FIPSMODE} = "YES" ]; then + MYSQL_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${BRANCH}/MYSQL_VERSION && grep MYSQL_VERSION_MINOR MYSQL_VERSION | awk -F= '{print \$2}') + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${MYSQL_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o percona-xtrabackup-8.0_builder.sh \ + "https://api.github.com/repos/percona/percona-xtrabackup-private-build/contents/percona-xtrabackup-8.0_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${BRANCH}/storage/innobase/xtrabackup/utils/percona-xtrabackup-8.0_builder.sh -O percona-xtrabackup-8.0_builder.sh + fi + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-xtrabackup-private-build.git percona-xtrabackup-private-build + mv -f \${build_dir}/percona-xtrabackup-private-build \${build_dir}/test/. + ls \${build_dir}/test + fi + bash -x ./percona-xtrabackup-8.0_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} ${STAGE_PARAM}" + """ + } } void cleanUpWS() { @@ -141,9 +163,13 @@ def git_repo = "https://github.com/Percona-QA/package-testing.git" pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtrabackup.git', description: 'URL for PXB git repository', @@ -165,7 +191,7 @@ pipeline { description: 'Enable fipsmode', name: 'FIPSMODE') choice( - choices: 'laboratory\ntesting\nexperimental', + choices: 'testing\nlaboratory\nexperimental', description: 'Repo component to push packages to', name: 'COMPONENT') } @@ -179,7 +205,13 @@ pipeline { steps { // slackNotify("", "#00FF00", "[${JOB_NAME}]: starting build for ${BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("ubuntu:focal", "--get_sources=1") + script { + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:focal", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:focal", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "UPLOAD" test/percona-xtrabackup-8.0.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -196,8 +228,8 @@ pipeline { XB_VERSION_EXTRA = sh(returnStdout: true, script: "grep 'XB_VERSION_EXTRA' ./test/percona-xtrabackup-8.0.properties | cut -d = -f 2 | sed 's/-//g'").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } @@ -205,11 +237,11 @@ pipeline { parallel { stage('Build PXB generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") @@ -218,17 +250,17 @@ pipeline { } } - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXB generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:focal", "--build_source_deb=1 --enable_fipsmode=1") @@ -237,8 +269,8 @@ pipeline { } } - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -248,7 +280,7 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -256,18 +288,18 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -275,75 +307,132 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") } else { buildStage("oraclelinux:9", "--build_rpm=1") } - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } -/* stage('Amazon Linux 2023') { + stage('Oracle Linux 10') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { - buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") } else { - buildStage("amazonlinux:2023", "--build_rpm=1") + buildStage("oraclelinux:10", "--build_rpm=1") + } + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:10", "--build_rpm=1") + } + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + stage('Amazon Linux 2023') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } + } + } + } + stage('Amazon Linux 2023 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'NO') { + echo "The step is skipped" + } else { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } } } - }*/ + } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -351,18 +440,18 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -370,94 +459,94 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -465,18 +554,18 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { @@ -484,54 +573,54 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("debian:bookworm", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") } else { buildStage("debian:bookworm", "--build_deb=1") } - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Oracle Linux 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -539,37 +628,37 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Oracle Linux 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("oraclelinux:9", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Ubuntu Focal(20.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -577,56 +666,56 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("ubuntu:jammy", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Ubuntu Noble(24.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("ubuntu:noble", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Debian Bullseye(11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { @@ -634,31 +723,31 @@ pipeline { echo "The step is skipped" } else { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } } stage('Debian Bookworm(12) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_tarball=1 --enable_fipsmode=1") } else { buildStage("debian:bookworm", "--build_tarball=1") } - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } @@ -671,7 +760,23 @@ pipeline { signDEB() } } - + stage('Push Tarballs to TESTING download area') { + steps { + script { + try { + if (env.FIPSMODE == 'YES') { + uploadTarballToDownloadsTesting(params.CLOUD, "pxb-gated", "${BRANCH}") + } else { + uploadTarballToDownloadsTesting(params.CLOUD, "pxb", "${BRANCH}") + } + } + catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } + } + } stage('Push to public repository') { steps { script { @@ -680,28 +785,114 @@ pipeline { // sync packages if ("${MYSQL_VERSION_MINOR}" == "0") { if (env.FIPSMODE == 'YES') { - sync2PrivateProdAutoBuild("pxb-80-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxb-80-pro", COMPONENT) } else { - sync2ProdAutoBuild("pxb-80", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxb-80", COMPONENT) } } else { if (env.FIPSMODE == 'YES') { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2PrivateProdAutoBuild("pxb-84-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxb-84-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("pxb-8x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxb-8x-innovation-pro", COMPONENT) } } else { if ("${MYSQL_VERSION_MINOR}" == "4") { - sync2ProdAutoBuild("pxb-84-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxb-84-lts", COMPONENT) } else { - sync2ProdAutoBuild("pxb-8x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxb-8x-innovation", COMPONENT) } } } } } } + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + sleep 1200 + ''' + unstash 'uploadPath' + sh ''' + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + curl -O https://raw.githubusercontent.com/percona/percona-server/refs/heads/${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}/MYSQL_VERSION + . ./MYSQL_VERSION + rm -rf percona-docker + git clone https://github.com/percona/percona-docker + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + cd percona-docker/percona-xtrabackup-8.0 + else + cd percona-docker/percona-xtrabackup-8.x + fi + sed -i "s/ENV XTRABACKUP_VERSION.*/ENV XTRABACKUP_VERSION ${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.1/g" Dockerfile + if [ \${MYSQL_VERSION_MINOR} = "0" ]; then + sed -i "s/pxb-80 testing/pxb-80 ${COMPONENT}/g" Dockerfile + else + sed -i "s/pxb-84-lts testing/pxb-84-lts ${COMPONENT}/g" Dockerfile + fi + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --platform "linux/amd64" . + sudo docker build --no-cache -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" . + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtrabackup/${BRANCH}/XB_VERSION + . ./XB_VERSION + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + + sudo docker manifest create perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + + sudo docker manifest push perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR} perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtrabackup:latest perconalab/percona-xtrabackup:${XB_VERSION_MAJOR}.${XB_VERSION_MINOR}.${XB_VERSION_PATCH}${XB_VERSION_EXTRA}.${RPM_RELEASE} + ''' + } + } // if + } // scripts + } + } } post { success { diff --git a/pxb/jenkins/pxb-80.yml b/pxb/jenkins/pxb-80.yml index 22c13fa84c..f640673947 100644 --- a/pxb/jenkins/pxb-80.yml +++ b/pxb/jenkins/pxb-80.yml @@ -1,5 +1,5 @@ - job: - name: percona-xtrabackup-8.0-RELEASE + name: hetzner-percona-xtrabackup-8.0-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxb/jenkins/pxb-80.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-5.7.groovy b/pxc/jenkins/percona-xtradb-cluster-5.7.groovy index 88c5e2e950..e9cb1a8a8d 100644 --- a/pxc/jenkins/percona-xtradb-cluster-5.7.groovy +++ b/pxc/jenkins/percona-xtradb-cluster-5.7.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -49,9 +49,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtradb-cluster-private.git', description: 'URL for percona-xtradb-cluster repository', @@ -115,40 +119,40 @@ pipeline { } stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PXC generic source packages') { parallel { stage('Build PXC generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_src_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXC generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:xenial", "--build_source_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -157,284 +161,284 @@ pipeline { parallel { stage('Centos 7') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:7", "--build_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Centos 8') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("centos:8", "--build_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Bionic(18.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Buster(10)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:buster", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_deb=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 7 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } /* stage('Centos 7 debug tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:7", "--build_tarball=1 --debug=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("debug/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "debug/", AWS_STASH_PATH) } } */ stage('Centos 8 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("centos:8", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Centos 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Ubuntu Bionic (18.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:bionic", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Ubuntu Focal (20.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Debian Buster (10) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:buster", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Debian Bullseye (11) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } stage('Debian Bookworm (12) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-57.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("debian:bookworm", "--build_tarball=1") stash includes: 'test/pxc-57.properties', name: 'pxc-57.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) } } } @@ -448,9 +452,9 @@ pipeline { installCli("deb") unstash 'pxc-57.properties' - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Sign packages') { @@ -463,7 +467,7 @@ pipeline { steps { // sync packages // sync2ProdAutoBuild(PXC_REPO, COMPONENT) - sync2PrivateProdAutoBuild("pxc-57-eol", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-57-eol", COMPONENT) } } stage('Build docker container') { @@ -533,7 +537,7 @@ pipeline { steps { script { try { - uploadTarballToDownloadsTesting("pxc-gated", "${GIT_BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "pxc-gated", "${GIT_BRANCH}") } catch (err) { echo "Caught: ${err}" diff --git a/pxc/jenkins/percona-xtradb-cluster-5.7.yml b/pxc/jenkins/percona-xtradb-cluster-5.7.yml index 0ef9231615..e2a8c387c0 100644 --- a/pxc/jenkins/percona-xtradb-cluster-5.7.yml +++ b/pxc/jenkins/percona-xtradb-cluster-5.7.yml @@ -1,5 +1,5 @@ - job: - name: pxc57-autobuild-RELEASE + name: hetzner-pxc57-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxc/jenkins/percona-xtradb-cluster-5.7.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy new file mode 100644 index 0000000000..d3e1946c50 --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy @@ -0,0 +1,304 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + choice( + choices: 'perconalab\npercona', + description: 'Organization on hub.docker.com', + name: 'ORGANIZATION') + string( + defaultValue: 'https://github.com/percona/percona-xtradb-cluster.git', + description: 'URL for percona-xtradb-cluster repository', + name: 'GIT_REPO') + string( + defaultValue: '8.0', + description: 'Tag/Branch for percona-xtradb-cluster repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + choice( + choices: '#releases-ci\n#releases', + description: 'Channel for notifications', + name: 'SLACKNOTIFY') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Build docker containers') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + script { + echo "====> Build docker containers" + cleanUpWS() + sh ''' + PXC_RELEASE=$(echo ${GIT_BRANCH} | sed 's/release-//g') + PXC_MAJOR_RELEASE=$(echo ${GIT_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version + sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + git clone https://github.com/percona/percona-docker + cd percona-docker/percona-xtradb-cluster-${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} + sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXC_REPO .*/ENV PXC_REPO testing/g" Dockerfile + if [ ${PXC_MAJOR_RELEASE} != "80" ]; then + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile + fi + fi + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker build --no-cache --platform "linux/amd64" -t ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 . + sudo docker build --no-cache --platform "linux/amd64" --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 . + else + sudo docker pull perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker tag perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 percona/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker pull perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 + sudo docker tag perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 percona/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 + fi + sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile.aarch64 + sed -i "s/ENV PXC_REPO .*/ENV PXC_REPO testing/g" Dockerfile.aarch64 + if [ ${PXC_MAJOR_RELEASE} != "80" ]; then + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile.aarch64 + fi + fi + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker build --no-cache -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build --no-cache --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + else + sudo docker pull perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker tag perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 percona/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker pull perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 + sudo docker tag perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 percona/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 + fi + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + cd ../percona-xtradb-cluster-8.0-backup + else + cd ../percona-xtradb-cluster-8.4-backup + fi + sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION=${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXC_REPO.*/ENV PXC_REPO=testing/g" Dockerfile + sed -i "s:yum/release:yum/testing:g" Dockerfile + if [ ${PXC_MAJOR_RELEASE} != "80" ]; then + #sed -i "s/ENV PXB_VERSION.*/ENV PXB_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + sed -i "s/ENV PXB_VERSION.*/ENV PXB_VERSION 8.4.0-4.1/g" Dockerfile + sed -i "s/ENV PS_VERSION.*/ENV PS_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile + if [ ${PXC_MAJOR_RELEASE} != "84" ]; then + sed -i "s/tools/pxb-8x-innovation/g" Dockerfile + sed -i "s/ps-80/ps-8x-innovation/g" Dockerfile + sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile + else + sed -i "s/tools/pxb-84-lts/g" Dockerfile + sed -i "s/ps-80/ps-84-lts/g" Dockerfile + sed -i "s/pxc-80/pxc-84-lts/g" Dockerfile + fi + sed -i "s/percona-xtrabackup-80/percona-xtrabackup-${PXC_MAJOR_RELEASE}/g" Dockerfile + fi + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster-operator:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}-pxc8.${MYSQL_VERSION_MINOR}-backup . + fi + + sudo docker images + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo docker push ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 + sudo docker push ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker push ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 + if [ ${ORGANIZATION} != "percona" ]; then + sudo docker push perconalab/percona-xtradb-cluster-operator:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}-pxc8.${MYSQL_VERSION_MINOR}-backup + fi + ''' + } + sh ''' + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo docker manifest create ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} \ + ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + ''' + withCredentials([ + usernamePassword(credentialsId: 'hub.docker.com', + passwordVariable: 'PASS', + usernameVariable: 'USER' + )]) { + sh ''' + echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION + . ./MYSQL_VERSION + sudo docker manifest push ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH} ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t ${ORGANIZATION}/percona-xtradb-cluster:latest ${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + ''' + } + } + } + } +stage('Check by trivy') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + environment { + TRIVY_LOG = "trivy-high-junit.xml" + } + steps { + script { + try { + echo "🔄 Fetching MySQL version..." + + // 🔹 Capture the file content directly from curl + def mysqlVersion = sh( + script: "curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION", + returnStdout: true + ).trim() + + echo "🔎 Raw MYSQL_VERSION: '${mysqlVersion}'" + + if (!mysqlVersion) { + error "❌ MYSQL_VERSION file is empty or not found!" + } + + def versionMap = [:] + mysqlVersion.split('\n').each { line -> + def (key, value) = line.tokenize('=') + versionMap[key.trim()] = value.trim().replaceAll('"', '') + } + + def MYSQL_VERSION_MAJOR = versionMap['MYSQL_VERSION_MAJOR'] + def MYSQL_VERSION_MINOR = versionMap['MYSQL_VERSION_MINOR'] + def MYSQL_VERSION_PATCH = versionMap['MYSQL_VERSION_PATCH'] + def MYSQL_VERSION_EXTRA = versionMap['MYSQL_VERSION_EXTRA'] + def fullVersion = "${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}" + + echo "✅ Parsed MySQL version: ${fullVersion}" + + // 🔹 Install Trivy if not already installed + sh ''' + if ! command -v trivy &> /dev/null; then + echo "🔄 Installing Trivy..." + sudo apt-get update + sudo apt-get -y install wget apt-transport-https gnupg lsb-release + wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - + echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list + sudo apt-get update + sudo apt-get -y install trivy + else + echo "✅ Trivy is already installed." + fi + ''' + + // 🔹 Define the image tags + def imageList = [ + "${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64", + "${ORGANIZATION}/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64" + ] + + // 🔹 Scan images and store logs + imageList.each { image -> + echo "🔍 Scanning ${image}..." + def result = sh(script: """#!/bin/bash + sudo trivy image --quiet \ + --format table \ + --timeout 10m0s \ + --ignore-unfixed \ + --exit-code 1 \ + --severity HIGH,CRITICAL ${image} | tee -a ${TRIVY_LOG} + """, returnStatus: true) + + if (result != 0) { + error "❌ Trivy detected vulnerabilities in ${image}. See ${TRIVY_LOG} for details." + } else { + echo "✅ No critical vulnerabilities found in ${image}." + } + } + } catch (Exception e) { + error "❌ Trivy scan failed: ${e.message}" + } // try + } // script + } // steps + } // stage + } + post { + success { + script { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} pushed to ${ORGANIZATION}") + } + deleteDir() + } + failure { + script { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH}]") + } + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + currentBuild.description = "Built on ${GIT_BRANCH} pushed to ${ORGANIZATION}" + } + deleteDir() + } + } +} diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0-docker.yml b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.yml new file mode 100644 index 0000000000..a4db063a58 --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-8.0-docker.yml @@ -0,0 +1,14 @@ +- job: + name: hetzner-pxc8.0-docker-build + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/adivinho/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pxc/jenkins/percona-xtradb-cluster-8.0-docker.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0.groovy b/pxc/jenkins/percona-xtradb-cluster-8.0.groovy index ddeec39b36..37c57c58a3 100644 --- a/pxc/jenkins/percona-xtradb-cluster-8.0.groovy +++ b/pxc/jenkins/percona-xtradb-cluster-8.0.groovy @@ -1,22 +1,46 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ void buildStage(String DOCKER_OS, String STAGE_PARAM) { - sh """ - set -o xtrace - mkdir -p test - wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/build-ps/pxc_builder.sh -O pxc_builder.sh - pwd -P - ls -laR - export build_dir=\$(pwd -P) - docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " - set -o xtrace - cd \${build_dir} - bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --install_deps=1 - bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --bin_release=${BIN_RELEASE} ${STAGE_PARAM}" - """ + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + withCredentials([usernamePassword(credentialsId: 'PS_PRIVATE_REPO_ACCESS', passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) { + sh """ + set -o xtrace + mkdir -p test + if [ \${FIPSMODE} = "YES" ]; then + PXC_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${GIT_BRANCH}/MYSQL_VERSION && cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') + if [ \${PXC_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${PXC_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o pxc_builder.sh \ + "https://api.github.com/repos/percona/percona-xtradb-cluster-private-build/contents/build-ps/pxc_builder.sh?ref=\${PRO_BRANCH}" + sed -i "s/PRIVATE_USERNAME/\${USERNAME}/g" pxc_builder.sh + sed -i "s/PRIVATE_TOKEN/\${PASSWORD}/g" pxc_builder.sh + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/build-ps/pxc_builder.sh -O pxc_builder.sh + fi + pwd -P + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-xtradb-cluster-private-build.git percona-xtradb-cluster-private-build + mv -f \${build_dir}/percona-xtradb-cluster-private-build/build-ps \${build_dir}/test/. + fi + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --bin_release=${BIN_RELEASE} ${STAGE_PARAM}" + """ + } + } } void cleanUpWS() { @@ -29,9 +53,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/percona/percona-xtradb-cluster.git', description: 'URL for percona-xtradb-cluster repository', @@ -52,20 +80,16 @@ pipeline { defaultValue: '1', description: 'BIN release value', name: 'BIN_RELEASE') - choice( - choices: 'pxc-80\npxc-8x-innovation\npxc-84-lts', - description: 'PXC repo name', - name: 'PXC_REPO') choice( choices: 'NO\nYES', description: 'Enable fipsmode', name: 'FIPSMODE') choice( - choices: 'laboratory\ntesting\nexperimental', + choices: 'testing\nexperimental\nlaboratory', description: 'Repo component to push packages to', name: 'COMPONENT') choice( - choices: '#releases\n#releases-ci', + choices: '#releases-ci\n#releases', description: 'Channel for notifications', name: 'SLACKNOTIFY') } @@ -77,10 +101,19 @@ pipeline { } stages { stage('Create PXC source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } steps { slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") cleanUpWS() - buildStage("centos:7", "--get_sources=1") + script { + if (env.FIPSMODE == 'YES') { + buildStage("centos:7", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("centos:7", "--get_sources=1") + } + } sh ''' REPO_UPLOAD_PATH=$(grep "DEST=UPLOAD" test/pxc-80.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") @@ -95,20 +128,20 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build PXC generic source packages') { parallel { stage('Build PXC generic source rpm') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") @@ -117,18 +150,18 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build PXC generic source deb') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:xenial", "--build_source_deb=1 --enable_fipsmode=1") @@ -137,82 +170,82 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel } // stage stage('Build PXC RPMs/DEBs/Binary tarballs') { parallel { - stage('Centos 7') { + stage('Centos 8') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { - echo "The step is skipped" -/* - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("centos:7", "--build_rpm=1") + script { + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) -*/ + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } } } - stage('Centos 8') { + stage('Centos 8 ARM') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("centos:8", "--build_rpm=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } } - stage('Centos 8 ARM') { + stage('Oracle Linux 9') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - echo "The step is skipped" + buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("centos:8", "--build_rpm=1") - - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + buildStage("oraclelinux:9", "--build_rpm=1") } } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9') { + stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") @@ -221,95 +254,138 @@ pipeline { } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } - stage('Oracle Linux 9 ARM') { + stage('Oracle Linux 10') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { - buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") } else { - buildStage("oraclelinux:9", "--build_rpm=1") + buildStage("oraclelinux:10", "--build_rpm=1") } } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } -/* stage('Amazon Linux 2023') { + stage('Oracle Linux 10 ARM') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("srpm/", AWS_STASH_PATH) - buildStage("amazonlinux:2023", "--build_rpm=1") - + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:10", "--build_rpm=1") + } + } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } - }*/ + } + stage('Amazon Linux 2023') { + when { + expression { env.FIPSMODE == 'YES' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + stage('Amazon Linux 2023 ARM') { + when { + expression { env.FIPSMODE == 'YES' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } stage('Ubuntu Focal(20.04)') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Focal(20.04) ARM') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("ubuntu:focal", "--build_deb=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("ubuntu:focal", "--build_deb=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") @@ -319,18 +395,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") @@ -340,18 +416,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") @@ -361,18 +437,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") @@ -382,60 +458,58 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("debian:bullseye", "--build_deb=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bullseye(11) ARM') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("debian:bullseye", "--build_deb=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_deb=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } } stage('Debian Bookworm(12)') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") @@ -445,18 +519,18 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") @@ -466,39 +540,38 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Centos 8 tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("centos:8", "--build_tarball=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("centos:8", "--build_tarball=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Oracle Linux 9 tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") @@ -508,39 +581,38 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } stage('Debian Bullseye(11) tarball') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { - cleanUpWS() - unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("debian:bullseye", "--build_tarball=1") + cleanUpWS() + unstash 'pxc-80.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("debian:bullseye", "--build_tarball=1") - stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') - } + stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } stage('Ubuntu Jammy(22.04) tarball') { agent { - label 'docker-32gb' + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' } steps { cleanUpWS() unstash 'pxc-80.properties' - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) script { if (env.FIPSMODE == 'YES') { buildStage("ubuntu:jammy", "--build_tarball=1 --enable_fipsmode=1") @@ -550,8 +622,8 @@ pipeline { } stash includes: 'test/pxc-80.properties', name: 'pxc-80.properties' - pushArtifactFolder("test/tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("test/tarball/", AWS_STASH_PATH, 'binary') + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') } } } @@ -570,22 +642,22 @@ pipeline { if ("${PXC_VERSION_MINOR}" == "0") { // sync packages if (env.FIPSMODE == 'YES') { - sync2PrivateProdAutoBuild("pxc-80-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-80-pro", COMPONENT) } else { - sync2ProdAutoBuild("pxc-80", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxc-80", COMPONENT) } } else { if (env.FIPSMODE == 'YES') { if ("${PXC_VERSION_MINOR}" == "4") { - sync2PrivateProdAutoBuild("pxc-84-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-84-pro", COMPONENT) } else { - sync2PrivateProdAutoBuild("pxc-8x-innovation-pro", COMPONENT) + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-8x-innovation-pro", COMPONENT) } } else { if ("${PXC_VERSION_MINOR}" == "4") { - sync2ProdAutoBuild("pxc-84-lts", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxc-84-lts", COMPONENT) } else { - sync2ProdAutoBuild("pxc-8x-innovation", COMPONENT) + sync2ProdAutoBuild(params.CLOUD, "pxc-8x-innovation", COMPONENT) } } } @@ -597,9 +669,9 @@ pipeline { script { try { if (env.FIPSMODE == 'YES') { - uploadTarballToDownloadsTesting("pxc-gated", "${GIT_BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "pxc-gated", "${GIT_BRANCH}") } else { - uploadTarballToDownloadsTesting("pxc", "${GIT_BRANCH}") + uploadTarballToDownloadsTesting(params.CLOUD, "pxc", "${GIT_BRANCH}") } } catch (err) { @@ -610,14 +682,14 @@ pipeline { } } stage('Build docker containers') { + when { + expression { env.FIPSMODE == 'NO' } + } agent { - label 'min-focal-x64' + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' } steps { script { - if (env.FIPSMODE == 'YES') { - echo "The step is skipped" - } else { echo "====> Build docker containers" cleanUpWS() sh ''' @@ -627,31 +699,38 @@ pipeline { sh ''' PXC_RELEASE=$(echo ${GIT_BRANCH} | sed 's/release-//g') PXC_MAJOR_RELEASE=$(echo ${GIT_BRANCH} | sed "s/release-//g" | sed "s/\\.//g" | awk '{print substr($0, 0, 2)}') + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common - sudo apt-get install -y docker.io - sudo systemctl status docker - sudo apt-get install -y qemu binfmt-support qemu-user-static + sudo apt-get -y install apparmor + sudo aa-status + sudo systemctl stop apparmor + sudo systemctl disable apparmor + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + export DOCKER_CLI_EXPERIMENTAL=enabled + sudo mkdir -p /usr/libexec/docker/cli-plugins/ + sudo curl -L https://github.com/docker/buildx/releases/download/v0.21.2/buildx-v0.21.2.linux-amd64 -o /usr/libexec/docker/cli-plugins/docker-buildx + sudo chmod +x /usr/libexec/docker/cli-plugins/docker-buildx + sudo systemctl restart docker + sudo apt-get install -y qemu-system binfmt-support qemu-user-static + sudo qemu-system-x86_64 --version sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION git clone https://github.com/percona/percona-docker - cd percona-docker/percona-xtradb-cluster-8.0 + cd percona-docker/percona-xtradb-cluster-${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile sed -i "s/ENV PXC_REPO .*/ENV PXC_REPO testing/g" Dockerfile if [ ${PXC_MAJOR_RELEASE} != "80" ]; then if [ ${PXC_MAJOR_RELEASE} != "84" ]; then sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile - else - sed -i "s/pdpxc-8.0/pdpxc-84-lts/g" Dockerfile - sed -i "s/pxc-80/pxc-84-lts/g" Dockerfile - sed -i "s/default_authentication_plugin=mysql_native_password/mysql-native-password=ON\\nrequire_secure_transport=OFF/g" dockerdir/etc/mysql/node.cnf - sed -i "s/skip-host-cache/host_cache_size = 0/g" dockerdir/etc/mysql/node.cnf - sed -i "s/--skip-ssl//g" dockerdir/entrypoint.sh fi fi - sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} . - sudo docker build --no-cache --platform "linux/amd64" --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug . + sudo docker build --no-cache --platform "linux/amd64" -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 . + sudo docker build --no-cache --platform "linux/amd64" --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 . sed -i "s/ENV PXC_VERSION.*/ENV PXC_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}/g" Dockerfile.aarch64 sed -i "s/ENV PXC_TELEMETRY_VERSION.*/ENV PXC_TELEMETRY_VERSION ${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}-${RPM_RELEASE}/g" Dockerfile.aarch64 @@ -659,16 +738,10 @@ pipeline { if [ ${PXC_MAJOR_RELEASE} != "80" ]; then if [ ${PXC_MAJOR_RELEASE} != "84" ]; then sed -i "s/pxc-80/pxc-8x-innovation/g" Dockerfile.aarch64 - else - sed -i "s/pdpxc-8.0/pdpxc-84-lts/g" Dockerfile.aarch64 - sed -i "s/pxc-80/pxc-84-lts/g" Dockerfile.aarch64 - sed -i "s/default_authentication_plugin=mysql_native_password/mysql-native-password=ON\\nrequire_secure_transport=OFF/g" dockerdir/etc/mysql/node.cnf - sed -i "s/skip-host-cache/host_cache_size = 0/g" dockerdir/etc/mysql/node.cnf - sed -i "s/--skip-ssl//g" dockerdir/entrypoint.sh fi fi - sudo docker build --no-cache -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 --platform="linux/arm64" -f Dockerfile.aarch64 . - sudo docker build --no-cache --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-aarch64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build --no-cache -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . + sudo docker build --no-cache --build-arg DEBUG=1 -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 --platform="linux/arm64" -f Dockerfile.aarch64 . if [ ${PXC_MAJOR_RELEASE} != "84" ]; then cd ../percona-xtradb-cluster-8.0-backup @@ -705,22 +778,22 @@ pipeline { echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 - sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-aarch64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-amd64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-debug-arm64 sudo docker push perconalab/percona-xtradb-cluster-operator:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}-pxc8.${MYSQL_VERSION_MINOR}-backup ''' } sh ''' curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION - sudo docker manifest create perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi \ - perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} \ - perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 - sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-aarch64 --os linux --arch arm64 --variant v8 - sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} --os linux --arch amd64 - sudo docker manifest inspect perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi + sudo docker manifest create perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} \ + perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 \ + perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 + sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-arm64 --os linux --arch arm64 --variant v8 + sudo docker manifest annotate perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-amd64 --os linux --arch amd64 + sudo docker manifest inspect perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} ''' withCredentials([ usernamePassword(credentialsId: 'hub.docker.com', @@ -731,10 +804,12 @@ pipeline { echo "${PASS}" | sudo docker login -u "${USER}" --password-stdin curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${GIT_BRANCH}/MYSQL_VERSION . ./MYSQL_VERSION - sudo docker manifest push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE}-multi + sudo docker manifest push perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR} perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} + sudo docker buildx imagetools create -t perconalab/percona-xtradb-cluster:latest perconalab/percona-xtradb-cluster:${MYSQL_VERSION_MAJOR}.${MYSQL_VERSION_MINOR}.${MYSQL_VERSION_PATCH}${MYSQL_VERSION_EXTRA}.${RPM_RELEASE} ''' } - } } } } diff --git a/pxc/jenkins/percona-xtradb-cluster-8.0.yml b/pxc/jenkins/percona-xtradb-cluster-8.0.yml index 87d235dcb2..cdf5b2bdfc 100644 --- a/pxc/jenkins/percona-xtradb-cluster-8.0.yml +++ b/pxc/jenkins/percona-xtradb-cluster-8.0.yml @@ -1,5 +1,5 @@ - job: - name: pxc80-autobuild-RELEASE + name: hetzner-pxc80-autobuild-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: pxc/jenkins/percona-xtradb-cluster-8.0.groovy diff --git a/pxc/jenkins/percona-xtradb-cluster-9.x.groovy b/pxc/jenkins/percona-xtradb-cluster-9.x.groovy new file mode 100644 index 0000000000..4ef472129f --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-9.x.groovy @@ -0,0 +1,656 @@ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' +]) _ + +void buildStage(String DOCKER_OS, String STAGE_PARAM) { + withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'TOKEN')]) { + sh """ + set -o xtrace + mkdir -p test + if [ \${FIPSMODE} = "YES" ]; then + PXC_VERSION_MINOR=\$(curl -s -O \$(echo \${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/\${GIT_BRANCH}/MYSQL_VERSION && cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') + if [ \${PXC_VERSION_MINOR} = "0" ]; then + PRO_BRANCH="8.0" + elif [ \${PXC_VERSION_MINOR} = "4" ]; then + PRO_BRANCH="8.4" + else + PRO_BRANCH="trunk" + fi + curl -L -H "Authorization: Bearer \${TOKEN}" \ + -H "Accept: application/vnd.github.v3.raw" \ + -o pxc_builder.sh \ + "https://api.github.com/repos/percona/percona-xtradb-cluster-private-build/contents/build-ps/pxc_builder.sh?ref=\${PRO_BRANCH}" + else + wget \$(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git\$||')/${GIT_BRANCH}/build-ps/pxc_builder.sh -O pxc_builder.sh + fi + pwd -P + ls -laR + export build_dir=\$(pwd -P) + docker run -u root -v \${build_dir}:\${build_dir} ${DOCKER_OS} sh -c " + set -o xtrace + cd \${build_dir} + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --install_deps=1 + if [ \${FIPSMODE} = "YES" ]; then + git clone --depth 1 --branch \${PRO_BRANCH} https://x-access-token:${TOKEN}@github.com/percona/percona-xtradb-cluster-private-build.git percona-xtradb-cluster-private-build + mv -f \${build_dir}/percona-xtradb-cluster-private-build/build-ps \${build_dir}/test/. + fi + bash -x ./pxc_builder.sh --builddir=\${build_dir}/test --repo=${GIT_REPO} --branch=${GIT_BRANCH} --rpm_release=${RPM_RELEASE} --deb_release=${DEB_RELEASE} --bin_release=${BIN_RELEASE} ${STAGE_PARAM}" + """ + } +} + +void cleanUpWS() { + sh """ + sudo rm -rf ./* + """ +} + +def AWS_STASH_PATH + +pipeline { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) + string( + defaultValue: 'https://github.com/percona/percona-xtradb-cluster.git', + description: 'URL for percona-xtradb-cluster repository', + name: 'GIT_REPO') + string( + defaultValue: '9.0', + description: 'Tag/Branch for percona-xtradb-cluster repository', + name: 'GIT_BRANCH') + string( + defaultValue: '1', + description: 'RPM release value', + name: 'RPM_RELEASE') + string( + defaultValue: '1', + description: 'DEB release value', + name: 'DEB_RELEASE') + string( + defaultValue: '1', + description: 'BIN release value', + name: 'BIN_RELEASE') + choice( + choices: 'pxc-9x-innovation\npxc-97-lts', + description: 'PXC repo name', + name: 'PXC_REPO') + choice( + choices: 'NO\nYES', + description: 'Enable fipsmode', + name: 'FIPSMODE') + choice( + choices: 'YES\nNO', + description: 'Experimental packages only', + name: 'EXPERIMENTALMODE') + choice( + choices: 'experimental\nlaboratory\ntesting', + description: 'Repo component to push packages to', + name: 'COMPONENT') + choice( + choices: '#releases-ci\n#releases', + description: 'Channel for notifications', + name: 'SLACKNOTIFY') + } + options { + skipDefaultCheckout() + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) + timestamps () + } + stages { + stage('Create PXC source tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64' + } + steps { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: starting build for ${GIT_BRANCH} - [${BUILD_URL}]") + cleanUpWS() + script { + if (env.FIPSMODE == 'YES') { + buildStage("centos:7", "--get_sources=1 --enable_fipsmode=1") + } else { + buildStage("centos:7", "--get_sources=1") + } + } + sh ''' + REPO_UPLOAD_PATH=$(grep "DEST=UPLOAD" test/pxc-9x.properties | cut -d = -f 2 | sed "s:$:${BUILD_NUMBER}:") + AWS_STASH_PATH=$(echo ${REPO_UPLOAD_PATH} | sed "s:UPLOAD/experimental/::") + echo ${REPO_UPLOAD_PATH} > uploadPath + echo ${AWS_STASH_PATH} > awsUploadPath + cat test/pxc-9x.properties + cat uploadPath + cat awsUploadPath + ''' + script { + AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + stash includes: 'uploadPath', name: 'uploadPath' + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') + } + } + stage('Build PXC generic source packages') { + parallel { + stage('Build PXC generic source rpm') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("centos:7", "--build_src_rpm=1 --enable_fipsmode=1") + } else { + buildStage("centos:7", "--build_src_rpm=1") + } + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) + } + } + stage('Build PXC generic source deb') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:xenial", "--build_source_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:xenial", "--build_source_deb=1") + } + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) + } + } + } //parallel + } // stage + stage('Build PXC RPMs/DEBs/Binary tarballs') { + parallel { + stage('Centos 8') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + } + stage('Centos 8 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("centos:8", "--build_rpm=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + } + stage('Oracle Linux 9') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Oracle Linux 9 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:9", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:9", "--build_rpm=1") + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:10", "--build_rpm=1") + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:10", "--build_rpm=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:10", "--build_rpm=1") + } + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Amazon Linux 2023') { + when { + expression { env.FIPSMODE == 'YES' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Amazon Linux 2023 ARM') { + when { + expression { env.FIPSMODE == 'YES' } + } + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("amazonlinux:2023", "--build_rpm=1 --enable_fipsmode=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + } + } + stage('Ubuntu Jammy(22.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + stage('Ubuntu Jammy(22.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:jammy", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:jammy", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + stage('Ubuntu Noble(24.04)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + stage('Ubuntu Noble(24.04) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:noble", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:noble", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + stage('Debian Bookworm(12)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("debian:bookworm", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Debian Bookworm(12) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("debian:bookworm", "--build_deb=1 --enable_fipsmode=1") + } else { + buildStage("debian:bookworm", "--build_deb=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + } + } + stage('Centos 8 tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + if (env.FIPSMODE == 'YES') { + echo "The step is skipped" + } else { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("centos:8", "--build_tarball=1") + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + } + stage('Oracle Linux 9 tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + script { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + if (env.FIPSMODE == 'YES') { + buildStage("oraclelinux:9", "--build_tarball=1 --enable_fipsmode=1") + } else { + buildStage("oraclelinux:9", "--build_tarball=1") + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + if (env.EXPERIMENTALMODE == 'NO') { + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + stage('Ubuntu Noble(24.04) tarball') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64' : 'docker-32gb' + } + steps { + cleanUpWS() + unstash 'pxc-9x.properties' + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + script { + if (env.FIPSMODE == 'YES') { + buildStage("ubuntu:noble", "--build_tarball=1 --enable_fipsmode=1") + } else { + buildStage("ubuntu:noble", "--build_tarball=1") + } + } + + stash includes: 'test/pxc-9x.properties', name: 'pxc-9x.properties' + pushArtifactFolder(params.CLOUD, "test/tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "test/tarball/", AWS_STASH_PATH, 'binary') + } + } + } + } + + stage('Sign packages') { + steps { + script { + if (env.EXPERIMENTALMODE == 'NO') { + signRPM() + } + } + signDEB() + } + } + stage('Push to public repository') { + steps { + script { + PXC_VERSION_MINOR = sh(returnStdout: true, script: ''' curl -s -O $(echo ${GIT_REPO} | sed -re 's|github.com|raw.githubusercontent.com|; s|\\.git$||')/${GIT_BRANCH}/MYSQL_VERSION; cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print $2}' ''').trim() + if ("${PXC_VERSION_MINOR}" == "0") { + // sync packages + if (env.FIPSMODE == 'YES') { + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-90-pro", COMPONENT) + } else { + sync2ProdAutoBuild(params.CLOUD, "pxc-90", COMPONENT) + } + } else { + if (env.FIPSMODE == 'YES') { + if ("${PXC_VERSION_MINOR}" == "7") { + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-97-pro", COMPONENT) + } else { + sync2PrivateProdAutoBuild(params.CLOUD, "pxc-9x-innovation-pro", COMPONENT) + } + } else { + if ("${PXC_VERSION_MINOR}" == "7") { + sync2ProdAutoBuild(params.CLOUD, "pxc-97-lts", COMPONENT) + } else { + sync2ProdAutoBuild(params.CLOUD, "pxc-9x-innovation", COMPONENT) + } + } + } + } + } + } + stage('Push Tarballs to TESTING download area') { + steps { + script { + try { + if (env.FIPSMODE == 'YES') { + uploadTarballToDownloadsTesting(params.CLOUD, "pxc-gated", "${GIT_BRANCH}") + } else { + uploadTarballToDownloadsTesting(params.CLOUD, "pxc", "${GIT_BRANCH}") + } + } + catch (err) { + echo "Caught: ${err}" + currentBuild.result = 'UNSTABLE' + } + } + } + } + } + post { + success { + script { + if (env.FIPSMODE == 'YES') { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + } else { + slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${GIT_BRANCH} - [${BUILD_URL}]") + } + } + deleteDir() + } + failure { + script { + if (env.FIPSMODE == 'YES') { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: PRO build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } else { + slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: build failed for ${GIT_BRANCH} - [${BUILD_URL}]") + } + } + deleteDir() + } + always { + sh ''' + sudo rm -rf ./* + ''' + script { + if (env.FIPSMODE == 'YES') { + currentBuild.description = "PRO -> Built on ${GIT_BRANCH} - packages [${COMPONENT}/${AWS_STASH_PATH}]" + } else { + currentBuild.description = "Built on ${GIT_BRANCH} - packages [${COMPONENT}/${AWS_STASH_PATH}]" + } + } + deleteDir() + } + } +} diff --git a/pxc/jenkins/percona-xtradb-cluster-9.x.yml b/pxc/jenkins/percona-xtradb-cluster-9.x.yml new file mode 100644 index 0000000000..dadc4f5e76 --- /dev/null +++ b/pxc/jenkins/percona-xtradb-cluster-9.x.yml @@ -0,0 +1,15 @@ +- job: + name: pxc9x-autobuild-RELEASE + project-type: pipeline + description: | + Do not edit this job through the web! + pipeline-scm: + scm: + - git: + url: https://github.com/Percona-Lab/jenkins-pipelines.git + branches: + - 'hetzner' + wipe-workspace: false + lightweight-checkout: true + script-path: pxc/jenkins/percona-xtradb-cluster-9.x.groovy + diff --git a/rel/jenkins/release-from-repository.groovy b/rel/jenkins/release-from-repository.groovy index b5635297f6..890daeaa59 100644 --- a/rel/jenkins/release-from-repository.groovy +++ b/rel/jenkins/release-from-repository.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -8,6 +8,10 @@ pipeline { label 'jenkins' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: '', description: 'PATH_TO_BUILD must be in form $DESTINATION/**release**/$revision', @@ -33,8 +37,8 @@ pipeline { name: 'PROBUILD') booleanParam(name: 'SKIP_RPM_PUSH', defaultValue: false, description: 'Skip push to RPM repository') booleanParam(name: 'SKIP_DEB_PUSH', defaultValue: false, description: 'Skip push to DEB repository') - booleanParam(name: 'SKIP_REPO_SYNC', defaultValue: false, description: 'Skip sync repos to production') booleanParam(name: 'SKIP_PACKAGES_SYNC', defaultValue: false, description: 'Skip sync packages to production download') + booleanParam(name: 'SKIP_REPO_SYNC', defaultValue: false, description: 'Skip sync repos to production') } options { skipDefaultCheckout() @@ -205,6 +209,9 @@ ENDSSH } } stage('Sync packages to production download') { + when { + expression { env.PATH_TO_BUILD?.trim() } + } steps { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { sh """ diff --git a/rel/jenkins/release-from-repository.yml b/rel/jenkins/release-from-repository.yml index 7b2b8c73f4..1041a8f0d8 100644 --- a/rel/jenkins/release-from-repository.yml +++ b/rel/jenkins/release-from-repository.yml @@ -1,5 +1,5 @@ - job: - name: RELEASE-from-repository + name: hetzner-RELEASE-from-repository project-type: pipeline description: | Do not edit this job through the web! @@ -9,7 +9,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: rel/jenkins/release-from-repository.groovy diff --git a/sysbench/sysbench.groovy b/sysbench/sysbench.groovy index 9a8114cc8a..4cfeafeed6 100644 --- a/sysbench/sysbench.groovy +++ b/sysbench/sysbench.groovy @@ -1,4 +1,4 @@ -library changelog: false, identifier: 'lib@master', retriever: modernSCM([ +library changelog: false, identifier: 'lib@hetzner', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/Percona-Lab/jenkins-pipelines.git' ]) _ @@ -28,9 +28,13 @@ def AWS_STASH_PATH pipeline { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } parameters { + choice( + choices: [ 'Hetzner','AWS' ], + description: 'Cloud infra for build', + name: 'CLOUD' ) string( defaultValue: 'https://github.com/akopytov/sysbench.git', description: 'URL for sysbench repository', @@ -99,36 +103,36 @@ pipeline { AWS_STASH_PATH = sh(returnStdout: true, script: "cat awsUploadPath").trim() } stash includes: 'uploadPath', name: 'uploadPath' - pushArtifactFolder("source_tarball/", AWS_STASH_PATH) - uploadTarballfromAWS("source_tarball/", AWS_STASH_PATH, 'source') + pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source') } } stage('Build Sysbench generic source packages') { parallel { stage('Build Sysbench generic source rpm') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_src_rpm=1") - pushArtifactFolder("srpm/", AWS_STASH_PATH) - uploadRPMfromAWS("srpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "srpm/", AWS_STASH_PATH) } } stage('Build Sysbench generic source deb') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_tarball/", AWS_STASH_PATH) - buildStage("ubuntu:bionic", "--build_source_deb=1") + popArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH) + buildStage("ubuntu:jammy", "--build_source_deb=1") - pushArtifactFolder("source_deb/", AWS_STASH_PATH) - uploadDEBfromAWS("source_deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "source_deb/", AWS_STASH_PATH) } } } //parallel @@ -137,184 +141,236 @@ pipeline { parallel { stage('Oracle Linux 8') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 8 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:8", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Oracle Linux 9 ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("srpm/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) buildStage("oraclelinux:9", "--build_rpm=1") - pushArtifactFolder("rpm/", AWS_STASH_PATH) - uploadRPMfromAWS("rpm/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) + } + } + stage('Oracle Linux 10 ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "srpm/", AWS_STASH_PATH) + buildStage("oraclelinux:10", "--build_rpm=1") + + pushArtifactFolder(params.CLOUD, "rpm/", AWS_STASH_PATH) + uploadRPMfromAWS(params.CLOUD, "rpm/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Focal(20.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:focal", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Jammy(22.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:jammy", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Ubuntu Noble(24.04) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("ubuntu:noble", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bullseye(11) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) buildStage("debian:bullseye", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12)') { agent { - label 'docker' + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("debian:bullseye", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } stage('Debian Bookworm(12) ARM') { agent { - label 'docker-32gb-aarch64' + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' } steps { cleanUpWS() - popArtifactFolder("source_deb/", AWS_STASH_PATH) - buildStage("debian:bullseye", "--build_deb=1") + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:bookworm", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13)') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") + + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) + } + } + stage('Debian Trixie(13) ARM') { + agent { + label params.CLOUD == 'Hetzner' ? 'docker-aarch64' : 'docker-32gb-aarch64' + } + steps { + cleanUpWS() + popArtifactFolder(params.CLOUD, "source_deb/", AWS_STASH_PATH) + buildStage("debian:trixie", "--build_deb=1") - pushArtifactFolder("deb/", AWS_STASH_PATH) - uploadDEBfromAWS("deb/", AWS_STASH_PATH) + pushArtifactFolder(params.CLOUD, "deb/", AWS_STASH_PATH) + uploadDEBfromAWS(params.CLOUD, "deb/", AWS_STASH_PATH) } } @@ -323,14 +379,14 @@ pipeline { stage('Sign packages') { steps { - signRPM() - signDEB() + signRPM(params.CLOUD) + signDEB(params.CLOUD) } } stage('Push to public repository') { steps { // sync packages - sync2ProdAutoBuild(SYSBENCH_REPO, COMPONENT) + sync2ProdAutoBuild(params.CLOUD, SYSBENCH_REPO, COMPONENT) } } diff --git a/sysbench/sysbench.yml b/sysbench/sysbench.yml index be75172fd6..1002ba764b 100644 --- a/sysbench/sysbench.yml +++ b/sysbench/sysbench.yml @@ -1,5 +1,5 @@ - job: - name: sysbench-RELEASE + name: hetzner-sysbench-RELEASE project-type: pipeline description: | Do not edit this job through the web! @@ -8,7 +8,7 @@ - git: url: https://github.com/Percona-Lab/jenkins-pipelines.git branches: - - 'master' + - 'hetzner' wipe-workspace: false lightweight-checkout: true script-path: sysbench/sysbench.groovy diff --git a/vars/popArtifactFolder.groovy b/vars/popArtifactFolder.groovy index 67e8f899f9..802192c497 100644 --- a/vars/popArtifactFolder.groovy +++ b/vars/popArtifactFolder.groovy @@ -1,10 +1,14 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sh """ - pwd - S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} - aws s3 cp --recursive \$S3_PATH/${FOLDER_NAME} ${FOLDER_NAME} || : - """ +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + String S3_STASH = (CLOUD_NAME == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (CLOUD_NAME == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + retry(15) { + sh """ + pwd + S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp --recursive \$S3_PATH/${FOLDER_NAME} ${FOLDER_NAME} ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || : + """ + } } } diff --git a/vars/pushArtifactFolder.groovy b/vars/pushArtifactFolder.groovy index b80c26f29c..f1e8d93307 100644 --- a/vars/pushArtifactFolder.groovy +++ b/vars/pushArtifactFolder.groovy @@ -1,11 +1,15 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AWS_STASH', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { - sh """ - pwd - S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} - aws s3 ls \$S3_PATH/${FOLDER_NAME} || : - aws s3 cp --quiet --recursive ${FOLDER_NAME} \$S3_PATH/${FOLDER_NAME} || : - """ +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + String S3_STASH = (CLOUD_NAME == 'Hetzner') ? 'HTZ_STASH' : 'AWS_STASH' + String S3_ENDPOINT = (CLOUD_NAME == 'Hetzner') ? '--endpoint-url https://fsn1.your-objectstorage.com' : '--endpoint-url https://s3.amazonaws.com' + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: S3_STASH, secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + retry(15) { + sh """ + pwd + S3_PATH=s3://percona-jenkins-artifactory/${AWS_STASH_PATH} + aws s3 ls \$S3_PATH/${FOLDER_NAME} ${S3_ENDPOINT} || : + AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws s3 cp --recursive ${FOLDER_NAME} \$S3_PATH/${FOLDER_NAME} ${S3_ENDPOINT} --cli-connect-timeout 60 --cli-read-timeout 120 || : + """ + } } } diff --git a/vars/signDEB.groovy b/vars/signDEB.groovy index 865cc734b2..0836f6f9ed 100644 --- a/vars/signDEB.groovy +++ b/vars/signDEB.groovy @@ -1,5 +1,6 @@ -def call() { - node('master') { +def call(String CLOUD_NAME = 'default') { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' withCredentials([string(credentialsId: 'SIGN_PASSWORD', variable: 'SIGN_PASSWORD')]) { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { diff --git a/vars/signRPM.groovy b/vars/signRPM.groovy index 042a9d6450..df9ce77952 100644 --- a/vars/signRPM.groovy +++ b/vars/signRPM.groovy @@ -1,5 +1,6 @@ -def call() { - node('master') { +def call(String CLOUD_NAME = 'default') { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' withCredentials([string(credentialsId: 'SIGN_PASSWORD', variable: 'SIGN_PASSWORD')]) { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { diff --git a/vars/sync2PrivateProdAutoBuild.groovy b/vars/sync2PrivateProdAutoBuild.groovy index 9c676d0a84..82aba5bae5 100644 --- a/vars/sync2PrivateProdAutoBuild.groovy +++ b/vars/sync2PrivateProdAutoBuild.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' def path_to_build = sh(returnStdout: true, script: "cat uploadPath").trim() diff --git a/vars/sync2ProdAutoBuild.groovy b/vars/sync2ProdAutoBuild.groovy index e726b0db3b..62f5537065 100644 --- a/vars/sync2ProdAutoBuild.groovy +++ b/vars/sync2ProdAutoBuild.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { unstash 'uploadPath' def path_to_build = sh(returnStdout: true, script: "cat uploadPath").trim() diff --git a/vars/sync2ProdPPG.groovy b/vars/sync2ProdPPG.groovy index 1d8f15b61a..680b7daca0 100644 --- a/vars/sync2ProdPPG.groovy +++ b/vars/sync2ProdPPG.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { def path_to_build = "/srv/UPLOAD/POSTGRESQL_SYNC/${repo_version}" withCredentials([string(credentialsId: 'SIGN_PASSWORD', variable: 'SIGN_PASSWORD')]) { withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', passphraseVariable: '', usernameVariable: 'USER')]) { diff --git a/vars/sync2web.groovy b/vars/sync2web.groovy index 101e701dd0..866e2a44b9 100644 --- a/vars/sync2web.groovy +++ b/vars/sync2web.groovy @@ -1,5 +1,6 @@ -def call(String REPO_NAME, String DESTINATION) { - node('master') { +def call(String CLOUD_NAME, String REPO_NAME, String DESTINATION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() unstash 'uploadPath' def path_to_build = "/srv/UPLOAD/POSTGRESQL_SYNC/${repo_version}" diff --git a/vars/uploadDEBfromAWS.groovy b/vars/uploadDEBfromAWS.groovy index d5917e793a..9493feb9ea 100644 --- a/vars/uploadDEBfromAWS.groovy +++ b/vars/uploadDEBfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) //unstash 'debs' unstash 'uploadPath' withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { @@ -25,7 +26,12 @@ def call(String FOLDER_NAME, String AWS_STASH_PATH) { dist=`echo \${deb} | sed -re 's/.*\\.([^.]+)_(amd64|arm64|all).deb/\\1/'` package=`echo \${deb} | sed -re 's/\\.\\/deb\\/(.*)_(.*)\\.([^.]+)_(amd64|arm64|all).deb/\\1/'` version=`echo \${deb} | sed -re 's/\\.\\/deb\\/(.*)_(.*)\\.([^.]+)_(amd64|arm64|all).deb/\\2/'` - path_to_dist=\${path_to_build}/binary/debian/\${dist}/x86_64 + arch=`echo \${deb} | sed -re 's/\\.\\/deb\\/(.*)_(.*)\\.([^.]+)_(amd64|arm64|all).deb/\\4/'` + if [ "\${arch}" = "arm64" ]; then + path_to_dist=\${path_to_build}/binary/debian/\${dist}/aarch64 + else + path_to_dist=\${path_to_build}/binary/debian/\${dist}/x86_64 + fi ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p \${path_to_dist} if [ "\${package}" = "percona-release" ]; then diff --git a/vars/uploadPGTarballToDownloadsTesting.groovy b/vars/uploadPGTarballToDownloadsTesting.groovy index 13d4dd15a0..153d817fb9 100644 --- a/vars/uploadPGTarballToDownloadsTesting.groovy +++ b/vars/uploadPGTarballToDownloadsTesting.groovy @@ -1,5 +1,6 @@ -def call(String PRODUCT_NAME, String PRODUCT_VERSION) { - node('master') { +def call(String CLOUD_NAME, String PRODUCT_NAME, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'master' + node(nodeLabel) { deleteDir() unstash "uploadPath-${PRODUCT_VERSION}" def path_to_build = sh(returnStdout: true, script: "cat uploadPath-${PRODUCT_VERSION}").trim() diff --git a/vars/uploadPGTarballfromAWS.groovy b/vars/uploadPGTarballfromAWS.groovy index 2aeafbccf4..0404c4d781 100644 --- a/vars/uploadPGTarballfromAWS.groovy +++ b/vars/uploadPGTarballfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH, String TarballType, String PRODUCT_VERSION) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH, String TarballType, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) //unstash "${TarballType}.tarball" unstash "uploadPath-${PRODUCT_VERSION}" withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { diff --git a/vars/uploadRPMfromAWS.groovy b/vars/uploadRPMfromAWS.groovy index f4ace39139..c2228c4bfb 100644 --- a/vars/uploadRPMfromAWS.groovy +++ b/vars/uploadRPMfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) unstash 'uploadPath' withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { sh """ @@ -19,7 +20,7 @@ def call(String FOLDER_NAME, String AWS_STASH_PATH) { ${USER}@repo.ci.percona.com:\${path_to_build}/source/redhat/ fi - export arch_list=\$( find . -name '*.el[6-9].*.rpm' -o -name '*.amzn2023.*.rpm' -o -name '*.noarch.rpm' | awk -F'[.]' '{print \$(NF -1)}' | sort -n | uniq ) + export arch_list=\$( find . -name '*.el[6-9].*.rpm' -o -name '*.el10.*.rpm' -o -name '*.amzn2023.*.rpm' -o -name '*.noarch.rpm' | awk -F'[.]' '{print \$(NF -1)}' | sort -n | uniq ) for arch in \${arch_list}; do if [ `find . -name "*.el6.\${arch}.rpm" | wc -l` -gt 0 ]; then @@ -54,6 +55,14 @@ def call(String FOLDER_NAME, String AWS_STASH_PATH) { ${USER}@repo.ci.percona.com:\${path_to_build}/binary/redhat/9/\${arch}/ fi + if [ `find . -name "*.el10.\${arch}.rpm" | wc -l` -gt 0 ]; then + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + mkdir -p \${path_to_build}/binary/redhat/10/\${arch} + scp -o StrictHostKeyChecking=no -i ${KEY_PATH} \ + `find . -name "*.el10.\${arch}.rpm"` \ + ${USER}@repo.ci.percona.com:\${path_to_build}/binary/redhat/10/\${arch}/ + fi + if [ `find . -name "*.amzn2023.\${arch}.rpm" | wc -l` -gt 0 ]; then ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p \${path_to_build}/binary/redhat/2023/\${arch} @@ -63,8 +72,7 @@ def call(String FOLDER_NAME, String AWS_STASH_PATH) { fi if [ `find . -name "*.noarch.rpm" | wc -l` -gt 0 ]; then - Vers=("6" "7" "8" "9" "2023") - for osVer in "\${Vers[@]}"; do + for osVer in 6 7 8 9 10 2023; do ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p \${path_to_build}/binary/redhat/\${osVer}/\${arch} scp -o StrictHostKeyChecking=no -i ${KEY_PATH} \ diff --git a/vars/uploadSBOMToDownloadsTesting.groovy b/vars/uploadSBOMToDownloadsTesting.groovy new file mode 100644 index 0000000000..f355455531 --- /dev/null +++ b/vars/uploadSBOMToDownloadsTesting.groovy @@ -0,0 +1,41 @@ +def call(String CLOUD_NAME, String PRODUCT_NAME, String PRODUCT_VERSION, String SBOMType) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'master' + node(nodeLabel) { + deleteDir() + unstash "uploadPath-${PRODUCT_VERSION}" + def path_to_build = sh(returnStdout: true, script: "cat uploadPath-${PRODUCT_VERSION}").trim() + withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { + sh """ + #!/bin/bash + set -o xtrace + + cat /etc/hosts > hosts + echo '10.30.6.9 repo.ci.percona.com' >> hosts + sudo cp ./hosts /etc || true + + # Cut prefix if it's provided + cutProductVersion=\$(echo ${PRODUCT_VERSION} | sed 's/release-//g'); + # Upload PSMDB SBOMS to tarball testing downloads + if [ \"x${PRODUCT_NAME}\" = \"xpsmdb_sbom\" ]; then + PSMDB_VERSION=\$(echo \${cutProductVersion} | cut -d'-' -f1) + echo \${cutProductVersion} + echo \${PSMDB_VERSION} + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/psmdb-\${PSMDB_VERSION}/${PRODUCT_NAME}-\${cutProductVersion} + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/${SBOMType}/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/psmdb-\${PSMDB_VERSION}/${PRODUCT_NAME}-\${cutProductVersion}/ + else + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion} + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/${SBOMType}/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion}/ + fi + + curl https://www.percona.com/admin/config/percona/percona_downloads/crawl_directory + """ + } + deleteDir() + } +} diff --git a/vars/uploadSBOMfromAWS.groovy b/vars/uploadSBOMfromAWS.groovy new file mode 100644 index 0000000000..289a85ef03 --- /dev/null +++ b/vars/uploadSBOMfromAWS.groovy @@ -0,0 +1,26 @@ +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH, String SBOMType, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { + deleteDir() + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) + unstash "uploadPath-${PRODUCT_VERSION}" + withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { + sh """ + export path_to_build=`cat uploadPath-${PRODUCT_VERSION}` + + cat /etc/hosts > hosts + echo '10.30.6.9 repo.ci.percona.com' >> hosts + sudo cp ./hosts /etc || true + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + mkdir -p \${path_to_build}/${SBOMType}/ + + scp -o StrictHostKeyChecking=no -i ${KEY_PATH} \ + `find . -name '*.json'` \ + ${USER}@repo.ci.percona.com:\${path_to_build}/${SBOMType}/ + + """ + } + deleteDir() + } +} diff --git a/vars/uploadTarballToDownloadsTesting.groovy b/vars/uploadTarballToDownloadsTesting.groovy index f31b52deae..89502eb4b1 100644 --- a/vars/uploadTarballToDownloadsTesting.groovy +++ b/vars/uploadTarballToDownloadsTesting.groovy @@ -1,10 +1,12 @@ -def call(String PRODUCT_NAME, String PRODUCT_VERSION) { - node('master') { +def call(String CLOUD_NAME, String PRODUCT_NAME, String PRODUCT_VERSION) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'master' + node(nodeLabel) { deleteDir() unstash 'uploadPath' def path_to_build = sh(returnStdout: true, script: "cat uploadPath").trim() withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) { sh """ + #!/bin/bash set -o xtrace cat /etc/hosts > hosts @@ -15,23 +17,16 @@ def call(String PRODUCT_NAME, String PRODUCT_VERSION) { cutProductVersion=\$(echo ${PRODUCT_VERSION} | sed 's/release-//g'); # Get PXC version from a file - if [ "x${PRODUCT_NAME}" == "xpxc" ]; then - if [[ "${PRODUCT_VERSION}" != *CUSTOM* ]]; then - curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${PRODUCT_VERSION}/MYSQL_VERSION - MYSQL_VERSION_MAJOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MAJOR | awk -F= '{print \$2}') - MYSQL_VERSION_MINOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') - MYSQL_VERSION_PATCH=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_PATCH | awk -F= '{print \$2}') - cutProductVersion=\${MYSQL_VERSION_MAJOR}.\${MYSQL_VERSION_MINOR}.\${MYSQL_VERSION_PATCH} - fi + if [ \"x${PRODUCT_NAME}\" = \"xpxc\" ]; then + curl -O https://raw.githubusercontent.com/percona/percona-xtradb-cluster/${PRODUCT_VERSION}/MYSQL_VERSION + MYSQL_VERSION_MAJOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MAJOR | awk -F= '{print \$2}') + MYSQL_VERSION_MINOR=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_MINOR | awk -F= '{print \$2}') + MYSQL_VERSION_PATCH=\$(cat MYSQL_VERSION | grep MYSQL_VERSION_PATCH | awk -F= '{print \$2}') + cutProductVersion=\${MYSQL_VERSION_MAJOR}.\${MYSQL_VERSION_MINOR}.\${MYSQL_VERSION_PATCH} fi - - if [[ "${PRODUCT_NAME}" != *gated* ]]; then - ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ - ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion} - - ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ - rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/binary/tarball/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion}/ - else + echo \${cutProductVersion} + if expr \"${PRODUCT_NAME}\" : \".*gated.*\" > /dev/null; then + echo "Processing gated product" ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ mkdir -p /srv/repo-copy/private/qa-test/${PRODUCT_NAME}-\${cutProductVersion} @@ -40,6 +35,13 @@ def call(String PRODUCT_NAME, String PRODUCT_VERSION) { ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ rsync -avt --delete --delete-excluded --delete-after --progress --exclude=*.sh --exclude=*.bak /srv/repo-copy/private/qa-test/* 10.30.9.32:/www/repo.percona.com/htdocs/private/qa-test/ + else + echo "Processing non-gated product" + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + ssh -p 2222 jenkins-deploy.jenkins-deploy.web.r.int.percona.com mkdir -p /data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion} + + ssh -o StrictHostKeyChecking=no -i ${KEY_PATH} ${USER}@repo.ci.percona.com \ + rsync -avt -e '"ssh -p 2222"' --bwlimit=50000 --progress ${path_to_build}/binary/tarball/* jenkins-deploy.jenkins-deploy.web.r.int.percona.com:/data/downloads/TESTING/${PRODUCT_NAME}-\${cutProductVersion}/ fi curl https://www.percona.com/admin/config/percona/percona_downloads/crawl_directory diff --git a/vars/uploadTarballfromAWS.groovy b/vars/uploadTarballfromAWS.groovy index 4c9d363c55..f941cf26e6 100644 --- a/vars/uploadTarballfromAWS.groovy +++ b/vars/uploadTarballfromAWS.groovy @@ -1,7 +1,8 @@ -def call(String FOLDER_NAME, String AWS_STASH_PATH, String TarballType) { - node('master') { +def call(String CLOUD_NAME, String FOLDER_NAME, String AWS_STASH_PATH, String TarballType) { + def nodeLabel = (CLOUD_NAME == 'Hetzner') ? 'launcher-x64' : 'micro-amazon' + node(nodeLabel) { deleteDir() - popArtifactFolder(FOLDER_NAME, AWS_STASH_PATH) + popArtifactFolder(CLOUD_NAME, FOLDER_NAME, AWS_STASH_PATH) //unstash "${TarballType}.tarball" unstash 'uploadPath' withCredentials([sshUserPrivateKey(credentialsId: 'repo.ci.percona.com', keyFileVariable: 'KEY_PATH', usernameVariable: 'USER')]) {