MDBF-814 for install/upgrade save logs on failure

master-libvirt/master.cfg

@@ -198,12 +198,27 @@ def getPAMTestStep():
         command=["./pam-test.sh"],
     )
 
+def getSaveArtifactsStep():
+    return steps.DirectoryUpload(
+        name="save mariadb log files",
+        doStepIf=hasFailed,
+        workersrc="/home/buildbot/logs/",
+        masterdest=util.Interpolate(
+            "/srv/buildbot/packages/"
+            + "%(prop:tarbuildnum)s"
+            + "/logs/"
+            + "%(prop:buildername)s"
+        ),
+    )
+
+
 # FACTORY
 
 ## f_deb_install
 f_deb_install = util.BuildFactory()
 f_deb_install.addStep(getScript("deb-install.sh"))
 f_deb_install.addStep(getDebInstallStep())
+f_deb_install.addStep(getSaveArtifactsStep())
 f_deb_install.addStep(getScript("pam-test.sh"))
 f_deb_install.addStep(getPAMTestStep())
 
@@ -212,11 +227,13 @@ f_deb_upgrade = util.BuildFactory()
 f_deb_upgrade.addStep(getMajorVersionStep())
 f_deb_upgrade.addStep(getScript("deb-upgrade.sh"))
 f_deb_upgrade.addStep(getDebUpgradeStep())
+f_deb_upgrade.addStep(getSaveArtifactsStep())
 
 ## f_rpm_install
 f_rpm_install = util.BuildFactory()
 f_rpm_install.addStep(getScript("rpm-install.sh"))
 f_rpm_install.addStep(getRpmInstallStep())
+f_rpm_install.addStep(getSaveArtifactsStep())
 f_rpm_install.addStep(getScript("pam-test.sh"))
 f_rpm_install.addStep(getPAMTestStep())
 
@@ -225,6 +242,7 @@ f_rpm_upgrade = util.BuildFactory()
 f_rpm_upgrade.addStep(getMajorVersionStep())
 f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh"))
 f_rpm_upgrade.addStep(getRpmUpgradeStep())
+f_rpm_upgrade.addStep(getSaveArtifactsStep())
 
 ####### WORKERS and BUILDERS
 

scripts/bash_lib.sh

@@ -437,27 +437,37 @@ upgrade_test_type() {
   esac
 }
 
-get_columnstore_logs() {
+save_failure_logs() {
+  local logdir=../logs/
+  local logfile=$logdir/$test_mode.log
+  mkdir -p $logdir
+  bb_log_err "Previous step failed, saving mariadb logs"
   if [[ $test_mode == "columnstore" ]]; then
     bb_log_info "storing Columnstore logs in columnstore_logs"
     set +ex
-    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
-    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
-    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-      f=/var/log/mariadb/columnstore/$f
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
+    for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do
+      echo "----------- $f -----------" >>"$logfile"
+      sudo cat "$f" 1>>"$logfile" 2>&1
     done
-    for f in /tmp/columnstore_tmp_files/*; do
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
+    for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \
+            mcs-loadbrm.service mcs-primproc.service [email protected]; do
+        echo "----------- $s -----------" >>"$logfile"
+        sudo journalctl -u "$s" | tee -a "$logfile" 2>&1
     done
+    if [ -d /var/lib/columnstore ]; then
+      tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore
+    fi
+  fi
+  echo "----------- mariadb.service -----------" >>"$logfile"
+  sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1
+  if [ -f "$logfile" ]; then
+    bzip2 "$logfile"
   fi
+  sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \;
 }
 
 check_mariadb_server_and_create_structures() {
   # All the commands below should succeed
-  set -e
   sudo mariadb -e "CREATE DATABASE db"
   sudo mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'foo'),(2,'bar')"
   sudo mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'foo'),(2,'bar')"
@@ -468,19 +478,14 @@ check_mariadb_server_and_create_structures() {
   sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
   sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"
   fi
-  set +e
 }
 
 check_mariadb_server_and_verify_structures() {
   # Print "have_xx" capabilitites for the new server
   sudo mariadb -e "select 'Stat' t, variable_name name, variable_value val from information_schema.global_status where variable_name like '%have%' union select 'Vars' t, variable_name name, variable_value val from information_schema.global_variables where variable_name like '%have%' order by t, name"
   # All the commands below should succeed
-  set -e
   sudo mariadb -e "select @@version, @@version_comment"
   sudo mariadb -e "SHOW TABLES IN db"
   sudo mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'foo'),(4,'bar')"
@@ -495,12 +500,8 @@ check_mariadb_server_and_verify_structures() {
   sudo mariadb -e "SELECT db.f()"
 
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"
   fi
-  set +e
 }
 
 control_mariadb_server() {

scripts/deb-install.sh

@@ -69,6 +69,9 @@ fi
 # apt get update may be running in the background (Ubuntu start).
 apt_get_update
 
+# set -e already set at start of script
+trap save_failure_logs ERR
+
 sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
   apt-get install -y $package_list $columnstore_package_list"
 
@@ -81,22 +84,6 @@ if [[ $systemdCapability == "yes" ]]; then
   if ! sudo systemctl status mariadb --no-pager; then
     sudo journalctl -xe --no-pager
     bb_log_warn "mariadb service isn't running properly after installation"
-    if echo "$package_list" | grep -q columnstore; then
-      bb_log_info "It is likely to be caused by ColumnStore"
-      bb_log_info "problems upon installation, getting the logs"
-      set +e
-      # It is done in such a weird way, because Columnstore currently makes its
-      # logs hard to read
-      for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-        f=/var/log/mariadb/columnstore/$f
-        echo "----------- $f -----------"
-        sudo cat "$f"
-      done
-      for f in /tmp/columnstore_tmp_files/*; do
-        echo "----------- $f -----------"
-        sudo cat "$f"
-      done
-    fi
     bb_log_err "mariadb service didn't start properly after installation"
     exit 1
   fi
@@ -120,6 +107,8 @@ sudo mariadb --verbose -e "create database test; \
   grant all on *.* to galera;"
 sudo mariadb -e "select @@version"
 bb_log_info "test for MDEV-18563, MDEV-18526"
+
+# disabling -e so save_failure_logs won't have an effect while having setting up for next test
 set +e
 
 control_mariadb_server stop
@@ -134,6 +123,7 @@ for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do
   fi
 done
 sudo mariadb-install-db --no-defaults --user=mysql --plugin-maturity=unknown
+
 set +e
 ## Install mariadb-test for further use
 # sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install -y mariadb-test"

scripts/deb-upgrade.sh

@@ -130,12 +130,8 @@ fi
 # To avoid confusing errors in further logic, do an explicit check
 # whether the service is up and running
 if [[ $systemdCapability == "yes" ]]; then
-  if ! sudo systemctl status mariadb --no-pager; then
-    sudo journalctl -xe --no-pager
-    get_columnstore_logs
-    bb_log_err "mariadb service didn't start properly after installation"
-    exit 1
-  fi
+  bb_log_info "Ensure mariadb.service is running"
+  sudo systemctl status mariadb --no-pager
 fi
 
 if [[ $test_mode == "all" ]]; then
@@ -164,6 +160,9 @@ deb_setup_bb_galera_artifacts_mirror
 deb_setup_bb_artifacts_mirror
 apt_get_update
 
+# now we upgrade, this is what we should save
+trap save_failure_logs ERR
+set -e
 # Install the new packages
 if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
   apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then

scripts/rpm-install.sh

@@ -58,6 +58,10 @@ set -u
 
 sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' |
   sudo tee /etc/my.cnf.d/galera.cnf
+
+# Any of the below steps could fail
+trap save_failure_logs ERR
+set -e
 case "$systemdCapability" in
   yes)
     if ! sudo systemctl start mariadb; then

scripts/rpm-upgrade.sh

@@ -194,6 +194,12 @@ fi
 
 rpm_setup_bb_galera_artifacts_mirror
 rpm_setup_bb_artifacts_mirror
+
+# Any of the below steps could fail
+# This is where the new packages are processed from
+trap save_failure_logs ERR
+set -e
+
 if [[ $test_type == "major" ]]; then
   # major upgrade (remove then install)
   echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install