diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 2d1a2592..455a2926 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -119,50 +119,6 @@ def getRpmInstallStep(): command=["./rpm-install.sh"], ) -def getDebGaleraStep(port): - def if_run_galera_test(step): - if step.getProperty("sst_mode") == "off": - return False - return True - - return Test( - name="galera", - warningPattern="Test warning:.*", - description=["testing", "galera", "SST"], - descriptionDone=["galera", "SST"], - timeout=300, - lazylogfiles=True, - logfiles={ - "daemon": "/home/buildbot/logs/daemon.log", - "syslog": "/home/buildbot/logs/syslog", - "node1": "/home/buildbot/logs/node1.err", - "node2": "/home/buildbot/logs/node2.err", - "node3": "/home/buildbot/logs/node3.err", - "node1.mariabackup.prepare": "/home/buildbot/logs/node1.mariabackup.prepare.log", - "node2.mariabackup.prepare": "/home/buildbot/logs/node2.mariabackup.prepare.log", - "node3.mariabackup.prepare": "/home/buildbot/logs/node3.mariabackup.prepare.log", - "node1.mariabackup.move": "/home/buildbot/logs/node1.mariabackup.move.log", - "node2.mariabackup.move": "/home/buildbot/logs/node2.mariabackup.move.log", - "node3.mariabackup.move": "/home/buildbot/logs/node3.mariabackup.move.log", - "node1.mariabackup.backup": "/home/buildbot/logs/node1.mariabackup.backup.log", - "node2.mariabackup.backup": "/home/buildbot/logs/node2.mariabackup.backup.log", - "node4.mariabackup.backup": "/home/buildbot/logs/node4.mariabackup.backup.log", - }, - doStepIf=if_run_galera_test, - env=envFromProperties( - [ - "BB_CI", - "arch", - "artifactsURL", - "master_branch", - "sst_mode", - "version_name", - ] - ), - command=["./deb-galera.sh"], - ) - - def getDebUpgradeStep(): return Test( name="upgrade", @@ -235,7 +191,6 @@ def getMajorVersionStep(): f_deb_install = util.BuildFactory() f_deb_install.addStep(getScript("deb-install.sh")) f_deb_install.addStep(getDebInstallStep()) -f_deb_install.addStep(getScript("deb-galera.sh")) ## f_deb_upgrade f_deb_upgrade = util.BuildFactory() diff --git a/master.cfg b/master.cfg index 770f1cab..6337e48b 100644 --- a/master.cfg +++ b/master.cfg @@ -248,55 +248,6 @@ f_deb_autobake.addStep( "mariadb_version": Property("mariadb_version"), "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), - "sst_mode": "off", - }, - doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), - ) -) -f_deb_autobake.addStep( - steps.Trigger( - name="galera-sst-mariabackup", - schedulerNames=["s_install"], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - "sst_mode": "mariabackup", - }, - doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), - ) -) -f_deb_autobake.addStep( - steps.Trigger( - name="galera-sst-mysqldump", - schedulerNames=["s_install"], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - "sst_mode": "mysqldump", - }, - doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), - ) -) -f_deb_autobake.addStep( - steps.Trigger( - name="galera-sst-rsync", - schedulerNames=["s_install"], - waitForFinish=False, - updateSourceStamp=False, - set_properties={ - "tarbuildnum": Property("tarbuildnum"), - "mariadb_version": Property("mariadb_version"), - "master_branch": Property("master_branch"), - "parentbuildername": Property("buildername"), - "sst_mode": "rsync", }, doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), ) diff --git a/scripts/deb-galera.sh b/scripts/deb-galera.sh deleted file mode 100755 index e9d0e0db..00000000 --- a/scripts/deb-galera.sh +++ /dev/null @@ -1,300 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2154 - -set -e - -# Buildbot installation test script -# this script can be called manually by providing the build URL as argument: -# ./script.sh "https://buildbot.mariadb.org/#/builders/171/builds/7351" - -# load common functions -# shellcheck disable=SC1091 -. ./bash_lib.sh - -store_logs() { - # Make sure we store all existing logs, whenever we decide to exit game - set +e - mkdir /home/buildbot/logs - for log in /var/log/daemon.log /var/log/syslog; do - [[ -f $log ]] && sudo cp $log /home/buildbot/logs - done - # It looks like buildbot may be enveloping the path into single quotes - # if it has wildcards, as in '/var/lib/node*/node*.err'. - # Trying to get rid of the wildcards - for node in 1 2 3; do - sudo cp /var/lib/node${node}/node${node}.err /home/buildbot/logs - done - if [[ $sst_mode == "mariabackup" ]]; then - for node in 1 2 3; do - for log in prepare move backup; do - if [[ -f /var/lib/node${node}/mariabackup.${log}.log ]]; then - sudo cp /var/lib/node${node}/mariabackup.${log}.log /home/buildbot/logs/node${node}.mariabackup.${log}.log - fi - done - done - sudo chown -R buildbot:buildbot /home/buildbot/logs - fi - sudo chmod -R +r /home/buildbot/logs - ls -l /home/buildbot/logs -} - -# function to be able to run the script manually (see bash_lib.sh) -manual_run_switch "$1" - -# Mandatory variables -for var in arch master_branch version_name sst_mode; do - if [[ -z $var ]]; then - bb_log_err "$var is not defined" - exit 1 - fi -done - -# Limitations coming from Percona -if [[ $sst_mode == "xtrabackup-v2" ]]; then - case $version_name in - "bionic" | "focal" | "groovy" | "hirsute" | "bullseye") - xtrabackup_version="0.1-5" - ;; - *) - xtrabackup_version="0.1-4" - ;; - esac -fi - -# //TEMP probably need to remove mysqld in the future -bb_log_info "make sure mariadb is not running" -for process in mariadbd mysqld; do - if pgrep $process >/dev/null; then - sudo killall $process >/dev/null || true - fi -done - -# give mariadb the time to shutdown -for i in {1..10}; do - if pgrep 'mysqld|mariadbd'; then - bb_log_info "give mariadb the time to shutdown ($i)" - sleep 3 - else - break - fi -done - -# We don't want crash recovery, but if mariadb hasn't stopped, we'll have to -# deal with it -for process in mariadbd mysqld; do - if pgrep $process >/dev/null; then - sudo killall -s 9 $process >/dev/null || true - fi -done - -if [[ $sst_mode == "mariabackup" ]]; then - # Starting from 10.3.6, MariaBackup packages have a generic name mariadb-backup. - # Before 10.3, MariaBackup packages have a version number in them, e.g. mariadb-backup-10.2 etc. - # First, try to find the generic package, if can't, then the name with the number - - if [ "$master_branch" != 10.3 ]; then - mbackup="mariadb-backup" - else - #//TEMP to be tested !! - mbackup=$(apt-cache search mariadb-backup | grep -v dbgsym | awk '{print $1}' | uniq) - if [[ -z $mbackup ]]; then - bb_log_warn "mariabackup is not available for installing?" - exit 1 - fi - fi - bb_log_info "installing $mbackup" - if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install --allow-unauthenticated -y $mbackup socat"; then - bb_log_warn "failed to install MariaBackup" - exit 1 - fi -elif [[ $sst_mode == "xtrabackup-v2" ]]; then - wget "https://repo.percona.com/apt/percona-release_${xtrabackup_version}.${version_name}_all.deb" - sudo dpkg -i "percona-release_${xtrabackup_version}.${version_name}_all.deb" - apt_get_update - if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install --allow-unauthenticated -y percona-xtrabackup-24 socat"; then - bb_log_warn "could not install XtraBackup, check if it's available for this version/architecture" - exit 1 - fi -fi - -# # //TEMP not done by corp, should we? -# bb_log_info "run MTR tests for the corresponding SST method ($sst_mode)" -# sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install --allow-unauthenticated $allow_downgrades -y mariadb-test" -# cd /usr/share/mysql/mysql-test -# perl mysql-test-run.pl --vardir="$(readlink -f /dev/shm/var)" --force --max-save-core=0 --max-save-datadir=0 --big-test --suite=galera --do-test="galera_sst_${sst_mode}*" -# res=$? -# rm -rf /home/buildbot/var -# cp -r /dev/shm/var /home/buildbot -# if ((res != 0)); then -# bb_log_err "MTR tests failed" -# exit $res -# fi - -sudo cp -r /var/lib/mysql /var/lib/node1 -sudo cp -r /var/lib/mysql /var/lib/node2 -sudo cp -r /var/lib/mysql /var/lib/node3 -sudo chown -R mysql:mysql /var/lib/node1 /var/lib/node2 /var/lib/node3 -# To make sure that xtrabackup / mariabackup works with the right datadir -sudo mv /var/lib/mysql /var/lib/mysql.save - -cat >/home/buildbot/galera.cnf <> /etc/mysql/conf.d/galera.cnf" -fi - -if [[ $sst_mode != "rsync" ]]; then - sudo sh -c "echo 'wsrep_sst_auth=galera:gal3ra123' >> /etc/mysql/conf.d/galera.cnf" -fi - -cat >/home/buildbot/node1.cnf </home/buildbot/node2.cnf </home/buildbot/node3.cnf </dev/null; then - sudo killall rsync >/dev/null || true - fi - fi - sudo mysqld_safe --defaults-extra-file=/home/buildbot/node$node.cnf --user=mysql & - res=1 - set +x - bb_log_info "waiting till node $node comes up..." - for i in {1..20}; do - sleep 5 - # echoing to 2>/dev/null because we do not want mysql connection error to - # be printed ERROR 2002 (HY000): Can't connect to MySQL server on 'localhost' - if mysql -uroot -prootpass --port=830$node --protocol=tcp -e "select * from mgc.t1\G" 2>/dev/null; then - res=0 - break - fi - if [ ! -f /tmp/node${node}.pid ]; then - bb_log_info "missing pid file" - fi - bb_log_info "still waiting for node $node to come up ($i)..." - date +'%Y-%m-%dT%H:%M:%S%z' - sudo tail -n 5 /var/lib/node${node}/node${node}.err || true - done - set -x - if [ "$res" != "0" ]; then - bb_log_err "failed to start node $node or to connect to it after the start" - sudo tail -n 50 /var/lib/node${node}/node${node}.err || true - store_logs - exit 1 - fi - mysql -uroot -prootpass --port=830$node --protocol=tcp -e "select * from mgc.t1\G" -done - -mysql -uroot -prootpass --port=8301 --protocol=tcp -e "show status like 'wsrep_cluster_size'" - -store_logs - -set -e -mysql -uroot -prootpass --port=8301 --protocol=tcp -e "show status like 'wsrep_cluster_size'" | grep 3 - -for node in 2 3; do - mysql -uroot -prootpass --port=830${node} --protocol=tcp -e "select * from mgc.t1\G" -done - -mysql -uroot -prootpass --port=8303 --protocol=tcp -e "drop table mgc.t1" -# check that previous drop was replicated to other nodes -for node in 2 1; do - if mysql -uroot -prootpass --port=830${node} --protocol=tcp -e "set wsrep_sync_wait=15; use mgc; show tables" | grep -q t1; then - bb_log_err "modification on node 3 was not replicated on node ${node}" - mysql -uroot -prootpass --port=830${node} --protocol=tcp -e "use mgc; show tables" - exit 1 - fi -done - -bb_log_info "stop cluster" -for process in mariadbd mysqld mysqld_safe; do - if pgrep $process >/dev/null; then - sudo killall -s 9 $process >/dev/null || true - fi -done - -bb_log_ok "all done"