OpenStack Xena

defaults/main.yml

@@ -301,6 +301,8 @@ galera_mariadb_backups_user: galera_mariadb_backup
 galera_mariadb_backups_suffix: "{{ inventory_hostname }}"
 galera_mariadb_backups_cnf_file: "/etc/mysql/mariabackup.cnf"
 galera_mariadb_backups_nodes: ["{{ galera_cluster_members[0] }}"]
+galera_mariadb_backups_compress: False
+galera_mariadb_backups_compressor: gzip
 
 galera_mariadb_encryption_enabled: false
 galera_mariadb_encryption_plugin: "file_key_management"

releasenotes/notes/mariabackup-compression-337b04c68f370c1d.yaml

@@ -0,0 +1,28 @@
+---
+features:
+  - |
+    Adds optional compression for backups created with mariabackup. Adds two
+    new CLI parameters to the mariabackup script that are used to enable
+    compression and to choose a compression tool.
+
+    * ``--compress=True|False``
+    * ``--compressor=<compressor>``
+
+    Also introduces new Ansible variables that control the above mentioned
+    parameters.
+
+    * ``galera_mariadb_backups_compress``
+    * ``galera_mariadb_backups_compressor``
+
+    Each backup archive is stored in a dedicated directory, alongside the
+    backup metadata.
+upgrade:
+  - |
+    Backup compression is disabled by default, so no changes need to be made
+    for existing deployments. Should compression be desired, set
+    ``galera_mariadb_backups_compress`` to ``True``. Choose a compression tool
+    with ``galera_mariadb_backups_compressor``, default is ``gzip``.
+others:
+  - |
+    Compressed backups cannot be prepared in advance, this step must be
+    manually carried out by the user before importing it into MariaDB.

tasks/galera_server_backups.yml

@@ -39,6 +39,7 @@
           - /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }}
             --full-backup --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }}
             --defaults-file={{ galera_mariadb_backups_cnf_file }}
+            --compress={{ galera_mariadb_backups_compress }} --compressor={{ galera_mariadb_backups_compressor }}
         environment:
           UMASK: '0640'
           UMASK_DIR: '0750'
@@ -65,6 +66,7 @@
           - /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }}
             --increment --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }}
             --defaults-file={{ galera_mariadb_backups_cnf_file }}
+            --compress={{ galera_mariadb_backups_compress }} --compressor={{ galera_mariadb_backups_compressor }}
         environment:
           UMASK: '0640'
           UMASK_DIR: '0750'

tasks/galera_server_cluster_state.yml

@@ -26,9 +26,10 @@
       The cluster may be broken, the cluster state is not known to be good.
       Fix the cluster state before re-running the playbooks. To ignore the
       cluster state set '-e galera_ignore_cluster_state=true'.
-  when:
-    - _node_status.rc != 0
-    - (_node_status.stdout.split()[-1] | default(false)) in ["2", "4"]
+  # State 2 means Donor, State 4 means Synced
+  when: >-
+    _node_status.rc != 0
+    or (_node_status.stdout.split()[-1] | default(false)) not in ["2", "4"]
 
 - name: Check cluster name
   command: >
@@ -46,6 +47,6 @@
       "/etc/mysql/conf.d/cluster.cnf" on the host, and the "wsrep_cluster_name"
       value set in your running galera cluster. To ignore the
       cluster state set '-e galera_ignore_cluster_state=true'.
-  when:
-    - _cluster_name.rc != 0
-    - (_cluster_name.stdout.split()[-1] | default(false)) != galera_cluster_name
+  when: >-
+    _cluster_name.rc != 0
+    or (_cluster_name.stdout.split()[-1] | default(false)) != galera_cluster_name

templates/mariabackup_script.py.j2

@@ -1,11 +1,10 @@
 #!/usr/bin/python3
 # {{ ansible_managed }}
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, check_output, run
 from argparse import ArgumentParser
 from shutil import rmtree
 from time import strftime, mktime, sleep
 from datetime import datetime, timedelta
-import socket
 import os
 
 def get_opts():
@@ -35,6 +34,21 @@ def get_opts():
         default=False,
         help="Flag to make incremental backup, based on the latest backup",
         )
+    parser.add_argument(
+        "--compress",
+        dest="compress_flag",
+        default=False,
+        type=eval,
+        choices=[True, False],
+        help="Flag to compress created backups",
+        )
+    parser.add_argument(
+        "--compressor",
+        dest="compressor",
+        default="gzip",
+        type=str,
+        help="The compressor to use when compressing backups (default: gzip)",
+        )
     parser.add_argument(
         "-c",
         "--copies",
@@ -102,30 +116,44 @@ def check_backups(dest, warning, critical, full_backup_filename):
         raise SystemExit(0)
 
 
-def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args):
+def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args, compress, compressor):
     check_lock_file()
     get_lock_file()
     try:
-        #Creating full backup
         err = open(os.path.normpath(dest+"/backup.log"), "w")
-        mariabackup_run = Popen(
-        ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err
-        )
-        mariabackup_run.wait()
-        mariabackup_res = mariabackup_run.communicate()
-        if mariabackup_run.returncode:
-            print(mariabackup_res[1])
+        if compress:
+            #Creating compressed full backup
+            os.makedirs(dest+"/"+full_backup_filename+curtime, exist_ok=True)
+            mariabackup_run = Popen(
+            ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--extra-lsndir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=PIPE, stderr=err
+            )
+            compressed_backup = open(os.path.normpath(dest+"/"+full_backup_filename+curtime+"/"+full_backup_filename+curtime), "wb")
+            run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup)
+            mariabackup_run.wait()
+            mariabackup_res = mariabackup_run.communicate()
+            if mariabackup_run.returncode:
+                print(mariabackup_res[1])
+            compressed_backup.close()
+        else:
+            #Creating full backup
+            mariabackup_run = Popen(
+            ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err
+            )
+            mariabackup_run.wait()
+            mariabackup_res = mariabackup_run.communicate()
+            if mariabackup_run.returncode:
+                print(mariabackup_res[1])
+            #Preparing full backup
+            err_p = open(os.path.normpath(dest+"/prepare.log"), "w")
+            mariabackup_prep = Popen(
+            ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p
+            )
+            mariabackup_prep.wait()
+            mariabackup_prep_res = mariabackup_prep.communicate()
+            if mariabackup_prep.returncode:
+                print(mariabackup_prep_res[1])
+            err_p.close()
         err.close()
-        #Preparing full backup
-        err_p = open(os.path.normpath(dest+"/prepare.log"), "w")
-        mariabackup_prep = Popen(
-        ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p
-        )
-        mariabackup_prep.wait()
-        mariabackup_prep_res = mariabackup_prep.communicate()
-        if mariabackup_prep.returncode:
-            print(mariabackup_prep_res[1])
-        err_p.close()
     except OSError:
         print("Please, check that Mariabackup is installed")
     except Exception as e:
@@ -134,7 +162,7 @@ def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_ar
         os.unlink("/var/run/mariabackup-galera/db_backup.pid")
 
 
-def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args):
+def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args, compress, compressor):
     check_lock_file()
     get_lock_file()
     try:
@@ -145,14 +173,28 @@ def create_increment_backup(dest, curtime, increment_backup_filename, extra_mari
         raise SystemExit(1)
     try:
         err = open(os.path.normpath(dest+"/increment.err"), "w")
-        #Creating incremental backup
-        mariabackup_run = Popen(
-        ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err
-        )
-        mariabackup_run.wait()
-        mariabackup_res = mariabackup_run.communicate()
-        if mariabackup_run.returncode:
-            print(mariabackup_res[1])
+        if compress:
+            #Creating compressed incremental backup
+            os.makedirs(dest+"/"+increment_backup_filename+curtime, exist_ok=True)
+            mariabackup_run = Popen(
+            ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--incremental-basedir="+basedir, "--extra-lsndir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime)], stdout=PIPE, stderr=err
+            )
+            compressed_backup = open(os.path.normpath(dest+"/"+increment_backup_filename+curtime+"/"+increment_backup_filename+curtime), "wb")
+            run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup)
+            mariabackup_run.wait()
+            mariabackup_res = mariabackup_run.communicate()
+            if mariabackup_run.returncode:
+                print(mariabackup_res[1])
+            compressed_backup.close()
+        else:
+            #Creating incremental backup
+            mariabackup_run = Popen(
+            ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err
+            )
+            mariabackup_run.wait()
+            mariabackup_res = mariabackup_run.communicate()
+            if mariabackup_run.returncode:
+                print(mariabackup_res[1])
         err.close()
     except OSError:
         print("Please, check that Mariabackup is installed")
@@ -226,11 +268,11 @@ def main():
     if opts.fullbackup_flag and opts.increment_flag:
         raise NameError("Only one flag can be specified per operation")
     elif opts.fullbackup_flag:
-        create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args)
+        create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor)
         rotate_backups(opts.destdir, opts.copies_flag, full_backup_filename, increment_backup_filename)
         raise SystemExit()
     elif opts.increment_flag:
-        create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args)
+        create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor)
         raise SystemExit()
     elif opts.check_flag:
         pass