DRAFT: MDEV-34705: Storing binlog in InnoDB

include/rpl_gtid_base.h

@@ -0,0 +1,130 @@
+/* Copyright (c) 2013,2024, Kristian Nielsen and MariaDB Services Ab.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA */
+
+#ifndef RPL_GTID_BASE_H
+#define RPL_GTID_BASE_H
+
+#include "hash.h"
+
+
+/* Definitions for MariaDB global transaction ID (GTID). */
+
+struct slave_connection_state;
+
+struct rpl_gtid
+{
+  uint32 domain_id;
+  uint32 server_id;
+  uint64 seq_no;
+};
+
+/*
+  Binlog state.
+
+  A binlog state records the last GTID written to the binlog for every
+  distinct (domain_id, server_id) pair. Thus, each point in the binlog
+  corresponds to a specific binlog state.
+
+  When starting replication from a specific GTID position, the starting point
+  is identified as the most recent one where the binlog state has no higher
+  seq_no than the GTID position for any (domain_id, server_id) combination.
+
+  We also remember the most recent logged GTID for every domain_id. This is
+  used to know where to start when a master is changed to a slave. As a side
+  effect, it also allows to skip a hash lookup in the very common case of
+  logging a new GTID with same server id as last GTID.
+
+  This base class rpl_binlog_state_base contains just be basic data operations
+  to insert/update GTIDs, and is used eg. from Gtid_index_*.
+*/
+struct rpl_binlog_state_base
+{
+  struct element {
+    uint32 domain_id;
+    HASH hash;                /* Containing all server_id for one domain_id */
+    /* The most recent entry in the hash. */
+    rpl_gtid *last_gtid;
+    /* Counter to allocate next seq_no for this domain. */
+    uint64 seq_no_counter;
+
+    int update_element(const rpl_gtid *gtid);
+  };
+
+  /* Mapping from domain_id to collection of elements. */
+  HASH hash;
+  my_bool initialized;
+
+  rpl_binlog_state_base() : initialized(0) {}
+  ~rpl_binlog_state_base();
+  void init();
+  void reset_nolock();
+  void free();
+  bool load_nolock(struct rpl_gtid *list, uint32 count);
+  bool load_nolock(rpl_binlog_state_base *orig_state);
+  int update_nolock(const struct rpl_gtid *gtid);
+  int alloc_element_nolock(const rpl_gtid *gtid);
+  uint32 count_nolock();
+  int get_gtid_list_nolock(rpl_gtid *gtid_list, uint32 list_size);
+  rpl_gtid *find_nolock(uint32 domain_id, uint32 server_id);
+  bool is_before_pos(slave_connection_state *pos);
+
+  /*
+    Inline iterator over a binlog state, most recent GTID comes last for each
+    domain just like get_gtid_list_nolock().
+
+    The ITERATOR_FUNC should have signature bool f(const rpl_gtid *), and
+    return true in case of error (in which case iterate() aborts and also
+    returns true).
+
+    Intended to do custom GTID state processing without requiring the overhead
+    of an intermediate list, and where the extra code generation is justified.
+  */
+  template <typename F> bool iterate(F iterator_func)
+  {
+    uint32 i, j;
+    ulong outer_records= hash.records;
+
+    for (i= 0; i < outer_records; ++i)
+    {
+      element *e= (element *)my_hash_element(&hash, i);
+      ulong inner_records= e->hash.records;
+      const rpl_gtid *last_gtid= e->last_gtid;
+      if (unlikely(!last_gtid))
+      {
+        DBUG_ASSERT(inner_records==0);
+        continue;
+      }
+      for (j= 0; j <= inner_records; ++j)
+      {
+        const rpl_gtid *gtid;
+        if (j < inner_records)
+        {
+          gtid= (rpl_gtid *)my_hash_element(&e->hash, j);
+          if (gtid == last_gtid)
+            continue;
+        }
+        else
+          gtid= e->last_gtid;
+        if (iterator_func(gtid))
+          return true;
+      }
+    }
+
+    return false;  // No error
+  }
+};
+
+
+#endif  /* RPL_GTID_BASE_H */

mysql-test/include/have_innodb_binlog.inc

@@ -0,0 +1 @@
+--source include/have_innodb.inc

mysql-test/include/have_innodb_binlog.opt

@@ -0,0 +1 @@
+--log-bin --binlog-storage-engine=innodb --max-binlog-size=256K

mysql-test/include/rpl_sync.inc

@@ -126,14 +126,30 @@ while ($_rpl_i) {
     if (!$rpl_only_running_threads)
     {
       disable_connect_log;
+      # ToDo: With binlog-in-engine, we need to sync up using GTID.
+      # Is it ok to change things to always do it with GTID when GTID enabled?
+      --connection server_$_rpl_server
+      --let $_rpl_gtid_mode= query_get_value(SHOW SLAVE STATUS, Using_Gtid, 1)
       --connection server_$_rpl_prev_server
-      if ($rpl_debug)
-      {
-        --let $_rpl_master_file= query_get_value("SHOW MASTER STATUS", File, 1)
-        --let $_rpl_master_pos= query_get_value("SHOW MASTER STATUS", Position, 1)
-        --echo syncing master_file='$_rpl_master_file' master_pos='$_rpl_master_pos'
+      if ($_rpl_gtid_mode == 'No') {
+	if ($rpl_debug)
+	{
+	  --let $_rpl_master_file= query_get_value("SHOW MASTER STATUS", File, 1)
+	  --let $_rpl_master_pos= query_get_value("SHOW MASTER STATUS", Position, 1)
+	  --echo syncing master_file='$_rpl_master_file' master_pos='$_rpl_master_pos'
+	}
+        --sync_slave_with_master server_$_rpl_server
+      }
+      if ($_rpl_gtid_mode != 'No') {
+        --source include/save_master_gtid.inc
+	disable_connect_log;
+        --connection server_$_rpl_server
+	if ($rpl_debug)
+	{
+	  --echo syncing with GTID position'$master_pos'
+	}
+        --source include/sync_with_master_gtid.inc
       }
-      --sync_slave_with_master server_$_rpl_server
       enable_connect_log;
     }
   }

mysql-test/lib/v1/mtr_cases.pl

@@ -864,6 +864,7 @@ ($$$$$$$$$)
   "binlog_formats", ["row", "statement"]],
 
  ["include/have_innodb.inc", "innodb_test", 1],
+ ["include/have_innodb_binlog.inc", "innodb_test", 1],
  ["include/have_log_bin.inc", "need_binlog", 1],
  ["include/big_test.inc", "big_test", 1],
  ["include/have_debug.inc", "need_debug", 1],

mysql-test/main/mysqld--help.result

@@ -78,6 +78,10 @@ The following specify which files/extra groups are read (specified before remain
  tables such as in the statement INSERT INTO t_myisam
  SELECT * FROM t_innodb; otherwise, slaves may diverge
  from the master.
+ --binlog-directory=name 
+ Directory path (absolute or relative to datadir) where
+ binlog files are stored. If this is used, must not
+ specify a directory path for --log-bin
  --binlog-do-db=name Tells the master it should log updates for the specified
  database, and exclude all others not explicitly
  mentioned.
@@ -158,6 +162,10 @@ The following specify which files/extra groups are read (specified before remain
  non-transactional engines for the binary log. If you
  often use statements updating a great number of rows, you
  can increase this to get more performance.
+ --binlog-storage-engine=name 
+ Store the binlog transactionally in a supporting storage
+ engine instead of as separate files. Currently only
+ InnoDB is supported.
  --block-encryption-mode=name 
  Default block encryption mode for AES_ENCRYPT() and
  AES_DECRYPT() functions. One of: aes-128-ecb, aes-192-ecb,
@@ -1629,6 +1637,7 @@ binlog-checksum CRC32
 binlog-commit-wait-count 0
 binlog-commit-wait-usec 100000
 binlog-direct-non-transactional-updates FALSE
+binlog-directory (No default value)
 binlog-expire-logs-seconds 0
 binlog-file-cache-size 16384
 binlog-format MIXED
@@ -1642,6 +1651,7 @@ binlog-row-image FULL
 binlog-row-metadata NO_LOG
 binlog-space-limit 0
 binlog-stmt-cache-size 32768
+binlog-storage-engine 
 block-encryption-mode aes-128-ecb
 bulk-insert-buffer-size 8388608
 character-set-client-handshake TRUE

mysql-test/mariadb-test-run.pl

@@ -180,6 +180,7 @@ END
     atomic-
     binlog-
     binlog_encryption-
+    binlog_in_engine-
     client-
     csv-
     compat/oracle-

mysql-test/suite/binlog_in_engine/binlog_flush_purge.test

@@ -0,0 +1,35 @@
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+
+RESET MASTER;
+
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(2048)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+COMMIT;
+INSERT INTO t2 VALUES (0, REPEAT("x", 2048));
+INSERT INTO t2 SELECT a+1, b FROM t2;
+INSERT INTO t2 SELECT a+2, b FROM t2;
+INSERT INTO t2 SELECT a+4, b FROM t2;
+INSERT INTO t2 SELECT a+8, b FROM t2;
+
+SHOW BINARY LOGS;
+FLUSH BINARY LOGS;
+SHOW BINARY LOGS;
+
+RESET MASTER;
+# ToDo: This will be racy, the second binlog file binlog-000001.ibb is
+# pre-allocated in the background, so will be visible or not depending on exact
+# timing. So just omit this SHOW BINARY LOGS or wait for both to be created
+# or something.
+SHOW BINARY LOGS;
+
+INSERT INTO t1 VALUES (100);
+INSERT INTO t2 VALUES (100, 'xyzzy');
+
+DROP TABLE t1, t2;
+
+--exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 --start-position=0-1-1 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog.txt

mysql-test/suite/binlog_in_engine/binlog_in_engine.test

@@ -0,0 +1,38 @@
+--source include/have_binlog_format_mixed.inc
+# ToDo: For now, this has to come _after_ have_log_bin.inc (or
+# have_binlog_format_*.inc), to override --log-bin=master-bin with empty
+# --log-bin, as engine does not allow to set the binlog name.
+# Alternatively, maybe could have separate have_innodb_binlog_format_*.inc
+# files and only need to include the one.
+--source include/have_innodb_binlog.inc
+
+RESET MASTER;
+
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+--let $gtid_pos= `SELECT @@last_gtid`
+INSERT INTO t1 VALUES (1);
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+COMMIT;
+SELECT * FROM t1 ORDER BY a;
+DROP TABLE t1;
+SELECT @@GLOBAL.binlog_checksum;
+
+CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(2048)) ENGINE=InnoDB;
+
+SET SESSION binlog_format= ROW;
+--let num_trx= 1500
+--echo *** Do $num_trx transactions ...
+--disable_query_log
+--let $i= 0
+while ($i < $num_trx) {
+  eval INSERT INTO t2 VALUES ($i, REPEAT("x", 2048));
+  inc $i;
+}
+--enable_query_log
+SET SESSION binlog_format= MIXED;
+
+--exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 --start-position=$gtid_pos > $MYSQLTEST_VARDIR/tmp/mysqlbinlog.txt
+
+DROP TABLE t2;

mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.opt

@@ -0,0 +1,2 @@
+--innodb-binlog-state-interval=65536
+--binlog-directory=binlogs

mysql-test/suite/binlog_in_engine/binlog_in_engine_restart.test

@@ -0,0 +1,49 @@
+--source include/big_test.inc
+--source include/have_binlog_format_row.inc
+--source include/have_innodb_binlog.inc
+--source include/have_sequence.inc
+
+# Note: This test also tests the --binlog-directory option by putting it
+# in binlog_in_engine_restart.opt .
+
+RESET MASTER;
+
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0);
+let $i= 0;
+while ($i < 10) {
+  eval INSERT INTO t1 SELECT seq+1000000+100000*$i, seq FROM seq_1_to_4000;
+  inc $i;
+}
+SELECT @@GLOBAL.gtid_binlog_state;
+--exec $MYSQL_BINLOG --start-position=42-42-42 --gtid-strict-mode=0 --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog1.txt
+
+--source include/restart_mysqld.inc
+
+SELECT @@GLOBAL.gtid_binlog_state;
+--let $gtid_pos2= `SELECT @@GLOBAL.gtid_binlog_pos`
+INSERT INTO t1 VALUES (2, 0);
+let $i= 0;
+while ($i < 10) {
+  eval INSERT INTO t1 SELECT seq+2000000+100000*$i, seq FROM seq_1_to_4000;
+  inc $i;
+}
+SELECT @@GLOBAL.gtid_binlog_state;
+--exec $MYSQL_BINLOG --start-position=$gtid_pos2 --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog2.txt
+
+--source include/restart_mysqld.inc
+
+SELECT @@GLOBAL.gtid_binlog_state;
+--let $gtid_pos3= `SELECT @@GLOBAL.gtid_binlog_pos`
+INSERT INTO t1 VALUES (3, 0);
+let $i= 0;
+while ($i < 10) {
+  eval INSERT INTO t1 SELECT seq+3000000+100000*$i, seq FROM seq_1_to_4000;
+  inc $i;
+}
+
+SHOW BINARY LOGS;
+
+--exec $MYSQL_BINLOG --start-position=$gtid_pos3 --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog3.txt
+
+DROP TABLE t1;

mysql-test/suite/binlog_in_engine/my.cnf

@@ -0,0 +1,29 @@
+!include include/default_my.cnf
+
+[mysqld.1]
+innodb
+binlog-storage-engine=innodb
+max-binlog-size=2048k
+innodb-binlog-state-interval=128k
+log-basename= master
+
+[mysqld.2]
+#!use-slave-opt
+innodb
+binlog-storage-engine=innodb
+max-binlog-size=2048k
+innodb-binlog-state-interval=128k
+log-slave-updates
+log-basename= slave
+
+[ENV]
+
+# We will adopt tests with master-slave setup as well as rpl_init setup,
+# so need both sets of variables
+MASTER_MYPORT=   @mysqld.1.port
+SERVER_MYPORT_1= @mysqld.1.port
+SERVER_MYSOCK_1= @mysqld.1.socket
+
+SLAVE_MYPORT=    @mysqld.2.port
+SERVER_MYPORT_2= @mysqld.2.port
+SERVER_MYSOCK_2= @mysqld.2.socket

mysql-test/suite/binlog_in_engine/rpl_gtid_index-master.opt

@@ -0,0 +1,3 @@
+--max-binlog-size=128k
+--innodb-binlog-state-interval=64k
+--log-bin

mysql-test/suite/binlog_in_engine/rpl_gtid_index-slave.opt

@@ -0,0 +1 @@
+--log-bin