MDEV-35828: Assertion fails in alloc_root() when memory causes it to …

…call itself

(Variant 2: use stack for buffers)
my_malloc_size_cb_func() has a call to thd->alloc() to produce an error message.
thd->alloc() calls alloc_root(), so one can end up with this stack trace:

  alloc_root()
  THD::alloc()
  my_malloc_size_cb_func()
  my_malloc()
  alloc_root()

where alloc_root() calls itself. This is a problem, as alloc_root() is not
reenterable.

Fixed this by switching my_malloc_size_cb_func() to use space on the stack
instead.

mysql-test/main/errors.result

@@ -231,3 +231,16 @@ Error	1327	Undeclared variable: foo
 Error	1305	PROCEDURE P1 does not exist
 drop procedure P1;
 # End of 10.4 tests
+#
+# MDEV-35828: Assertion fails in alloc_root() when memory causes it to call itself
+#
+CREATE TEMPORARY TABLE t1 (a INT,b INT);
+INSERT INTO t1 VALUES (1,1),(2,2);
+SET
+@tmp=@@max_session_mem_used,
+max_session_mem_used=8192;
+SELECT * FROM (t1 AS t2 LEFT JOIN t1 AS t3 USING (a)),t1;
+ERROR HY000: The MariaDB server is running with the --max-session-mem-used=8192 option so it cannot execute this statement
+DROP TABLE t1;
+SET max_session_mem_used=@tmp;
+# End of 10.6 tests

mysql-test/main/errors.test

@@ -284,3 +284,23 @@ show warnings;
 drop procedure P1;
 
 -- echo # End of 10.4 tests
+
+
+--echo #
+--echo # MDEV-35828: Assertion fails in alloc_root() when memory causes it to call itself
+--echo #
+CREATE TEMPORARY TABLE t1 (a INT,b INT);
+INSERT INTO t1 VALUES (1,1),(2,2);
+
+SET
+  @tmp=@@max_session_mem_used,
+  max_session_mem_used=8192;
+
+--error ER_OPTION_PREVENTS_STATEMENT
+SELECT * FROM (t1 AS t2 LEFT JOIN t1 AS t3 USING (a)),t1;
+
+DROP TABLE t1;
+SET max_session_mem_used=@tmp;
+
+
+--echo # End of 10.6 tests

sql/mysqld.cc

@@ -3513,22 +3513,14 @@ static void my_malloc_size_cb_func(long long size, my_bool is_thread_specific)
         LOCK_thd_kill here (the limit will be enforced on the next allocation).
       */
       if (!mysql_mutex_trylock(&thd->LOCK_thd_kill)) {
-        char buf[50], *buf2;
+        char buf[50], buf2[256];
         thd->set_killed_no_mutex(KILL_QUERY);
         my_snprintf(buf, sizeof(buf), "--max-session-mem-used=%llu",
                     thd->variables.max_mem_used);
-        if ((buf2= (char*) thd->alloc(256)))
-        {
-          my_snprintf(buf2, 256,
-                      ER_THD(thd, ER_OPTION_PREVENTS_STATEMENT), buf);
-          thd->set_killed_no_mutex(KILL_QUERY,
-                                   ER_OPTION_PREVENTS_STATEMENT, buf2);
-        }
-        else
-        {
-          thd->set_killed_no_mutex(KILL_QUERY, ER_OPTION_PREVENTS_STATEMENT,
-                          "--max-session-mem-used");
-        }
+        my_snprintf(buf2, 256,
+                    ER_THD(thd, ER_OPTION_PREVENTS_STATEMENT), buf);
+        thd->set_killed_no_mutex(KILL_QUERY,
+                                 ER_OPTION_PREVENTS_STATEMENT, buf2);
         mysql_mutex_unlock(&thd->LOCK_thd_kill);
       }
     }