Optimize binlog handling of large transactions (BOLT)

client/mysqlbinlog.cc

@@ -1199,6 +1199,7 @@ static bool shall_skip_gtids(const Log_event *ev) {
     case mysql::binlog::event::FORMAT_DESCRIPTION_EVENT:
     case mysql::binlog::event::ROTATE_EVENT:
     case mysql::binlog::event::IGNORABLE_LOG_EVENT:
+    case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT:
     case mysql::binlog::event::INCIDENT_EVENT:
       filtered = false;
       break;

include/my_sys.h

@@ -431,6 +431,13 @@ struct IO_CACHE /* Used when caching files */
   void *arg{nullptr};       /* for use by pre/post_read */
   char *file_name{nullptr}; /* if used with 'open_cached_file' */
   char *dir{nullptr}, *prefix{nullptr};
+  /*
+    With 'open_cached_file': create the lazily created temporary file as a
+    named file in the filesystem namespace instead of an anonymous file
+    unlinked at creation. Its name is then recorded in file_name, and closing
+    the cache deletes the file.
+  */
+  bool named_file{false};
   File file{-1};                               /* file descriptor */
   PSI_file_key file_key{PSI_NOT_INSTRUMENTED}; /* instrumented file key */
 

libs/mysql/binlog/event/CMakeLists.txt

@@ -65,6 +65,7 @@ SET(TARGET_SRCS
   binlog_event.cpp
   control_events.cpp
   event_reader.cpp
+  large_transaction_header_event.cpp
   load_data_events.cpp
   rows_event.cpp
   statement_events.cpp

libs/mysql/binlog/event/binlog_event.cpp

@@ -77,6 +77,8 @@ static const std::unordered_map<Log_event_type, const std::string>
                             {PARTIAL_UPDATE_ROWS_EVENT, "Update_rows_partial"},
                             {TRANSACTION_PAYLOAD_EVENT, "Transaction_payload"},
                             {GTID_TAGGED_LOG_EVENT, "Gtid_tagged_log_event"},
+                            {LARGE_TRANSACTION_HEADER_EVENT,
+                             "Large_transaction_header"},
                             {UNKNOWN_EVENT, "Unknown"}};
 
 const std::string &get_event_type_as_string(Log_event_type type) {

libs/mysql/binlog/event/binlog_event.h

@@ -365,6 +365,11 @@ enum Log_event_type {
   HEARTBEAT_LOG_EVENT_V2 = 41,
 
   GTID_TAGGED_LOG_EVENT = 42,
+
+  /**
+    A binlog event used by the large transaction optimization.
+  */
+  LARGE_TRANSACTION_HEADER_EVENT = 43,
   /**
     Add new events here - right above this comment!
     Existing events (except ENUM_END_EVENT) should never change their numbers

libs/mysql/binlog/event/control_events.cpp

@@ -124,7 +124,8 @@ Format_description_event::Format_description_event(uint8_t binlog_ver,
           IGNORABLE_HEADER_LEN, TRANSACTION_CONTEXT_HEADER_LEN,
           VIEW_CHANGE_HEADER_LEN, XA_PREPARE_HEADER_LEN, ROWS_HEADER_LEN_V2,
           TRANSACTION_PAYLOAD_EVENT, 0 /* HEARTBEAT_LOG_EVENT_V2*/,
-          0 /* GTID_TAGGED_LOG_EVENT */
+          0 /* GTID_TAGGED_LOG_EVENT */,
+          0 /* LARGE_TRANSACTION_HEADER_EVENT */
       };
       /*
         Allows us to sanity-check that all events initialized their

libs/mysql/binlog/event/large_transaction_header_event.cpp

@@ -0,0 +1,59 @@
+#include "mysql/binlog/event/large_transaction_header_event.h"
+
+#include "mysql/binlog/event/control_events.h"  // Format_description_event
+#include "mysql/binlog/event/event_reader_macros.h"
+
+namespace mysql::binlog::event {
+
+Large_transaction_header_event::
+    Large_transaction_header_event(
+        const char *buf, const Format_description_event *fde)
+    : Binary_log_event(&buf, fde) {
+  BAPI_ENTER(
+      "Large_transaction_header_event::"
+      "Large_transaction_header_event(const char*, ...)");
+  READER_TRY_INITIALIZATION;
+  READER_ASSERT_POSITION(fde->common_header_len);
+
+  READER_TRY_SET(m_version, read<uint8_t>);
+  if (m_version == 0 || m_version > kVersion) {
+    READER_THROW("Invalid Large_transaction_header version");
+  }
+  READER_TRY_SET(m_terminating_event_offset, read<uint64_t>);
+  /* Read unconditionally: a truncated event missing the type byte is
+     reported as a read error rather than silently defaulting the type. */
+  READER_TRY_SET(m_terminating_event_type, read<uint8_t>);
+
+  /* The remainder of the body is padding; its contents are ignored. */
+  m_padding_size = READER_CALL(available_to_read);
+
+  READER_CATCH_ERROR;
+  BAPI_VOID_RETURN;
+}
+
+Large_transaction_header_event::
+    Large_transaction_header_event(uint64_t terminating_event_offset,
+                                   uint8_t terminating_event_type,
+                                   uint64_t padding_size)
+    : Binary_log_event(LARGE_TRANSACTION_HEADER_EVENT),
+      m_terminating_event_offset(terminating_event_offset),
+      m_terminating_event_type(terminating_event_type),
+      m_padding_size(padding_size) {}
+
+#ifndef HAVE_MYSYS
+void Large_transaction_header_event::print_event_info(std::ostream &info) {
+  info << "terminating event offset " << m_terminating_event_offset;
+}
+
+void Large_transaction_header_event::print_long_info(std::ostream &info) {
+  info << "Timestamp: " << header()->when.tv_sec;
+  info << "\tVersion: " << static_cast<unsigned>(m_version);
+  info << "\tTerminating event offset: " << m_terminating_event_offset;
+  info << "\tTerminating event type: "
+       << static_cast<unsigned>(m_terminating_event_type);
+  info << "\tPadding: " << m_padding_size << " bytes";
+  info << "\n";
+}
+#endif
+
+}  // namespace mysql::binlog::event

libs/mysql/binlog/event/large_transaction_header_event.h

@@ -0,0 +1,133 @@
+/**
+  @file large_transaction_header_event.h
+
+  @brief Deserialization of the Large_transaction_header
+         event. All serialization logic lives in the server class under
+         sql/log_event.*
+*/
+
+#ifndef MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H
+#define MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H
+
+#include <cstdint>
+
+#include "mysql/binlog/event/binlog_event.h"
+
+namespace mysql::binlog::event {
+
+/**
+  @class Large_transaction_header_event
+
+  Event needed for the large transaction optimization. It serves two
+  purposes:
+
+  1. It records the offset of the transaction's terminating event, so
+     that binary log recovery can seek directly past the transaction
+     body instead of scanning it.
+  2. Its variable-length padding fills the file's reserved header region
+     exactly, so the transaction body starts at the offset that was
+     assumed while events were being spilled.
+
+  Always written with the LOG_EVENT_IGNORABLE_F flag set: replicas and
+  binlog tools that do not recognize the type skip it.
+
+  @section Large_transaction_header_event_binary_format
+  Binary Format
+
+  The post-header is empty. The Body has the following components:
+
+  <table>
+  <caption>Body for Large_transaction_header_event</caption>
+  <tr>
+    <th>Name</th>
+    <th>Format</th>
+    <th>Description</th>
+  </tr>
+  <tr>
+    <td>version</td>
+    <td>1 byte unsigned integer</td>
+    <td>Event format version; currently 1. Retained for future
+        extensibility.</td>
+  </tr>
+  <tr>
+    <td>terminating_event_offset</td>
+    <td>8 byte unsigned little-endian integer</td>
+    <td>Offset, within this binary log file, of the transaction's
+        terminating event.</td>
+  </tr>
+  <tr>
+    <td>terminating_event_type</td>
+    <td>1 byte unsigned integer</td>
+    <td>Binary log event type of the terminating event.</td>
+  </tr>
+  <tr>
+    <td>padding</td>
+    <td>variable-length byte sequence</td>
+    <td>Filler occupying the remainder of the reserved header region;
+        contents are undefined and ignored on read.</td>
+  </tr>
+  </table>
+*/
+class Large_transaction_header_event : public Binary_log_event {
+ public:
+  /** Current version of the event's body format. The version field is
+      retained for future extensibility; only this version exists today. */
+  static constexpr uint8_t kVersion = 1;
+  /** Body bytes before padding: version (1) + offset (8) + event type (1). */
+  static constexpr size_t kFixedBodyLength = 1 + 8 + 1;
+
+  /**
+    Deserializing constructor.
+
+    @param buf  Contains the serialized event.
+    @param fde  An FDE event (see Rotate_event constructor for more info).
+  */
+  Large_transaction_header_event(
+      const char *buf, const Format_description_event *fde);
+
+  /**
+    Creates an event with the given terminating-event metadata and
+    padding size (used by the server when writing a promoted binary log
+    file's header).
+
+    @param terminating_event_offset  Offset of the transaction's
+                                     terminating event in the file.
+    @param terminating_event_type    Type code of the transaction's
+                                     terminating event in the file.
+    @param padding_size              Number of filler bytes to occupy the
+                                     remainder of the reserved region.
+  */
+  Large_transaction_header_event(uint64_t terminating_event_offset,
+                                 uint8_t terminating_event_type,
+                                 uint64_t padding_size);
+
+  ~Large_transaction_header_event() override = default;
+
+  uint8_t get_version() const { return m_version; }
+  uint64_t get_terminating_event_offset() const {
+    return m_terminating_event_offset;
+  }
+  uint8_t get_terminating_event_type() const {
+    return m_terminating_event_type;
+  }
+  uint64_t get_padding_size() const { return m_padding_size; }
+
+#ifndef HAVE_MYSYS
+  void print_event_info(std::ostream &info) override;
+  void print_long_info(std::ostream &info) override;
+#endif
+
+ protected:
+  /** Body format version read from (or to be written to) the wire. */
+  uint8_t m_version{kVersion};
+  /** Offset of the transaction's terminating event in this file. */
+  uint64_t m_terminating_event_offset{0};
+  /** Type code of the terminating event; absent in version-1 headers. */
+  uint8_t m_terminating_event_type{0};
+  /** Number of filler bytes following the fixed body fields. */
+  uint64_t m_padding_size{0};
+};
+
+}  // namespace mysql::binlog::event
+
+#endif  // MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H

libs/mysql/binlog/event/trx_boundary_parser.cpp

@@ -252,6 +252,7 @@ Transaction_boundary_parser::get_event_boundary_type(
     case mysql::binlog::event::SLAVE_EVENT:
     case mysql::binlog::event::DELETE_FILE_EVENT:
     case mysql::binlog::event::TRANSACTION_CONTEXT_EVENT:
+    case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT:
       boundary_type = EVENT_BOUNDARY_TYPE_IGNORE;
       break;
 

mysql-test/common/binlog/validate_bolt_file.inc

@@ -0,0 +1,96 @@
+# Requires $bolt_header_file to name a standalone BOLT-promoted binlog.
+# Decode the promoted file and validate its LTH metadata and layout.
+# These checks keep a plausible but corrupt LTH from silently directing
+# optimized recovery to an incorrect terminal event.
+# The awk program avoids `$` field syntax because mysqltest expands it.
+
+--let $bolt_promoted_dump = $MYSQLTEST_VARDIR/tmp/bolt_promoted_dump.txt
+--let $bolt_validator_script = $MYSQLTEST_VARDIR/tmp/validate_bolt_header.awk
+--exec $MYSQL_BINLOG --force-if-open --verify-binlog-checksum --verbose $MYSQLD_DATADIR/$bolt_header_file > $bolt_promoted_dump
+--exec grep -m 1 -E 'last_committed=.*sequence_number=' $bolt_promoted_dump | grep -Eq 'last_committed=0[[:space:]]+sequence_number=1'
+--write_file $bolt_validator_script EOF
+# Extract the end position printed by mysqlbinlog for the current event.
+# Keep the full line in a variable so this does not rely on awk `$` fields.
+function end_pos(line, value) {
+  value = line
+  sub(/^.*end_log_pos /, "", value)
+  sub(/ .*/, "", value)
+  return value + 0
+}
+
+BEGIN {
+  # Read mysqlbinlog's verbose output and associate each event description
+  # with the preceding '# at <offset>' line.
+  while ((getline line) > 0) {
+    if (line ~ /^# at [0-9][0-9]*/) {
+      split(line, words, " ")
+      event_start = words[3] + 0
+      continue
+    }
+
+    # The preceding Previous-GTIDs event must end where the LTH starts.
+    if (line ~ /Previous-GTIDs/ && event_start) {
+      previous_start = event_start
+      previous_end = end_pos(line)
+      continue
+    }
+
+    # A promoted file contains exactly one ignorable LTH in its prefix.
+    if (line ~ /Large transaction header[[:space:]]+Ignorable/ && event_start) {
+      if (lth_start) exit 1
+      lth_start = event_start
+      lth_end = end_pos(line)
+      continue
+    }
+
+    # Parse the LTH's recovery hint: the terminal-event offset, its expected
+    # type code, and the remaining padding that fills the reserved region.
+    if (line ~ /Terminating event offset /) {
+      if (!lth_start || lth_offset) exit 1
+      value = line
+      sub(/^.*Terminating event offset /, "", value)
+      split(value, fields, ", type ")
+      lth_offset = fields[1] + 0
+      split(fields[2], fields, ", padding ")
+      lth_type = fields[1] + 0
+      sub(/ bytes/, "", fields[2])
+      lth_padding = fields[2] + 0
+      continue
+    }
+
+    # Select only the first GTID after the LTH. The promoted active file can
+    # later contain additional GTIDs, which are not part of its header.
+    if (line ~ /GTID.*last_committed=.*sequence_number=/ && event_start == lth_end) {
+      if (gtid_start) exit 1
+      gtid_start = event_start
+      gtid_end = end_pos(line)
+      continue
+    }
+
+    # The recorded terminal offset must identify a decoded event with an end
+    # position beyond its start.
+    if (line ~ /end_log_pos/ && event_start == lth_offset)
+      terminal_end = end_pos(line)
+  }
+
+  # The event type is byte 4 of a binlog event header. Read the raw byte at
+  # the LTH offset so metadata cannot merely agree with a mis-decoded event.
+  command = "dd if=\"" binlog_file "\" bs=1 skip=" (lth_offset + 4) " count=1 2>/dev/null | od -An -tu1"
+  command | getline terminal_type
+  close(command)
+  gsub(/[[:space:]]/, "", terminal_type)
+
+  # Check the promoted prefix is contiguous, the first GTID ends exactly at
+  # the 64 KiB reserved boundary, the LTH points beyond that boundary, and
+  # the stored type is a recovery-supported terminal event (Query, XID, or
+  # XA PREPARE) that matches the raw event-header byte.
+  if (!(previous_start >= 4 && previous_end == lth_start &&
+        lth_end == gtid_start && gtid_end == 65536 &&
+        lth_offset >= gtid_end && terminal_end > lth_offset &&
+        lth_padding > 0 && (lth_type == 2 || lth_type == 16 || lth_type == 38) &&
+        terminal_type == lth_type)) exit 1
+}
+EOF
+--exec awk -v binlog_file=$MYSQLD_DATADIR/$bolt_header_file -f $bolt_validator_script $bolt_promoted_dump
+--remove_file $bolt_promoted_dump
+--remove_file $bolt_validator_script

mysql-test/suite/binlog/inc/validate_bolt_header.inc

@@ -0,0 +1,3 @@
+# Compatibility wrapper for existing BOLT tests.
+# The merged promoted-file validator is shared from mysql-test/common.
+--source common/binlog/validate_bolt_file.inc

mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result

@@ -0,0 +1,25 @@
+Warnings:
+Note	1051	Unknown table 'test.t1'
+# Pre-TC-prepare crash rolls back.
+include/assert.inc [A pre-TC-prepare crash rolls back all rows.]
+# Post-TC-prepare crash rolls back.
+include/assert.inc [A post-TC-prepare crash rolls back all rows.]
+# Post-header-sync crash rolls back.
+include/assert.inc [A post-header-sync crash rolls back all rows.]
+# Post-purge-index-sync crash rolls back.
+include/assert.inc [A post-purge-index-sync crash rolls back all rows.]
+# Post-promote-rename crash rolls back.
+include/assert.inc [A post-promote-rename crash rolls back all rows.]
+# Post-main-index-update crash commits exactly once.
+include/assert.inc [A post-main-index-update crash commits all rows exactly once.]
+include/assert.inc [A post-main-index-update crash preserves the original rows.]
+# Post-purge-index-removal crash commits exactly once.
+include/assert.inc [A post-purge-index-removal crash commits all rows exactly once.]
+include/assert.inc [A post-purge-index-removal crash preserves the original rows.]
+# Pre-engine-commit crash commits exactly once.
+include/assert.inc [A pre-engine-commit crash commits all rows exactly once.]
+include/assert.inc [A pre-engine-commit crash preserves the original rows.]
+# Post-main-index XA PREPARE crash remains prepared and commits exactly once.
+include/assert.inc [A post-main-index XA PREPARE remains prepared until XA COMMIT.]
+include/assert.inc [XA COMMIT commits the recovered promoted rows exactly once.]
+DROP TABLE t1;