diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 1ebd77a7fd02..553b613c35fd 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1199,6 +1199,7 @@ static bool shall_skip_gtids(const Log_event *ev) { case mysql::binlog::event::FORMAT_DESCRIPTION_EVENT: case mysql::binlog::event::ROTATE_EVENT: case mysql::binlog::event::IGNORABLE_LOG_EVENT: + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: case mysql::binlog::event::INCIDENT_EVENT: filtered = false; break; diff --git a/include/my_sys.h b/include/my_sys.h index ee6920fb08bc..b4cb3befb6ab 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -431,6 +431,13 @@ struct IO_CACHE /* Used when caching files */ void *arg{nullptr}; /* for use by pre/post_read */ char *file_name{nullptr}; /* if used with 'open_cached_file' */ char *dir{nullptr}, *prefix{nullptr}; + /* + With 'open_cached_file': create the lazily created temporary file as a + named file in the filesystem namespace instead of an anonymous file + unlinked at creation. Its name is then recorded in file_name, and closing + the cache deletes the file. + */ + bool named_file{false}; File file{-1}; /* file descriptor */ PSI_file_key file_key{PSI_NOT_INSTRUMENTED}; /* instrumented file key */ diff --git a/libs/mysql/binlog/event/CMakeLists.txt b/libs/mysql/binlog/event/CMakeLists.txt index 7864a3d15251..968d214efa85 100644 --- a/libs/mysql/binlog/event/CMakeLists.txt +++ b/libs/mysql/binlog/event/CMakeLists.txt @@ -65,6 +65,7 @@ SET(TARGET_SRCS binlog_event.cpp control_events.cpp event_reader.cpp + large_transaction_header_event.cpp load_data_events.cpp rows_event.cpp statement_events.cpp diff --git a/libs/mysql/binlog/event/binlog_event.cpp b/libs/mysql/binlog/event/binlog_event.cpp index 1f9edc9012a0..9860e5aa3f9e 100644 --- a/libs/mysql/binlog/event/binlog_event.cpp +++ b/libs/mysql/binlog/event/binlog_event.cpp @@ -77,6 +77,8 @@ static const std::unordered_map {PARTIAL_UPDATE_ROWS_EVENT, "Update_rows_partial"}, {TRANSACTION_PAYLOAD_EVENT, "Transaction_payload"}, {GTID_TAGGED_LOG_EVENT, "Gtid_tagged_log_event"}, + {LARGE_TRANSACTION_HEADER_EVENT, + "Large_transaction_header"}, {UNKNOWN_EVENT, "Unknown"}}; const std::string &get_event_type_as_string(Log_event_type type) { diff --git a/libs/mysql/binlog/event/binlog_event.h b/libs/mysql/binlog/event/binlog_event.h index 1036d5e781b6..9ba9616ea2ca 100644 --- a/libs/mysql/binlog/event/binlog_event.h +++ b/libs/mysql/binlog/event/binlog_event.h @@ -365,6 +365,11 @@ enum Log_event_type { HEARTBEAT_LOG_EVENT_V2 = 41, GTID_TAGGED_LOG_EVENT = 42, + + /** + A binlog event used by the large transaction optimization. + */ + LARGE_TRANSACTION_HEADER_EVENT = 43, /** Add new events here - right above this comment! Existing events (except ENUM_END_EVENT) should never change their numbers diff --git a/libs/mysql/binlog/event/control_events.cpp b/libs/mysql/binlog/event/control_events.cpp index 8d43b28c774b..f3ddbe7bd155 100644 --- a/libs/mysql/binlog/event/control_events.cpp +++ b/libs/mysql/binlog/event/control_events.cpp @@ -124,7 +124,8 @@ Format_description_event::Format_description_event(uint8_t binlog_ver, IGNORABLE_HEADER_LEN, TRANSACTION_CONTEXT_HEADER_LEN, VIEW_CHANGE_HEADER_LEN, XA_PREPARE_HEADER_LEN, ROWS_HEADER_LEN_V2, TRANSACTION_PAYLOAD_EVENT, 0 /* HEARTBEAT_LOG_EVENT_V2*/, - 0 /* GTID_TAGGED_LOG_EVENT */ + 0 /* GTID_TAGGED_LOG_EVENT */, + 0 /* LARGE_TRANSACTION_HEADER_EVENT */ }; /* Allows us to sanity-check that all events initialized their diff --git a/libs/mysql/binlog/event/large_transaction_header_event.cpp b/libs/mysql/binlog/event/large_transaction_header_event.cpp new file mode 100644 index 000000000000..d5fce84063ac --- /dev/null +++ b/libs/mysql/binlog/event/large_transaction_header_event.cpp @@ -0,0 +1,59 @@ +#include "mysql/binlog/event/large_transaction_header_event.h" + +#include "mysql/binlog/event/control_events.h" // Format_description_event +#include "mysql/binlog/event/event_reader_macros.h" + +namespace mysql::binlog::event { + +Large_transaction_header_event:: + Large_transaction_header_event( + const char *buf, const Format_description_event *fde) + : Binary_log_event(&buf, fde) { + BAPI_ENTER( + "Large_transaction_header_event::" + "Large_transaction_header_event(const char*, ...)"); + READER_TRY_INITIALIZATION; + READER_ASSERT_POSITION(fde->common_header_len); + + READER_TRY_SET(m_version, read); + if (m_version == 0 || m_version > kVersion) { + READER_THROW("Invalid Large_transaction_header version"); + } + READER_TRY_SET(m_terminating_event_offset, read); + /* Read unconditionally: a truncated event missing the type byte is + reported as a read error rather than silently defaulting the type. */ + READER_TRY_SET(m_terminating_event_type, read); + + /* The remainder of the body is padding; its contents are ignored. */ + m_padding_size = READER_CALL(available_to_read); + + READER_CATCH_ERROR; + BAPI_VOID_RETURN; +} + +Large_transaction_header_event:: + Large_transaction_header_event(uint64_t terminating_event_offset, + uint8_t terminating_event_type, + uint64_t padding_size) + : Binary_log_event(LARGE_TRANSACTION_HEADER_EVENT), + m_terminating_event_offset(terminating_event_offset), + m_terminating_event_type(terminating_event_type), + m_padding_size(padding_size) {} + +#ifndef HAVE_MYSYS +void Large_transaction_header_event::print_event_info(std::ostream &info) { + info << "terminating event offset " << m_terminating_event_offset; +} + +void Large_transaction_header_event::print_long_info(std::ostream &info) { + info << "Timestamp: " << header()->when.tv_sec; + info << "\tVersion: " << static_cast(m_version); + info << "\tTerminating event offset: " << m_terminating_event_offset; + info << "\tTerminating event type: " + << static_cast(m_terminating_event_type); + info << "\tPadding: " << m_padding_size << " bytes"; + info << "\n"; +} +#endif + +} // namespace mysql::binlog::event diff --git a/libs/mysql/binlog/event/large_transaction_header_event.h b/libs/mysql/binlog/event/large_transaction_header_event.h new file mode 100644 index 000000000000..fb940599b0eb --- /dev/null +++ b/libs/mysql/binlog/event/large_transaction_header_event.h @@ -0,0 +1,133 @@ +/** + @file large_transaction_header_event.h + + @brief Deserialization of the Large_transaction_header + event. All serialization logic lives in the server class under + sql/log_event.* +*/ + +#ifndef MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H +#define MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H + +#include + +#include "mysql/binlog/event/binlog_event.h" + +namespace mysql::binlog::event { + +/** + @class Large_transaction_header_event + + Event needed for the large transaction optimization. It serves two + purposes: + + 1. It records the offset of the transaction's terminating event, so + that binary log recovery can seek directly past the transaction + body instead of scanning it. + 2. Its variable-length padding fills the file's reserved header region + exactly, so the transaction body starts at the offset that was + assumed while events were being spilled. + + Always written with the LOG_EVENT_IGNORABLE_F flag set: replicas and + binlog tools that do not recognize the type skip it. + + @section Large_transaction_header_event_binary_format + Binary Format + + The post-header is empty. The Body has the following components: + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Body for Large_transaction_header_event
NameFormatDescription
version1 byte unsigned integerEvent format version; currently 1. Retained for future + extensibility.
terminating_event_offset8 byte unsigned little-endian integerOffset, within this binary log file, of the transaction's + terminating event.
terminating_event_type1 byte unsigned integerBinary log event type of the terminating event.
paddingvariable-length byte sequenceFiller occupying the remainder of the reserved header region; + contents are undefined and ignored on read.
+*/ +class Large_transaction_header_event : public Binary_log_event { + public: + /** Current version of the event's body format. The version field is + retained for future extensibility; only this version exists today. */ + static constexpr uint8_t kVersion = 1; + /** Body bytes before padding: version (1) + offset (8) + event type (1). */ + static constexpr size_t kFixedBodyLength = 1 + 8 + 1; + + /** + Deserializing constructor. + + @param buf Contains the serialized event. + @param fde An FDE event (see Rotate_event constructor for more info). + */ + Large_transaction_header_event( + const char *buf, const Format_description_event *fde); + + /** + Creates an event with the given terminating-event metadata and + padding size (used by the server when writing a promoted binary log + file's header). + + @param terminating_event_offset Offset of the transaction's + terminating event in the file. + @param terminating_event_type Type code of the transaction's + terminating event in the file. + @param padding_size Number of filler bytes to occupy the + remainder of the reserved region. + */ + Large_transaction_header_event(uint64_t terminating_event_offset, + uint8_t terminating_event_type, + uint64_t padding_size); + + ~Large_transaction_header_event() override = default; + + uint8_t get_version() const { return m_version; } + uint64_t get_terminating_event_offset() const { + return m_terminating_event_offset; + } + uint8_t get_terminating_event_type() const { + return m_terminating_event_type; + } + uint64_t get_padding_size() const { return m_padding_size; } + +#ifndef HAVE_MYSYS + void print_event_info(std::ostream &info) override; + void print_long_info(std::ostream &info) override; +#endif + + protected: + /** Body format version read from (or to be written to) the wire. */ + uint8_t m_version{kVersion}; + /** Offset of the transaction's terminating event in this file. */ + uint64_t m_terminating_event_offset{0}; + /** Type code of the terminating event; absent in version-1 headers. */ + uint8_t m_terminating_event_type{0}; + /** Number of filler bytes following the fixed body fields. */ + uint64_t m_padding_size{0}; +}; + +} // namespace mysql::binlog::event + +#endif // MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H diff --git a/libs/mysql/binlog/event/trx_boundary_parser.cpp b/libs/mysql/binlog/event/trx_boundary_parser.cpp index 4d30b66eef4a..63d5ae476e8f 100644 --- a/libs/mysql/binlog/event/trx_boundary_parser.cpp +++ b/libs/mysql/binlog/event/trx_boundary_parser.cpp @@ -252,6 +252,7 @@ Transaction_boundary_parser::get_event_boundary_type( case mysql::binlog::event::SLAVE_EVENT: case mysql::binlog::event::DELETE_FILE_EVENT: case mysql::binlog::event::TRANSACTION_CONTEXT_EVENT: + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: boundary_type = EVENT_BOUNDARY_TYPE_IGNORE; break; diff --git a/mysql-test/common/binlog/validate_bolt_file.inc b/mysql-test/common/binlog/validate_bolt_file.inc new file mode 100644 index 000000000000..3764919017b6 --- /dev/null +++ b/mysql-test/common/binlog/validate_bolt_file.inc @@ -0,0 +1,96 @@ +# Requires $bolt_header_file to name a standalone BOLT-promoted binlog. +# Decode the promoted file and validate its LTH metadata and layout. +# These checks keep a plausible but corrupt LTH from silently directing +# optimized recovery to an incorrect terminal event. +# The awk program avoids `$` field syntax because mysqltest expands it. + +--let $bolt_promoted_dump = $MYSQLTEST_VARDIR/tmp/bolt_promoted_dump.txt +--let $bolt_validator_script = $MYSQLTEST_VARDIR/tmp/validate_bolt_header.awk +--exec $MYSQL_BINLOG --force-if-open --verify-binlog-checksum --verbose $MYSQLD_DATADIR/$bolt_header_file > $bolt_promoted_dump +--exec grep -m 1 -E 'last_committed=.*sequence_number=' $bolt_promoted_dump | grep -Eq 'last_committed=0[[:space:]]+sequence_number=1' +--write_file $bolt_validator_script EOF +# Extract the end position printed by mysqlbinlog for the current event. +# Keep the full line in a variable so this does not rely on awk `$` fields. +function end_pos(line, value) { + value = line + sub(/^.*end_log_pos /, "", value) + sub(/ .*/, "", value) + return value + 0 +} + +BEGIN { + # Read mysqlbinlog's verbose output and associate each event description + # with the preceding '# at ' line. + while ((getline line) > 0) { + if (line ~ /^# at [0-9][0-9]*/) { + split(line, words, " ") + event_start = words[3] + 0 + continue + } + + # The preceding Previous-GTIDs event must end where the LTH starts. + if (line ~ /Previous-GTIDs/ && event_start) { + previous_start = event_start + previous_end = end_pos(line) + continue + } + + # A promoted file contains exactly one ignorable LTH in its prefix. + if (line ~ /Large transaction header[[:space:]]+Ignorable/ && event_start) { + if (lth_start) exit 1 + lth_start = event_start + lth_end = end_pos(line) + continue + } + + # Parse the LTH's recovery hint: the terminal-event offset, its expected + # type code, and the remaining padding that fills the reserved region. + if (line ~ /Terminating event offset /) { + if (!lth_start || lth_offset) exit 1 + value = line + sub(/^.*Terminating event offset /, "", value) + split(value, fields, ", type ") + lth_offset = fields[1] + 0 + split(fields[2], fields, ", padding ") + lth_type = fields[1] + 0 + sub(/ bytes/, "", fields[2]) + lth_padding = fields[2] + 0 + continue + } + + # Select only the first GTID after the LTH. The promoted active file can + # later contain additional GTIDs, which are not part of its header. + if (line ~ /GTID.*last_committed=.*sequence_number=/ && event_start == lth_end) { + if (gtid_start) exit 1 + gtid_start = event_start + gtid_end = end_pos(line) + continue + } + + # The recorded terminal offset must identify a decoded event with an end + # position beyond its start. + if (line ~ /end_log_pos/ && event_start == lth_offset) + terminal_end = end_pos(line) + } + + # The event type is byte 4 of a binlog event header. Read the raw byte at + # the LTH offset so metadata cannot merely agree with a mis-decoded event. + command = "dd if=\"" binlog_file "\" bs=1 skip=" (lth_offset + 4) " count=1 2>/dev/null | od -An -tu1" + command | getline terminal_type + close(command) + gsub(/[[:space:]]/, "", terminal_type) + + # Check the promoted prefix is contiguous, the first GTID ends exactly at + # the 64 KiB reserved boundary, the LTH points beyond that boundary, and + # the stored type is a recovery-supported terminal event (Query, XID, or + # XA PREPARE) that matches the raw event-header byte. + if (!(previous_start >= 4 && previous_end == lth_start && + lth_end == gtid_start && gtid_end == 65536 && + lth_offset >= gtid_end && terminal_end > lth_offset && + lth_padding > 0 && (lth_type == 2 || lth_type == 16 || lth_type == 38) && + terminal_type == lth_type)) exit 1 +} +EOF +--exec awk -v binlog_file=$MYSQLD_DATADIR/$bolt_header_file -f $bolt_validator_script $bolt_promoted_dump +--remove_file $bolt_promoted_dump +--remove_file $bolt_validator_script diff --git a/mysql-test/suite/binlog/inc/validate_bolt_header.inc b/mysql-test/suite/binlog/inc/validate_bolt_header.inc new file mode 100644 index 000000000000..79f2e2ad5970 --- /dev/null +++ b/mysql-test/suite/binlog/inc/validate_bolt_header.inc @@ -0,0 +1,3 @@ +# Compatibility wrapper for existing BOLT tests. +# The merged promoted-file validator is shared from mysql-test/common. +--source common/binlog/validate_bolt_file.inc diff --git a/mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result b/mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result new file mode 100644 index 000000000000..764ca49691ab --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result @@ -0,0 +1,25 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +# Pre-TC-prepare crash rolls back. +include/assert.inc [A pre-TC-prepare crash rolls back all rows.] +# Post-TC-prepare crash rolls back. +include/assert.inc [A post-TC-prepare crash rolls back all rows.] +# Post-header-sync crash rolls back. +include/assert.inc [A post-header-sync crash rolls back all rows.] +# Post-purge-index-sync crash rolls back. +include/assert.inc [A post-purge-index-sync crash rolls back all rows.] +# Post-promote-rename crash rolls back. +include/assert.inc [A post-promote-rename crash rolls back all rows.] +# Post-main-index-update crash commits exactly once. +include/assert.inc [A post-main-index-update crash commits all rows exactly once.] +include/assert.inc [A post-main-index-update crash preserves the original rows.] +# Post-purge-index-removal crash commits exactly once. +include/assert.inc [A post-purge-index-removal crash commits all rows exactly once.] +include/assert.inc [A post-purge-index-removal crash preserves the original rows.] +# Pre-engine-commit crash commits exactly once. +include/assert.inc [A pre-engine-commit crash commits all rows exactly once.] +include/assert.inc [A pre-engine-commit crash preserves the original rows.] +# Post-main-index XA PREPARE crash remains prepared and commits exactly once. +include/assert.inc [A post-main-index XA PREPARE remains prepared until XA COMMIT.] +include/assert.inc [XA COMMIT commits the recovered promoted rows exactly once.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_commit_rollback.result b/mysql-test/suite/binlog/r/binlog_bolt_commit_rollback.result new file mode 100644 index 000000000000..f264c8d7aef1 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_commit_rollback.result @@ -0,0 +1,224 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +Warnings: +Note 1051 Unknown table 'test.t2' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (x INT PRIMARY KEY AUTO_INCREMENT, y LONGTEXT) ENGINE=InnoDB; +# Rollback removes the spill file and leaves the active binlog unchanged. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +ROLLBACK; +include/assert.inc [Rollback leaves the BOLT promotion count unchanged.] +include/assert.inc [Rollback keeps the active binary log unchanged.] +include/assert.inc [Rollback removes all transaction rows.] +# Retry the same transaction and promote a new active binary log. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +include/assert.inc [Retry promotes one new active binary log.] +include/assert.inc [Retry changes the active binary log file.] +include/assert.inc [Retry commits the four rows exactly once.] +# XA PREPARE promotes the large transaction; XA COMMIT makes it visible. +XA START 'bolt_prepare_commit'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_commit'; +XA PREPARE 'bolt_prepare_commit'; +include/assert.inc [XA PREPARE promotes one large transaction.] +include/assert.inc [XA PREPARE keeps its rows invisible until resolution.] +XA COMMIT 'bolt_prepare_commit'; +include/assert.inc [XA COMMIT does not promote the prepared transaction again.] +include/assert.inc [XA COMMIT commits all prepared rows exactly once.] +# XA ROLLBACK resolves a promoted prepared transaction without committing rows. +XA START 'bolt_prepare_rollback'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_rollback'; +XA PREPARE 'bolt_prepare_rollback'; +include/assert.inc [XA PREPARE before rollback promotes one large transaction.] +XA ROLLBACK 'bolt_prepare_rollback'; +include/assert.inc [XA ROLLBACK does not promote the prepared transaction again.] +include/assert.inc [XA ROLLBACK removes all prepared rows.] +# XA COMMIT ONE PHASE promotes and commits a large transaction once. +XA START 'bolt_one_phase'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_one_phase'; +XA COMMIT 'bolt_one_phase' ONE PHASE; +include/assert.inc [XA COMMIT ONE PHASE promotes one large transaction.] +include/assert.inc [XA COMMIT ONE PHASE commits all rows exactly once.] +Warnings: +Note 1051 Unknown table 'test.t2' +# DDL implicitly commits and promotes the preceding large row transaction. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; +include/assert.inc [The implicit pre-DDL commit promotes the large row transaction.] +include/assert.inc [DDL commits all preceding large transaction rows.] +include/assert.inc [DDL completes after the promoted transaction.] +# KILL QUERY after spill leaves the same session able to roll back and retry. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (CONCAT(REPEAT('abcdefghijklmnop', 250000), IF(GET_LOCK('bolt_kill_query_lock', 100), '', ''))); +KILL QUERY ID; +ERROR 70100: Query execution was interrupted +ROLLBACK; +include/assert.inc [KILL QUERY followed by rollback does not promote a binary log.] +include/assert.inc [KILL QUERY followed by rollback removes all transaction rows.] +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +include/assert.inc [The same killed-query session promotes on its retry.] +include/assert.inc [The same killed-query session commits all retry rows once.] +# Disconnecting a session with a spill removes the temporary file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +include/assert.inc [Disconnecting an active spill does not promote a binary log.] +include/assert.inc [Disconnecting an active spill rolls back all rows.] +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +include/assert.inc [A new session promotes the disconnected transaction's retry.] +include/assert.inc [A new session commits all retry rows once.] +# A duplicate-key error preserves the spill until explicit rollback. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (1, REPEAT('abcdefghijklmnop', 250000)); +ERROR 23000: Duplicate entry '1' for key 't1.PRIMARY' +ROLLBACK; +include/assert.inc [A statement error followed by rollback does not promote a binary log.] +include/assert.inc [A statement error followed by rollback keeps the active binary log unchanged.] +include/assert.inc [A statement error followed by rollback removes all rows.] +SET GLOBAL max_binlog_size = 4096; +# A promoted oversized file becomes binary-log history and opens a fresh active file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +SET GLOBAL max_binlog_size = 1073741824; +include/assert.inc [The oversized transaction is promoted once before rotation.] +include/assert.inc [Immediate max-binlog-size rotation opens a fresh active binary log.] +include/assert.inc [Immediate max-binlog-size rotation keeps all promoted rows.] +SHOW BINARY LOGS; +Log_name File_size Encrypted +binlog.000001 203 No +binlog.000002 16066082 No +binlog.000003 199 No +BEGIN; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +# Five waiting spills do not prevent a small transaction using the active binlog. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +include/assert.inc [The small transaction stays in the existing active binary log.] +include/assert.inc [Five waiting large transactions leave five temporary files.] +COMMIT; +COMMIT; +COMMIT; +ROLLBACK; +ROLLBACK; +include/assert.inc [Three committed spills produce exactly three BOLT promotions.] +include/assert.inc [Three committed spills create three new binary log files.] +include/assert.inc [Only the three committed large transactions and small transaction persist.] +include/assert.inc [The committed transaction ranges are preserved exactly.] +XA START 'bolt_concurrent_one_phase_1'; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_1'; +XA START 'bolt_concurrent_one_phase_2'; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_2'; +XA START 'bolt_concurrent_one_phase_3'; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_3'; +XA START 'bolt_concurrent_prepare_1'; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_1'; +XA START 'bolt_concurrent_prepare_2'; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_2'; +# Five waiting XA branches leave the active binlog available for a small transaction. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +include/assert.inc [The small transaction stays in the existing active binary log with XA spills pending.] +XA ROLLBACK 'bolt_concurrent_one_phase_1'; +XA ROLLBACK 'bolt_concurrent_one_phase_2'; +XA COMMIT 'bolt_concurrent_one_phase_3' ONE PHASE; +XA PREPARE 'bolt_concurrent_prepare_1'; +XA ROLLBACK 'bolt_concurrent_prepare_1'; +XA PREPARE 'bolt_concurrent_prepare_2'; +XA COMMIT 'bolt_concurrent_prepare_2'; +include/assert.inc [One XA one-phase commit and two XA PREPARE paths produce three promotions.] +include/assert.inc [The XA commit decisions create exactly three new binary log files.] +include/assert.inc [Only the committed XA branches and small transaction persist.] +include/assert.inc [The committed XA transaction ranges are preserved exactly.] +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_fallback.result b/mysql-test/suite/binlog/r/binlog_bolt_fallback.result new file mode 100644 index 000000000000..d8949fb1c906 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_fallback.result @@ -0,0 +1,97 @@ +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY, marker INT) ENGINE=MyISAM; +# A large STATEMENT transaction uses the standard commit path. +SET SESSION binlog_format = STATEMENT; +SET @bolt_statement_payload = REPEAT('s', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_statement_payload); +COMMIT; +SET @bolt_statement_payload = NULL; +include/assert.inc [A large STATEMENT transaction does not promote a binary log.] +include/assert.inc [A large STATEMENT transaction increments the missed counter once.] +include/assert.inc [A large STATEMENT transaction stays in the active binary log.] +include/assert.inc [A large STATEMENT fallback commits its row.] +# A large MIXED transaction with a statement event uses the standard path. +SET SESSION binlog_format = MIXED; +SET @bolt_mixed_payload = REPEAT('m', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_mixed_payload); +COMMIT; +SET @bolt_mixed_payload = NULL; +include/assert.inc [A large MIXED transaction does not promote a binary log.] +include/assert.inc [A large MIXED transaction increments the missed counter once.] +include/assert.inc [A large MIXED transaction stays in the active binary log.] +include/assert.inc [A large MIXED fallback commits its row.] +# A checksum change during a large transaction uses the standard path. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('c', 5000000)); +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +INSERT INTO t1 VALUES (2, REPEAT('d', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('e', 5000000)); +COMMIT; +include/assert.inc [A checksum change does not promote a binary log.] +include/assert.inc [A checksum change increments the missed counter once.] +include/assert.inc [A checksum-change fallback commits all rows.] +# Restoring the original checksum before commit preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('r', 5000000)); +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +SET GLOBAL binlog_checksum = CRC32; +INSERT INTO t1 VALUES (2, REPEAT('s', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('t', 5000000)); +COMMIT; +include/assert.inc [Restoring the checksum before commit promotes the transaction.] +include/assert.inc [Restoring the checksum before commit does not increment the missed counter.] +include/assert.inc [The restored-checksum commit opens its promoted binary log.] +include/assert.inc [The restored-checksum promoted transaction commits all rows.] +include/assert.inc [The executed GTID set exceeds BOLT's reserved header.] +# A dynamically reserved Previous_gtids header preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 5000000)); +COMMIT; +include/assert.inc [A dynamically reserved Previous_gtids header promotes a binary log.] +include/assert.inc [A dynamically reserved Previous_gtids header does not increment the missed counter.] +include/assert.inc [The dynamically reserved-header commit opens its promoted binary log.] +include/assert.inc [A dynamically reserved-header commit commits all rows.] +INSERT INTO t1 VALUES (1, REPEAT('n', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('o', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('p', 5000000)); +INSERT INTO t2 VALUES (1, 0), (2, 0), (3, 0); +# A mixed InnoDB/MyISAM row statement falls back for its statement cache. +BEGIN; +UPDATE t1 JOIN t2 USING (id) SET t1.data = CONCAT(t1.data, 'x'), t2.marker = 1; +COMMIT; +include/assert.inc [An InnoDB/MyISAM statement promotes its qualifying InnoDB cache.] +include/assert.inc [An InnoDB/MyISAM statement increments the missed counter once.] +include/assert.inc [An InnoDB/MyISAM statement rotates to the promoted InnoDB binary log.] +include/assert.inc [An InnoDB/MyISAM fallback commits every InnoDB row.] +include/assert.inc [An InnoDB/MyISAM fallback updates every MyISAM row.] +# ---------------------------------------------------------------------- +# Setup +# Creating local configuration file for keyring component: component_keyring_file +# Creating manifest file for current MySQL server instance +# Re-starting mysql server with manifest file +# ---------------------------------------------------------------------- +# Restart the server with binary-log encryption enabled. +# An encrypted binary log uses the standard commit path. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('j', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('k', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('l', 5000000)); +COMMIT; +include/assert.inc [An encrypted binary log does not promote a binary log.] +include/assert.inc [An encrypted binary log increments the missed counter once.] +include/assert.inc [An encrypted fallback stays in the active binary log.] +include/assert.inc [An encrypted fallback commits all rows.] +DROP TABLE t2; +DROP TABLE t1; +# ---------------------------------------------------------------------- +# Teardown +# Removing manifest file for current MySQL server instance +# Removing local keyring file for keyring component: component_keyring_file +# Removing local configuration file for keyring component: component_keyring_file +# Restarting server without the manifest file +# ---------------------------------------------------------------------- diff --git a/mysql-test/suite/binlog/r/binlog_bolt_optimized_recovery.result b/mysql-test/suite/binlog/r/binlog_bolt_optimized_recovery.result new file mode 100644 index 000000000000..b54dab06529a --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_optimized_recovery.result @@ -0,0 +1,30 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Commit a qualifying transaction for optimized recovery. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +include/assert.inc [The transaction was promoted into a new binary log file.] +# Kill and restart: --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --max-binlog-size=1G +# Recovery logs the optimized large-transaction seek. +Pattern "Optimized binlog recovery by avoiding a sequential read of a large transaction body and seeking to its terminating event at offset" found +include/assert.inc [The committed promoted transaction survives recovery.] +# Commit a second qualifying transaction for corrupt-LTH recovery. +BEGIN; +INSERT INTO t1 VALUES (4, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (5, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (6, REPEAT('f', 4000000)); +COMMIT; +include/assert.inc [The second transaction was promoted into a new binary log file.] +# Truncate the active promoted file and reject the invalid LTH. +# Kill the server +Pattern "contains an invalid large transaction header" found +# restart +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_rotate_purge.result b/mysql-test/suite/binlog/r/binlog_bolt_rotate_purge.result new file mode 100644 index 000000000000..b9106494ae48 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_rotate_purge.result @@ -0,0 +1,26 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +FLUSH BINARY LOGS; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +# Purge older history while the qualifying spill file is still open. +PURGE BINARY LOGS TO 'binlog.000002'; +# Rotate while the qualifying spill file is still open. +FLUSH BINARY LOGS; +include/assert.inc [FLUSH opens a new active binary log while a BOLT spill exists.] +COMMIT; +include/assert.inc [The spill promotes after PURGE and FLUSH complete.] +include/assert.inc [The promoted transaction preserves every row.] +# Rotate the promoted active file, then purge it as ordinary history. +FLUSH BINARY LOGS; +include/assert.inc [FLUSH rotates the promoted active binary log.] +PURGE BINARY LOGS TO 'binlog.000005'; +SHOW BINARY LOGS; +Log_name File_size Encrypted +binlog.000005 199 No +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_savepoint.result b/mysql-test/suite/binlog/r/binlog_bolt_savepoint.result new file mode 100644 index 000000000000..ff32a46bf961 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_savepoint.result @@ -0,0 +1,40 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Savepoint rollback below binlog_cache_size keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 512000)); +SAVEPOINT before_spill; +INSERT INTO t1 VALUES (2, REPEAT('b', 2000000)); +ROLLBACK TO SAVEPOINT before_spill; +COMMIT; +include/assert.inc [A final cache below binlog_cache_size is copied to the active binlog.] +include/assert.inc [Savepoint rollback retains only the pre-spill row.] +include/assert.inc [Savepoint rollback discards the post-savepoint row.] +# Savepoint rollback below the BOLT threshold keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (11, REPEAT('c', 2000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (12, REPEAT('d', 3000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +include/assert.inc [A final cache below the BOLT threshold is copied to the active binlog.] +include/assert.inc [Savepoint rollback retains the above-cache pre-savepoint row.] +include/assert.inc [Savepoint rollback discards the above-cache post-savepoint row.] +# Savepoint rollback above the BOLT threshold promotes a new binlog. +BEGIN; +INSERT INTO t1 VALUES (21, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (22, REPEAT('f', 4000000)); +INSERT INTO t1 VALUES (23, REPEAT('g', 4000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (24, REPEAT('h', 2000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +include/assert.inc [A final cache above the BOLT threshold promotes a new active binlog.] +include/assert.inc [Savepoint rollback retains the promoted pre-savepoint rows.] +include/assert.inc [Savepoint rollback discards the promoted post-savepoint row.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_sysvars.result b/mysql-test/suite/binlog/r/binlog_bolt_sysvars.result new file mode 100644 index 000000000000..fb58b62c06c9 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_sysvars.result @@ -0,0 +1,65 @@ +# restart: +# Case 1: BOLT is ON; binlog_cache_size > binlog_large_transaction_optimization_threshold. +include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +Warnings: +Warning 6910 Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size. +include/assert_error_log.inc [server: 1, pattern: Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.]] +include/assert.inc [Increasing binlog_cache_size raises the optimization threshold.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +20971520 +include/assert_error_log.inc [server: 1, pattern: Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.]] +# Case 2: BOLT is ON; binlog_large_transaction_optimization_threshold < binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +Warnings: +Warning 6910 Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size. +include/assert_error_log.inc [server: 1, pattern: Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.]] +include/assert.inc [A lower optimization threshold is raised to match binlog_cache_size.] +# Case 3: BOLT is ON; binlog_large_transaction_optimization_threshold > binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [A threshold above binlog_cache_size is unchanged.] +include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 10485760; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [Lowering binlog_cache_size does not lower the threshold.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=ON --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +22020096 +include/assert_error_log.inc [server: 1, pattern: NONE] +# restart: +# Case 4: BOLT is OFF; binlog_cache_size > binlog_large_transaction_optimization_threshold. +include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [Increasing binlog_cache_size raises the threshold while BOLT is off.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +20971520 +include/assert_error_log.inc [server: 1, pattern: NONE] +# Case 5: BOLT is OFF; binlog_large_transaction_optimization_threshold < binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [A lower threshold is raised while BOLT is off.] +# Case 6: BOLT is OFF; binlog_large_transaction_optimization_threshold > binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [A threshold above binlog_cache_size is unchanged while BOLT is off.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +22020096 +include/assert_error_log.inc [server: 1, pattern: NONE] +# restart: diff --git a/mysql-test/suite/binlog/r/binlog_bolt_tmp_dir_cleanup.result b/mysql-test/suite/binlog/r/binlog_bolt_tmp_dir_cleanup.result new file mode 100644 index 000000000000..1f1d56b585f3 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_tmp_dir_cleanup.result @@ -0,0 +1,11 @@ +Created 3 managed BOLT spill files +# restart +Startup cleanup removed all managed BOLT spill files +# Reject a symbolic link for #binlog_temp_files. +# restart +# Reject an unrecognized regular entry. +# restart +# Reject a symbolic-link entry, even with a managed file name. +# restart +# Reject a nested directory, even with a managed file name. +# restart diff --git a/mysql-test/suite/binlog/r/binlog_encryption_random_access.result b/mysql-test/suite/binlog/r/binlog_encryption_random_access.result index 6fbe198b995d..b10cb177653d 100644 --- a/mysql-test/suite/binlog/r/binlog_encryption_random_access.result +++ b/mysql-test/suite/binlog/r/binlog_encryption_random_access.result @@ -9,7 +9,19 @@ CALL mtr.add_suppression('Unsafe statement written to the binary log using state CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 TEXT, pos INT); # Inserting 100 random transaction # Asserting we can show binlog events from each transaction -DROP TABLE t1; +# Binlog encryption forces a qualifying row transaction to use the standard path. +Warnings: +Warning 1287 '@@binlog_format' is deprecated and will be removed in a future release. +CREATE TABLE t_large (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +BEGIN; +INSERT INTO t_large VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t_large VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t_large VALUES (3, REPEAT('c', 5000000)); +COMMIT; +include/assert.inc [An encrypted binary log uses the standard path.] +include/assert.inc [An encrypted binary log increments the missed counter once.] +include/assert.inc [The encrypted fallback transaction commits all rows.] +DROP TABLE t_large, t1; # ---------------------------------------------------------------------- # Teardown # Removing manifest file for current MySQL server instance diff --git a/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery-master.opt new file mode 100644 index 000000000000..f10af27e5afb --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery-master.opt @@ -0,0 +1,3 @@ +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G diff --git a/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery.test b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery.test new file mode 100644 index 000000000000..e824d1a6ef5f --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery.test @@ -0,0 +1,319 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT crash recovery follows the binary-log 2PC decision. +# +# === Requirements === +# R1. A crash before the main-index update rolls the transaction back. +# R2. A crash after the main-index update recovers the transaction once. +# R3. A post-main-index XA PREPARE remains prepared until XA COMMIT. +# +# === Implementation === +# 1. Initialize BOLT and a clean test table. +# 2. Crash at each durable decision boundary around promotion. +# 3. Restart and verify rollback or rollforward according to the boundary. +# 4. Verify the same post-main-index decision for XA PREPARE. +# +--source include/not_crashrep.inc +--source include/not_valgrind.inc +--source include/have_debug.inc +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) +--let $save_sync_binlog = query_get_value(SELECT @@GLOBAL.sync_binlog, @@GLOBAL.sync_binlog, 1) + +# Setup +--let $BOLT_RESTART = restart: +--let $BOLT_RESTART = $BOLT_RESTART --binlog-large-transaction-optimization-enabled=ON +--let $BOLT_RESTART = $BOLT_RESTART --binlog-large-transaction-optimization-threshold=10M +--let $BOLT_RESTART = $BOLT_RESTART --max-binlog-size=1G +--let $BOLT_RESTART = $BOLT_RESTART --sync-binlog=1 + +--disable_query_log +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +RESET BINARY LOGS AND GTIDS; +--enable_query_log + +############################################################################ +# Case 1: Crash before TC prepare. +# Expected behavior: rollback. +############################################################################ +--echo # Pre-TC-prepare crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +SET SESSION DEBUG="+d,crash_bolt_before_tc_prepare"; +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A pre-TC-prepare crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 2: Crash after TC prepare before TC commit. +# Expected behavior: rollback. +############################################################################ +--echo # Post-TC-prepare crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +SET SESSION DEBUG="+d,crash_bolt_after_tc_prepare"; +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-TC-prepare crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 3: Crash during binlog commit after durable header in promoted file. +# Expected behavior: rollback. +############################################################################ +--echo # Post-header-sync crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_header_sync"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-header-sync crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 4: Crash during binlog commit after writing to purge_index_file. +# Expected behavior: rollback. +############################################################################ +--echo # Post-purge-index-sync crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_purge_index_sync"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-purge-index-sync crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 5: Crash during binlog commit after promoting the file. +# Expected behavior: rollback. +############################################################################ +--echo # Post-promote-rename crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_promote_rename"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-promote-rename crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 6: Crash during binlog commit after writing to main index. +# Expected behavior: rollforward. +############################################################################ +--echo # Post-main-index-update crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_main_index_update"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-main-index-update crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A post-main-index-update crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 7: Crash during binlog commit after removing entry from purge_index_file. +# Expected behavior: rollforward. +############################################################################ +--echo # Post-purge-index-removal crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_purge_index_remove"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-purge-index-removal crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A post-purge-index-removal crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 8: Crash after binlog commit before innodb commit. +# Expected behavior: commit. +############################################################################ +--echo # Pre-engine-commit crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_before_engine_commit"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A pre-engine-commit crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A pre-engine-commit crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 9: XA PREPARE crash after main-index update. +# Expected behavior: remain prepared, then commit. +############################################################################ +--echo # Post-main-index XA PREPARE crash remains prepared and commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_main_index_update"; +XA START 'bolt_post_index'; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +XA END 'bolt_post_index'; +--error 2013 +XA PREPARE 'bolt_post_index'; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +# Case 9 intentionally leaves one XA transaction prepared for recovery. +--exec echo "$BOLT_RESTART --log-error-suppression-list=MY-010225" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--disable_query_log +--disable_result_log +XA RECOVER; +XA COMMIT 'bolt_post_index'; +--enable_result_log +--enable_query_log +--let $assert_text = A post-main-index XA PREPARE remains prepared until XA COMMIT. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = XA COMMIT commits the recovered promoted rows exactly once. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc + +DROP TABLE t1; +--disable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--eval SET GLOBAL sync_binlog = $save_sync_binlog +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback-master.opt new file mode 100644 index 000000000000..0123ec2d69f6 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback-master.opt @@ -0,0 +1,4 @@ +--binlog-cache-size=1M +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G diff --git a/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback.test b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback.test new file mode 100644 index 000000000000..b6e68e57bc5d --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback.test @@ -0,0 +1,595 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT safely discards rolled-back spill files and handles normal, XA, +# and DDL transaction endings without leaving stale promotion state. +# +# === Requirements === +# R1. A rolled-back spilled transaction leaves no temporary file or binlog. +# R2. Retrying the same four-insert transaction commits as one promoted binlog. +# R3. XA PREPARE resolves correctly through both XA COMMIT and XA ROLLBACK. +# R4. XA COMMIT ONE PHASE and the implicit commit before DDL are promoted. +# R5. A killed query after spill can roll back and retry on the same session. +# R6. Disconnecting with an active spill discards it; a new session can retry. +# R7. A statement error after spill leaves the transaction rollback-safe. +# R8. A promoted file larger than max_binlog_size immediately rotates. +# R9. Five simultaneously spilled transactions leave the active binlog usable +# for a small transaction and promote only the three that commit. +# R10. The same five-spill boundary handles XA one-phase and prepared branches +# with the correct commit and rollback decisions. +# R11. Every promoted file has a checksummed Previous_gtids, LTH, GTID prefix +# with the GTID dependency clock reset for its new binary log. +# +# === Implementation === +# The source uses a 1 MiB cache and a 10 MiB BOLT threshold. Four 4 MB rows +# are used because four REPEAT('abcdefghijklmnop', 1000) values are only 64 KB, +# below BOLT's required promotion threshold. The killed-query case uses a user +# lock and PROCESSLIST polling rather than timing assumptions. The concurrent +# cases use five independent sessions with non-overlapping primary-key ranges. +# Each promoted file is decoded with checksum verification before its header +# ordering and logical timestamps are asserted. Transaction compression is +# disabled because BOLT and size-triggered rotation require an uncompressed +# promoted file. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (x INT PRIMARY KEY AUTO_INCREMENT, y LONGTEXT) ENGINE=InnoDB; + +############################################################################ +# Case 1: Roll back a spilled transaction, then retry and commit. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Rollback removes the spill file and leaves the active binlog unchanged. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +ROLLBACK; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = Rollback leaves the BOLT promotion count unchanged. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = Rollback keeps the active binary log unchanged. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Rollback removes all transaction rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +--echo # Retry the same transaction and promote a new active binary log. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +COMMIT; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_header_file = $binlog_file_after +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = Retry promotes one new active binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = Retry changes the active binary log file. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Retry commits the four rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc + +############################################################################ +# Case 2: XA PREPARE followed by XA COMMIT. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # XA PREPARE promotes the large transaction; XA COMMIT makes it visible. +XA START 'bolt_prepare_commit'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_commit'; +XA PREPARE 'bolt_prepare_commit'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after_prepare = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA PREPARE promotes one large transaction. +--let $assert_cond = $bolt_count_after_prepare = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = XA PREPARE keeps its rows invisible until resolution. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +XA COMMIT 'bolt_prepare_commit'; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA COMMIT does not promote the prepared transaction again. +--let $assert_cond = $bolt_count_after = $bolt_count_after_prepare +--source include/assert.inc +--let $assert_text = XA COMMIT commits all prepared rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc + +############################################################################ +# Case 3: XA PREPARE followed by XA ROLLBACK. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # XA ROLLBACK resolves a promoted prepared transaction without committing rows. +XA START 'bolt_prepare_rollback'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_rollback'; +XA PREPARE 'bolt_prepare_rollback'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after_prepare = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA PREPARE before rollback promotes one large transaction. +--let $assert_cond = $bolt_count_after_prepare = $bolt_count_before + 1 +--source include/assert.inc +XA ROLLBACK 'bolt_prepare_rollback'; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA ROLLBACK does not promote the prepared transaction again. +--let $assert_cond = $bolt_count_after = $bolt_count_after_prepare +--source include/assert.inc +--let $assert_text = XA ROLLBACK removes all prepared rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +############################################################################ +# Case 4: XA COMMIT ONE PHASE. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # XA COMMIT ONE PHASE promotes and commits a large transaction once. +XA START 'bolt_one_phase'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_one_phase'; +XA COMMIT 'bolt_one_phase' ONE PHASE; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA COMMIT ONE PHASE promotes one large transaction. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = XA COMMIT ONE PHASE commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc + +############################################################################ +# Case 5: Large DML transaction followed by DDL. +############################################################################ +--disable_query_log +TRUNCATE t1; +DROP TABLE IF EXISTS t2; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # DDL implicitly commits and promotes the preceding large row transaction. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The implicit pre-DDL commit promotes the large row transaction. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = DDL commits all preceding large transaction rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +--let $assert_text = DDL completes after the promoted transaction. +--let $ddl_table_exists = query_get_value(SELECT COUNT(*) AS count FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2', count, 1) +--let $assert_cond = $ddl_table_exists = 1 +--source include/assert.inc + +############################################################################ +# Case 6: KILL QUERY after spill, rollback, and same-session retry. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (kill_blocker, localhost, root,,); +connect (kill_victim, localhost, root,,); +connection kill_blocker; +--disable_query_log +--disable_result_log +SELECT GET_LOCK('bolt_kill_query_lock', 0); +--enable_result_log +--enable_query_log +connection kill_victim; +--let $victim_id = query_get_value(SELECT CONNECTION_ID() AS id, id, 1) +--echo # KILL QUERY after spill leaves the same session able to roll back and retry. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +send INSERT INTO t1 (y) VALUES (CONCAT(REPEAT('abcdefghijklmnop', 250000), IF(GET_LOCK('bolt_kill_query_lock', 100), '', ''))); +connection kill_blocker; +--let $wait_condition = SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE ID = $victim_id AND STATE = 'User lock'; +--source include/wait_condition_or_abort.inc +--replace_result $victim_id ID +--eval KILL QUERY $victim_id +connection kill_victim; +--error ER_QUERY_INTERRUPTED +reap; +ROLLBACK; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = KILL QUERY followed by rollback does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = KILL QUERY followed by rollback removes all transaction rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The same killed-query session promotes on its retry. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = The same killed-query session commits all retry rows once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +connection kill_blocker; +--disable_query_log +--disable_result_log +SELECT RELEASE_LOCK('bolt_kill_query_lock'); +--enable_result_log +--enable_query_log +connection kill_victim; +disconnect kill_victim; +connection kill_blocker; +disconnect kill_blocker; +connection default; + +############################################################################ +# Case 7: Disconnect with an active spill, then retry from a new session. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--source include/count_sessions.inc +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (spill_client, localhost, root,,); +connection spill_client; +--echo # Disconnecting a session with a spill removes the temporary file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +disconnect spill_client; +connection default; +--source include/wait_until_count_sessions.inc +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = Disconnecting an active spill does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = Disconnecting an active spill rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +connect (spill_retry, localhost, root,,); +connection spill_retry; +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection default; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = A new session promotes the disconnected transaction's retry. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A new session commits all retry rows once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +connection spill_retry; +disconnect spill_retry; +connection default; +--source include/wait_until_count_sessions.inc + +############################################################################ +# Case 8: Statement error after spill, followed by rollback. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A duplicate-key error preserves the spill until explicit rollback. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +--error ER_DUP_ENTRY +INSERT INTO t1 (x, y) VALUES (1, REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +ROLLBACK; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A statement error followed by rollback does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A statement error followed by rollback keeps the active binary log unchanged. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A statement error followed by rollback removes all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +############################################################################ +# Case 9: A promoted file larger than max_binlog_size rotates immediately. +############################################################################ +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +SET GLOBAL max_binlog_size = 4096; +--echo # A promoted oversized file becomes binary-log history and opens a fresh active file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +SET GLOBAL max_binlog_size = 1073741824; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--exec test -e $MYSQLD_DATADIR/binlog.000002 +--exec test -e $MYSQLD_DATADIR/binlog.000003 +--let $bolt_header_file = query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source ../inc/validate_bolt_header.inc +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The oversized transaction is promoted once before rotation. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = Immediate max-binlog-size rotation opens a fresh active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Immediate max-binlog-size rotation keeps all promoted rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +SHOW BINARY LOGS; + +############################################################################ +# Case 10: Five spilled transactions with regular commit and rollback. +############################################################################ +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--source include/count_sessions.inc +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (promote_one, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_two, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_three, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_four, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_five, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +connection default; +--let $binlog_file_before_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Five waiting spills do not prevent a small transaction using the active binlog. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +--let $binlog_file_after_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--exec ls $BOLT_TEMP_DIR/bolt_* | wc -l | grep -E '^[[:space:]]*5[[:space:]]*$' > /dev/null +--let $assert_text = The small transaction stays in the existing active binary log. +--let $assert_cond = "$binlog_file_after_small" = "$binlog_file_before_small" +--source include/assert.inc +--let $assert_text = Five waiting large transactions leave five temporary files. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 1 +--source include/assert.inc +connection promote_one; +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_three; +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_five; +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_two; +ROLLBACK; +connection promote_four; +ROLLBACK; +connection default; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Three committed spills produce exactly three BOLT promotions. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 3 +--source include/assert.inc +--let $assert_text = Three committed spills create three new binary log files. +--let $assert_cond = "$binlog_file_after" = "binlog.000004" +--source include/assert.inc +--let $assert_text = Only the three committed large transactions and small transaction persist. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 13 +--source include/assert.inc +--let $assert_text = The committed transaction ranges are preserved exactly. +--let $assert_cond = [SELECT SUM(x) AS id_sum FROM t1, id_sum, 1] = 3631 +--source include/assert.inc + +############################################################################ +# Case 11: Five spilled XA branches with one-phase and prepared resolution. +############################################################################ +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connection promote_one; +XA START 'bolt_concurrent_one_phase_1'; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_1'; +connection promote_two; +XA START 'bolt_concurrent_one_phase_2'; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_2'; +connection promote_three; +XA START 'bolt_concurrent_one_phase_3'; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_3'; +connection promote_four; +XA START 'bolt_concurrent_prepare_1'; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_1'; +connection promote_five; +XA START 'bolt_concurrent_prepare_2'; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_2'; +connection default; +--let $binlog_file_before_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Five waiting XA branches leave the active binlog available for a small transaction. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +--let $binlog_file_after_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--exec ls $BOLT_TEMP_DIR/bolt_* | wc -l | grep -E '^[[:space:]]*5[[:space:]]*$' > /dev/null +--let $assert_text = The small transaction stays in the existing active binary log with XA spills pending. +--let $assert_cond = "$binlog_file_after_small" = "$binlog_file_before_small" +--source include/assert.inc +connection promote_one; +XA ROLLBACK 'bolt_concurrent_one_phase_1'; +connection promote_two; +XA ROLLBACK 'bolt_concurrent_one_phase_2'; +connection promote_three; +XA COMMIT 'bolt_concurrent_one_phase_3' ONE PHASE; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_four; +XA PREPARE 'bolt_concurrent_prepare_1'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +XA ROLLBACK 'bolt_concurrent_prepare_1'; +connection promote_five; +XA PREPARE 'bolt_concurrent_prepare_2'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +XA COMMIT 'bolt_concurrent_prepare_2'; +connection default; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = One XA one-phase commit and two XA PREPARE paths produce three promotions. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 3 +--source include/assert.inc +--let $assert_text = The XA commit decisions create exactly three new binary log files. +--let $assert_cond = "$binlog_file_after" = "binlog.000004" +--source include/assert.inc +--let $assert_text = Only the committed XA branches and small transaction persist. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 9 +--source include/assert.inc +--let $assert_text = The committed XA transaction ranges are preserved exactly. +--let $assert_cond = [SELECT SUM(x) AS id_sum FROM t1, id_sum, 1] = 3221 +--source include/assert.inc +connection promote_one; +disconnect promote_one; +connection promote_two; +disconnect promote_two; +connection promote_three; +disconnect promote_three; +connection promote_four; +disconnect promote_four; +connection promote_five; +disconnect promote_five; +connection default; +--source include/wait_until_count_sessions.inc + +# Cleanup +DROP TABLE t2; +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_fallback.test b/mysql-test/suite/binlog/t/binlog_bolt_fallback.test new file mode 100644 index 000000000000..447b4bcf93e8 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_fallback.test @@ -0,0 +1,371 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT uses the ordinary binary-log commit path when a spilled, +# above-threshold transaction cannot safely be promoted, while retaining +# promotion eligibility with a dynamically sized reserved header. +# +# === Requirements === +# R1. Large STATEMENT and MIXED transactions do not promote. +# R2. A checksum change during a transaction prevents promotion. +# R3. A Previous_gtids event larger than 64 KiB expands the reservation and +# permits promotion. +# R4. A row statement touching InnoDB and MyISAM falls back for its +# statement-cache transaction while still promoting its InnoDB cache. +# R5. Binary-log encryption prevents promotion. +# R6. Restoring binlog_checksum before commit preserves promotion eligibility. +# +# === Implementation === +# Each case creates a spilled transaction above the 10 MiB threshold and +# verifies either the normal commit path or promotion as appropriate. The +# large-header case writes 1,700 explicitly assigned GTIDs, rotates to publish +# the serialized Previous_gtids size, and uses a fresh session so its cache +# opens with the dynamic reservation. Encryption is tested last because it +# requires restart. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/have_component_keyring_file.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_checksum = query_get_value(SELECT @@GLOBAL.binlog_checksum, @@GLOBAL.binlog_checksum, 1) +--let $save_binlog_format = query_get_value(SELECT @@SESSION.binlog_format, @@SESSION.binlog_format, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); + +# Setup +--disable_query_log +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY, marker INT) ENGINE=MyISAM; + +############################################################################ +# Case 1: Large STATEMENT transaction. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A large STATEMENT transaction uses the standard commit path. +--disable_warnings +SET SESSION binlog_format = STATEMENT; +--enable_warnings +SET @bolt_statement_payload = REPEAT('s', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_statement_payload); +COMMIT; +SET @bolt_statement_payload = NULL; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A large STATEMENT transaction does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A large STATEMENT transaction increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A large STATEMENT transaction stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A large STATEMENT fallback commits its row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 1, count, 1] = 1 +--source include/assert.inc +--exec grep -q "transaction contains non-ROW events" $MYSQLTEST_VARDIR/log/mysqld.1.err + +############################################################################ +# Case 2: Large MIXED transaction. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A large MIXED transaction with a statement event uses the standard path. +--disable_warnings +SET SESSION binlog_format = MIXED; +--enable_warnings +SET @bolt_mixed_payload = REPEAT('m', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_mixed_payload); +COMMIT; +SET @bolt_mixed_payload = NULL; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A large MIXED transaction does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A large MIXED transaction increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A large MIXED transaction stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A large MIXED fallback commits its row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 1, count, 1] = 1 +--source include/assert.inc + +############################################################################ +# Case 3: Checksum change after the transaction starts. +############################################################################ +--disable_query_log +--disable_warnings +TRUNCATE t1; +SET SESSION binlog_format = ROW; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +connect (checksum_change, localhost, root,,); +connection default; +--echo # A checksum change during a large transaction uses the standard path. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('c', 5000000)); +connection checksum_change; +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +connection default; +INSERT INTO t1 VALUES (2, REPEAT('d', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('e', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A checksum change does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A checksum change increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A checksum-change fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--exec grep -q "binlog_checksum changed during the transaction" $MYSQLTEST_VARDIR/log/mysqld.1.err +connection checksum_change; +disconnect checksum_change; +connection default; +--disable_query_log +--eval SET GLOBAL binlog_checksum = $save_checksum +--enable_query_log + +############################################################################ +# Case 4: Checksum changes back before commit. +############################################################################ +--disable_query_log +--disable_warnings +TRUNCATE t1; +SET SESSION binlog_format = ROW; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +connect (checksum_restored, localhost, root,,); +connection default; +--echo # Restoring the original checksum before commit preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('r', 5000000)); +connection checksum_restored; +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +--eval SET GLOBAL binlog_checksum = $save_checksum +connection default; +INSERT INTO t1 VALUES (2, REPEAT('s', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('t', 5000000)); +--let $binlog_file_before_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Restoring the checksum before commit promotes the transaction. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = Restoring the checksum before commit does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The restored-checksum commit opens its promoted binary log. +--let $assert_cond = "$binlog_file_after_commit" != "$binlog_file_before_commit" +--source include/assert.inc +--let $assert_text = The restored-checksum promoted transaction commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +connection checksum_restored; +disconnect checksum_restored; +connection default; + +############################################################################ +# Case 5: Large Previous_gtids receives a dynamic reservation. +############################################################################ +--disable_query_log +--disable_warnings +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +SET SESSION binlog_format = ROW; +--enable_warnings +--let $gtid_index = 1 +while ($gtid_index <= 1700) +{ + --let $gtid_suffix = `SELECT LPAD(HEX($gtid_index), 12, '0')` + --eval SET @@SESSION.GTID_NEXT = 'aaaaaaaa-aaaa-aaaa-aaaa-$gtid_suffix:1' + BEGIN; + COMMIT; + SET @@SESSION.GTID_NEXT = AUTOMATIC; + --inc $gtid_index +} +--enable_query_log +--let $executed_gtid_length = query_get_value(SELECT CHAR_LENGTH(@@GLOBAL.gtid_executed) AS length, length, 1) +--let $assert_text = The executed GTID set exceeds BOLT's reserved header. +--let $assert_cond = $executed_gtid_length > 64000 +--source include/assert.inc +--disable_query_log +# Publish the serialized Previous_gtids size before opening the BOLT cache. +FLUSH BINARY LOGS; +connect (dynamic_header, localhost, root,,); +connection dynamic_header; +--disable_warnings +SET SESSION binlog_format = ROW; +--enable_warnings +SET SESSION binlog_transaction_compression = OFF; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A dynamically reserved Previous_gtids header preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A dynamically reserved Previous_gtids header promotes a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A dynamically reserved Previous_gtids header does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The dynamically reserved-header commit opens its promoted binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A dynamically reserved-header commit commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +connection default; +disconnect dynamic_header; +connection default; + +############################################################################ +# Case 6: A row statement touches InnoDB and MyISAM. +############################################################################ +# Permanent transactional/nontransactional changes violate GTID consistency, +# so restart this isolated test server with GTID enforcement disabled. +--let $restart_parameters = restart:--gtid-mode=OFF --enforce-gtid-consistency=OFF +--source include/restart_mysqld_no_echo.inc +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +RESET BINARY LOGS AND GTIDS; +TRUNCATE t1; +TRUNCATE t2; +SET SESSION binlog_format = ROW; +--enable_query_log +INSERT INTO t1 VALUES (1, REPEAT('n', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('o', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('p', 5000000)); +INSERT INTO t2 VALUES (1, 0), (2, 0), (3, 0); +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A mixed InnoDB/MyISAM row statement falls back for its statement cache. +BEGIN; +UPDATE t1 JOIN t2 USING (id) SET t1.data = CONCAT(t1.data, 'x'), t2.marker = 1; +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = An InnoDB/MyISAM statement promotes its qualifying InnoDB cache. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM statement increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM statement rotates to the promoted InnoDB binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM fallback commits every InnoDB row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM fallback updates every MyISAM row. +--let $assert_cond = [SELECT SUM(marker) AS marker_sum FROM t2, marker_sum, 1] = 3 +--source include/assert.inc +--exec grep -q "statement cache is nonempty" $MYSQLTEST_VARDIR/log/mysqld.1.err + +############################################################################ +# Case 7: Binary-log encryption. +############################################################################ +--disable_query_log +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--source ../mysql-test/suite/component_keyring_file/inc/setup_component.inc +--echo # Restart the server with binary-log encryption enabled. +--let $restart_parameters = restart:--binlog_encryption=ON $PLUGIN_DIR_OPT +--source include/restart_mysqld_no_echo.inc + +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +TRUNCATE t1; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # An encrypted binary log uses the standard commit path. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('j', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('k', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('l', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = An encrypted binary log does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = An encrypted binary log increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = An encrypted fallback stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = An encrypted fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--exec grep -q "binary log encryption is enabled" $MYSQLTEST_VARDIR/log/mysqld.1.err + +# Cleanup +DROP TABLE t2; +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL binlog_checksum = $save_checksum +--eval SET SESSION binlog_format = $save_binlog_format +--eval SET SESSION binlog_transaction_compression = $save_compression +--enable_warnings +--enable_query_log +--source ../mysql-test/suite/component_keyring_file/inc/teardown_component.inc diff --git a/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery.test b/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery.test new file mode 100644 index 000000000000..24483274b27f --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery.test @@ -0,0 +1,112 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT recovery logs the optimized seek and rejects a truncated +# promoted binary log with an invalid Large_transaction_header. +# +# === Requirements === +# R1. Recovery logs the optimized large-transaction seek after promotion. +# R2. A promoted transaction survives normal restart recovery. +# R3. A corrupt LTH event causes the server to crash during recovery. +# +# === Implementation === +# 1. Initialize BOLT and a clean test table. +# 2. Commit a qualifying transaction and restart to inspect recovery logging. +# 3. Stop the server, truncate the promoted file at a fixed 1 MiB offset, +# and verify standalone recovery fails with the invalid-LTH diagnostic. +# +--source include/not_windows.inc +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_BACKUP = $MYSQLTEST_VARDIR/tmp/bolt_lth_backup +--let $BOLT_FAILURE_LOG = $MYSQLTEST_VARDIR/tmp/bolt_invalid_lth.err + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: Recover an intact promoted transaction. +# Expected behavior: commit. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Commit a qualifying transaction for optimized recovery. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; + +--let $promoted_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The transaction was promoted into a new binary log file. +--let $assert_cond = "$promoted_file" != "$binlog_file_before" +--source include/assert.inc + +--let $restart_parameters = restart: +--let $restart_parameters = $restart_parameters --binlog-large-transaction-optimization-enabled=ON +--let $restart_parameters = $restart_parameters --binlog-large-transaction-optimization-threshold=10M +--let $restart_parameters = $restart_parameters --max-binlog-size=1G +--source include/kill_and_restart_mysqld.inc + +--echo # Recovery logs the optimized large-transaction seek. +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN = Optimized binlog recovery by avoiding a sequential read of a large transaction body and seeking to its terminating event at offset +--source include/search_pattern.inc + +--let $assert_text = The committed promoted transaction survives recovery. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +############################################################################ +# Case 2: Try to recover a file with corrupt LTH +# We simulate a corrupt LTH by manually truncating the XID event, +# so the LTH points to a position that doesn't exist. +############################################################################ +--let $binlog_file_before_corruption = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Commit a second qualifying transaction for corrupt-LTH recovery. +BEGIN; +INSERT INTO t1 VALUES (4, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (5, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (6, REPEAT('f', 4000000)); +COMMIT; + +--let $corrupt_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The second transaction was promoted into a new binary log file. +--let $assert_cond = "$corrupt_file" != "$binlog_file_before_corruption" +--source include/assert.inc + +--echo # Truncate the active promoted file and reject the invalid LTH. +--source include/kill_mysqld.inc +--let BOLT_BINLOG = $MYSQLD_DATADIR/$corrupt_file +--exec cp $BOLT_BINLOG $BOLT_BACKUP +--exec truncate -s 1048576 $BOLT_BINLOG +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_FAILURE_LOG 2>&1 +--let SEARCH_FILE = $BOLT_FAILURE_LOG +--let SEARCH_PATTERN = contains an invalid large transaction header +--source include/search_pattern.inc + +--exec cp $BOLT_BACKUP $BOLT_BINLOG +--remove_file $BOLT_BACKUP +--remove_file $BOLT_FAILURE_LOG +--let $restart_parameters = restart +--source include/start_mysqld.inc + +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_rotate_purge.test b/mysql-test/suite/binlog/t/binlog_bolt_rotate_purge.test new file mode 100644 index 000000000000..5353fa810873 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_rotate_purge.test @@ -0,0 +1,94 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify normal PURGE and FLUSH rotation remain safe while a BOLT spill file +# exists and after that spill is promoted into the main binary-log index. +# +# === Requirements === +# R1. PURGE and FLUSH do not discard or corrupt an active BOLT spill file. +# R2. A spill promotes after a concurrent FLUSH creates a new active binlog. +# R3. A promoted historical file can be purged after a subsequent FLUSH. +# +# === Implementation === +# Create history, keep a transaction spilled, purge older history and rotate, +# then commit the spill. Rotate the resulting promoted active file once more +# and purge it through the normal binary-log lifecycle. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +# Keep one historical file that PURGE can remove while a spill exists. +FLUSH BINARY LOGS; +--let $binlog_before_spill = query_get_value(SHOW BINARY LOG STATUS, File, 1) + +connect (spill_client, localhost, root,,); +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null + +connection default; +--echo # Purge older history while the qualifying spill file is still open. +--eval PURGE BINARY LOGS TO '$binlog_before_spill' +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null + +--echo # Rotate while the qualifying spill file is still open. +FLUSH BINARY LOGS; +--let $binlog_after_spill_rotation = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = FLUSH opens a new active binary log while a BOLT spill exists. +--let $assert_cond = "$binlog_after_spill_rotation" != "$binlog_before_spill" +--source include/assert.inc +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null + +connection spill_client; +COMMIT; +connection default; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $promoted_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_header_file = $promoted_file +--source ../inc/validate_bolt_header.inc +--let $assert_text = The spill promotes after PURGE and FLUSH complete. +--let $assert_cond = "$promoted_file" != "$binlog_after_spill_rotation" +--source include/assert.inc +--let $assert_text = The promoted transaction preserves every row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +--echo # Rotate the promoted active file, then purge it as ordinary history. +FLUSH BINARY LOGS; +--let $successor_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = FLUSH rotates the promoted active binary log. +--let $assert_cond = "$successor_file" != "$promoted_file" +--source include/assert.inc +--exec test -e $MYSQLD_DATADIR/$promoted_file +--eval PURGE BINARY LOGS TO '$successor_file' +--let $file_does_not_exist = $MYSQLD_DATADIR/$promoted_file +--source include/file_does_not_exist.inc +SHOW BINARY LOGS; + +connection spill_client; +disconnect spill_client; +connection default; + +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_savepoint-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_savepoint-master.opt new file mode 100644 index 000000000000..0123ec2d69f6 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_savepoint-master.opt @@ -0,0 +1,4 @@ +--binlog-cache-size=1M +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G diff --git a/mysql-test/suite/binlog/t/binlog_bolt_savepoint.test b/mysql-test/suite/binlog/t/binlog_bolt_savepoint.test new file mode 100644 index 000000000000..cfc5f85a21d1 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_savepoint.test @@ -0,0 +1,118 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify a savepoint rollback uses the final transaction-cache size to choose +# either ordinary active-binlog copying or BOLT promotion. +# +# === Requirements === +# R1. A spilled cache truncated below binlog_cache_size commits normally. +# R2. A spilled cache above binlog_cache_size but below the BOLT threshold +# commits normally. +# R3. A spilled cache above the BOLT threshold promotes a new active binlog. +# +# === Implementation === +# Configure a 1 MiB cache and a 10 MiB BOLT threshold. Each case spills the +# transaction cache, rolls back to a savepoint, commits, and checks the final +# active binary-log filename and retained rows. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: Rollback leaves less than binlog_cache_size. +# Expected behavior: copy the retained cache to the current active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Savepoint rollback below binlog_cache_size keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 512000)); +SAVEPOINT before_spill; +INSERT INTO t1 VALUES (2, REPEAT('b', 2000000)); +ROLLBACK TO SAVEPOINT before_spill; +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A final cache below binlog_cache_size is copied to the active binlog. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Savepoint rollback retains only the pre-spill row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 1 +--source include/assert.inc +--let $assert_text = Savepoint rollback discards the post-savepoint row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 1 +--source include/assert.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 2: Rollback leaves more than binlog_cache_size but less than threshold. +# Expected behavior: copy the retained cache to the current active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Savepoint rollback below the BOLT threshold keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (11, REPEAT('c', 2000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (12, REPEAT('d', 3000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A final cache below the BOLT threshold is copied to the active binlog. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Savepoint rollback retains the above-cache pre-savepoint row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 1 +--source include/assert.inc +--let $assert_text = Savepoint rollback discards the above-cache post-savepoint row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 11 +--source include/assert.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 3: Rollback leaves more than binlog_cache_size and BOLT threshold. +# Expected behavior: promote a new active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Savepoint rollback above the BOLT threshold promotes a new binlog. +BEGIN; +INSERT INTO t1 VALUES (21, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (22, REPEAT('f', 4000000)); +INSERT INTO t1 VALUES (23, REPEAT('g', 4000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (24, REPEAT('h', 2000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A final cache above the BOLT threshold promotes a new active binlog. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Savepoint rollback retains the promoted pre-savepoint rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = Savepoint rollback discards the promoted post-savepoint row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 66 +--source include/assert.inc + +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_sysvars.test b/mysql-test/suite/binlog/t/binlog_bolt_sysvars.test new file mode 100644 index 000000000000..b86d66e4bece --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_sysvars.test @@ -0,0 +1,168 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT threshold and binlog-cache normalization during runtime SETs and +# server restart. +# +# === Requirements === +# R1. The optimization threshold is never below binlog_cache_size. +# R2. BOLT-on normalization is reported to the client and error log. +# R3. BOLT-off normalization preserves the invariant without BOLT warnings. +# +# === Implementation === +# 1. Initialize saved BOLT and cache-size configuration. +# 2. Exercise valid and invalid threshold/cache relationships while BOLT is on. +# 3. Repeat the relationships while BOLT is off and restore configuration. +# +--source include/have_log_bin.inc + +# Preserve the caller's configuration and restore it after the assertions. +--let $save_enabled=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_cache_size=query_get_value(SELECT @@GLOBAL.binlog_cache_size, @@GLOBAL.binlog_cache_size, 1) + +# Setup +# Start all BOLT-on runtime cases from a default server configuration. +--let $restart_parameters=restart: +--source include/restart_mysqld.inc +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL binlog_cache_size = 10485760; +--enable_query_log +--enable_warnings + +--echo # Case 1: BOLT is ON; binlog_cache_size > binlog_large_transaction_optimization_threshold. +# Verify normalization after a runtime SET. +--source include/save_error_log_position.inc +--replace_regex /\t[0-9]+\t/\t\t/ +SET GLOBAL binlog_cache_size = 20971520; +--let $error_pattern=Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.] +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= Increasing binlog_cache_size raises the optimization threshold. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +# Verify normalization during restart recovery. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.] +--source include/assert_error_log.inc + +--echo # Case 2: BOLT is ON; binlog_large_transaction_optimization_threshold < binlog_cache_size. +# Verify normalization after a runtime SET. +--source include/save_error_log_position.inc +--replace_regex /\t[0-9]+\t/\t\t/ +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $error_pattern=Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.] +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A lower optimization threshold is raised to match binlog_cache_size. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +--echo # Case 3: BOLT is ON; binlog_large_transaction_optimization_threshold > binlog_cache_size. +# Verify the valid relationship is unchanged after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A threshold above binlog_cache_size is unchanged. +--let $assert_cond= $threshold = 22020096 +--source include/assert.inc + +# Verify lowering binlog_cache_size does not lower the threshold. +--source include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 10485760; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $cache_size=query_get_value(SELECT @@GLOBAL.binlog_cache_size, @@GLOBAL.binlog_cache_size, 1) +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= Lowering binlog_cache_size does not lower the threshold. +--let $assert_cond= $cache_size = 10485760 AND $threshold = 22020096 +--source include/assert.inc + +# Verify the valid configuration restarts without adjustment. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=ON --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=NONE +--source include/assert_error_log.inc + +# Start all BOLT-off runtime cases from a default server configuration. +--let $restart_parameters=restart: +--source include/restart_mysqld.inc +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL binlog_cache_size = 10485760; +--enable_query_log +--enable_warnings + +--echo # Case 4: BOLT is OFF; binlog_cache_size > binlog_large_transaction_optimization_threshold. +# Verify normalization remains silent after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= Increasing binlog_cache_size raises the threshold while BOLT is off. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +# Verify startup normalization remains silent while BOLT is off. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=NONE +--source include/assert_error_log.inc + +--echo # Case 5: BOLT is OFF; binlog_large_transaction_optimization_threshold < binlog_cache_size. +# Verify normalization remains silent after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A lower threshold is raised while BOLT is off. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +--echo # Case 6: BOLT is OFF; binlog_large_transaction_optimization_threshold > binlog_cache_size. +# Verify the valid relationship remains unchanged after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A threshold above binlog_cache_size is unchanged while BOLT is off. +--let $assert_cond= $threshold = 22020096 +--source include/assert.inc + +# Verify the valid configuration restarts silently while BOLT is off. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=NONE +--source include/assert_error_log.inc + +--let $restart_parameters=restart: +--source include/restart_mysqld.inc + +# Restore the caller's variable-dependent configuration without recording +# environment-specific values in the result file. +--disable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL binlog_cache_size = $save_cache_size +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_tmp_dir_cleanup.test b/mysql-test/suite/binlog/t/binlog_bolt_tmp_dir_cleanup.test new file mode 100644 index 000000000000..4215bd2eb3cf --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_tmp_dir_cleanup.test @@ -0,0 +1,85 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify release binaries clean managed BOLT spill files and reject unsafe +# temporary-directory entries. +# +# === Requirements === +# R1. Startup removes valid managed BOLT spill files. +# R2. Startup rejects symlinked directories and unsafe directory entries. +# R3. Rejected startup fixtures can be removed before the MTR server restarts. +# +# === Implementation === +# 1. Initialize paths for the managed spill directory and rejection logs. +# 2. Restart with valid managed files and verify they are removed. +# 3. Attempt standalone startup with each unsafe fixture and verify rejection. +# +--source include/not_windows.inc +--source include/have_log_bin.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_CLEANUP_DIR = $MYSQLD_DATADIR/#binlog_temp_files +--let $BOLT_CLEANUP_REJECT_LOG = $MYSQL_TMP_DIR/binlog_bolt_cleanup_reject.err +--let $BOLT_CLEANUP_DIR_TARGET = $MYSQLD_DATADIR/binlog_bolt_cleanup_dir_target +--let $BOLT_CLEANUP_FILE_TARGET = $MYSQLD_DATADIR/binlog_bolt_cleanup_file_target + +# Setup +--exec touch $BOLT_CLEANUP_DIR/bolt_0123456789abcdef0123456789abcdef +--exec touch $BOLT_CLEANUP_DIR/bolt_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +--exec touch $BOLT_CLEANUP_DIR/bolt_0123456789abcdefabcdef0123456789 +--echo Created 3 managed BOLT spill files + +--source include/restart_mysqld.inc + +--list_files $BOLT_CLEANUP_DIR +--echo Startup cleanup removed all managed BOLT spill files + +# Prepare each startup fixture while the MTR server is running. Stop it only +# immediately before the independent release-binary startup attempt, then +# remove the rejected fixture before bringing the MTR server back. + +--echo # Reject a symbolic link for #binlog_temp_files. +--rmdir $BOLT_CLEANUP_DIR +--exec mkdir $BOLT_CLEANUP_DIR_TARGET +--exec ln -s $BOLT_CLEANUP_DIR_TARGET $BOLT_CLEANUP_DIR +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "it must be a directory and cannot be a symbolic link." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_DIR +--rmdir $BOLT_CLEANUP_DIR_TARGET +--exec mkdir $BOLT_CLEANUP_DIR +--source include/start_mysqld.inc + +--echo # Reject an unrecognized regular entry. +--exec touch $BOLT_CLEANUP_DIR/not_a_managed_bolt_file +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "found unsafe entry 'not_a_managed_bolt_file'." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_DIR/not_a_managed_bolt_file +--source include/start_mysqld.inc + +--echo # Reject a symbolic-link entry, even with a managed file name. +--exec touch $BOLT_CLEANUP_FILE_TARGET +--exec ln -s $BOLT_CLEANUP_FILE_TARGET $BOLT_CLEANUP_DIR/bolt_11111111111111111111111111111111 +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "found unsafe entry 'bolt_11111111111111111111111111111111'." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_DIR/bolt_11111111111111111111111111111111 +--remove_file $BOLT_CLEANUP_FILE_TARGET +--source include/start_mysqld.inc + +--echo # Reject a nested directory, even with a managed file name. +--exec mkdir $BOLT_CLEANUP_DIR/bolt_22222222222222222222222222222222 +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "found unsafe entry 'bolt_22222222222222222222222222222222'." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--rmdir $BOLT_CLEANUP_DIR/bolt_22222222222222222222222222222222 +--source include/start_mysqld.inc diff --git a/mysql-test/suite/binlog/t/binlog_encryption_random_access.test b/mysql-test/suite/binlog/t/binlog_encryption_random_access.test index 9d137946b57d..b78ac4dfe529 100644 --- a/mysql-test/suite/binlog/t/binlog_encryption_random_access.test +++ b/mysql-test/suite/binlog/t/binlog_encryption_random_access.test @@ -16,6 +16,10 @@ # Suppression of error messages CALL mtr.add_suppression('Unsafe statement written to the binary log using statement format'); +--let $messages = Could not optimize large transaction execution in the binary log because binary log encryption is enabled; standard binary logging was used instead. +--let $suppress_silent = 1 +--source include/suppress_messages.inc +--let $suppress_silent = --source include/have_component_keyring_file.inc --source ../mysql-test/suite/component_keyring_file/inc/setup_component.inc @@ -63,6 +67,38 @@ while ($trx) } --enable_query_log +--echo # Binlog encryption forces a qualifying row transaction to use the standard path. +--let $save_enabled=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--disable_query_log +SET SESSION binlog_format = ROW; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--enable_query_log +CREATE TABLE t_large (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +--let $count_before_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +BEGIN; +INSERT INTO t_large VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t_large VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t_large VALUES (3, REPEAT('c', 5000000)); +COMMIT; +--let $count_after_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text= An encrypted binary log uses the standard path. +--let $assert_cond= $count_after_encryption = $count_before_encryption +--source include/assert.inc +--let $assert_text= An encrypted binary log increments the missed counter once. +--let $assert_cond= $missed_after_encryption = $missed_before_encryption + 1 +--source include/assert.inc +--let $assert_text= The encrypted fallback transaction commits all rows. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t_large, count, 1] = 3 +--source include/assert.inc + # Cleanup -DROP TABLE t1; +DROP TABLE t_large, t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--enable_query_log --source ../mysql-test/suite/component_keyring_file/inc/teardown_component.inc diff --git a/mysql-test/suite/binlog_gtid/r/binlog_bolt_gtid_tag.result b/mysql-test/suite/binlog_gtid/r/binlog_bolt_gtid_tag.result new file mode 100644 index 000000000000..30c7375883b2 --- /dev/null +++ b/mysql-test/suite/binlog_gtid/r/binlog_bolt_gtid_tag.result @@ -0,0 +1,29 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Commit tagged and untagged transactions before the promoted transaction. +SET GTID_NEXT = 'AUTOMATIC:prior_large_trx_tag'; +INSERT INTO t1 VALUES (1, 'prior-tagged-transaction'); +SET GTID_NEXT = 'AUTOMATIC'; +INSERT INTO t1 VALUES (2, 'prior-untagged-transaction'); +SET GTID_NEXT = 'AUTOMATIC:promoted_large_trx_tag'; +BEGIN; +INSERT INTO t1 VALUES (3, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (4, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (5, REPEAT('c', 5000000)); +COMMIT; +SET GTID_NEXT = 'AUTOMATIC'; +include/assert.inc [The tagged GTID transaction was promoted.] +include/assert.inc [The tagged GTID promotion rotates to a new active binary log.] +include/assert.inc [The prior tagged GTID is persisted before promotion.] +include/assert.inc [The promoted tagged GTID is recorded in gtid_executed.] +# The promoted file retains both tag forms. +# Restart and verify both tags survive persisted GTID state reload. +# restart +include/assert.inc [The prior tagged GTID survives restart.] +include/assert.inc [The promoted tagged GTID survives restart.] +include/assert.inc [Both GTID tags remain in mysql.gtid_executed after restart.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result b/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result index 4397aa451e87..a95bb5dc9035 100644 --- a/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result +++ b/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result @@ -1,7 +1,7 @@ CREATE TABLE t1 (c1 INT); include/assert.inc [Verify that the starting offset (4) of an event after the invalid position 1 is displayed.] include/assert.inc [Verify that the starting offset (4) of an event at the valid position 4 is displayed.] -include/assert.inc [Verify that the starting offset (127) of an event after the invalid position 14 is displayed.] -include/assert.inc [Verify that the starting offset (158) of an event after the invalid position 127 is displayed.] -include/assert.inc [Verify that the starting offset (158) of an event at the valid position 157 is displayed.] +include/assert.inc [Verify that the starting offset (128) of an event after the invalid position 14 is displayed.] +include/assert.inc [Verify that the starting offset (159) of an event after the invalid position 129 is displayed.] +include/assert.inc [Verify that the starting offset (159) of an event at the valid position 159 is displayed.] DROP TABLE t1; diff --git a/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag.test b/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag.test new file mode 100644 index 000000000000..93473e402f8a --- /dev/null +++ b/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag.test @@ -0,0 +1,101 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT promotion persists tagged GTIDs in the binary log and GTID +# execution state. +# +# === Requirements === +# R1. A qualifying tagged-GTID transaction is promoted into a new active file. +# R2. Recovery preserves its GTID in the binary log and mysql.gtid_executed. +# R3. The promoted tagged-GTID file satisfies the shared BOLT structural +# validation, including its LTH terminal-event metadata. +# +# === Implementation === +# 1. Initialize BOLT, GTID state, and a clean test table. +# 2. Commit a qualifying tagged-GTID transaction and verify file rotation. +# 3. Restart and verify persisted GTID state. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +--echo # Commit tagged and untagged transactions before the promoted transaction. +SET GTID_NEXT = 'AUTOMATIC:prior_large_trx_tag'; +INSERT INTO t1 VALUES (1, 'prior-tagged-transaction'); +SET GTID_NEXT = 'AUTOMATIC'; +INSERT INTO t1 VALUES (2, 'prior-untagged-transaction'); + +--let $server_uuid = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +--let $count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +SET GTID_NEXT = 'AUTOMATIC:promoted_large_trx_tag'; +BEGIN; +INSERT INTO t1 VALUES (3, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (4, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (5, REPEAT('c', 5000000)); +COMMIT; +SET GTID_NEXT = 'AUTOMATIC'; + +--let $count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The tagged GTID transaction was promoted. +--let $assert_cond = $count_after = $count_before + 1 +--source include/assert.inc +--let $assert_text = The tagged GTID promotion rotates to a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $prior_tag_rows = query_get_value(SELECT COUNT(*) AS count FROM mysql.gtid_executed WHERE source_uuid = "$server_uuid" AND gtid_tag = 'prior_large_trx_tag' AND interval_start = 1 AND interval_end = 1, count, 1) +--let $assert_text = The prior tagged GTID is persisted before promotion. +--let $assert_cond = $prior_tag_rows = 1 +--source include/assert.inc +--let $promoted_tag_executed = `SELECT GTID_SUBSET('$server_uuid:promoted_large_trx_tag:1', @@GLOBAL.GTID_EXECUTED)` +--let $assert_text = The promoted tagged GTID is recorded in gtid_executed. +--let $assert_cond = $promoted_tag_executed = 1 +--source include/assert.inc + +--echo # The promoted file retains both tag forms. +--let $promoted_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $MYSQLD_DATADIR = `SELECT @@datadir` +--let $bolt_header_file = $promoted_file +--source common/binlog/validate_bolt_file.inc +--let $gtid_output = $MYSQLTEST_VARDIR/tmp/large_trx_tagged_gtids.txt +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$promoted_file > $gtid_output +--exec grep -Eq '[0-9a-f-]+:1-2' $gtid_output +--exec grep -Eq '[0-9a-f-]+:prior_large_trx_tag:1' $gtid_output +--exec grep -Eq '[0-9a-f-]+:promoted_large_trx_tag:1' $gtid_output +--remove_file $gtid_output + +--echo # Restart and verify both tags survive persisted GTID state reload. +--let $do_not_echo_parameters = 1 +--source include/restart_mysqld.inc +--let $do_not_echo_parameters = + +--let $prior_tag_executed = `SELECT GTID_SUBSET('$server_uuid:prior_large_trx_tag:1', @@GLOBAL.GTID_EXECUTED)` +--let $assert_text = The prior tagged GTID survives restart. +--let $assert_cond = $prior_tag_executed = 1 +--source include/assert.inc +--let $promoted_tag_executed = `SELECT GTID_SUBSET('$server_uuid:promoted_large_trx_tag:1', @@GLOBAL.GTID_EXECUTED)` +--let $assert_text = The promoted tagged GTID survives restart. +--let $assert_cond = $promoted_tag_executed = 1 +--source include/assert.inc +--let $persisted_tag_rows = query_get_value(SELECT COUNT(*) AS count FROM mysql.gtid_executed WHERE source_uuid = "$server_uuid" AND (gtid_tag = 'prior_large_trx_tag' OR gtid_tag = 'promoted_large_trx_tag'), count, 1) +--let $assert_text = Both GTID tags remain in mysql.gtid_executed after restart. +--let $assert_cond = $persisted_tag_rows = 2 +--source include/assert.inc + +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--enable_query_log diff --git a/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test b/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test index 2837dd838e9f..9bbba8248224 100644 --- a/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test +++ b/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test @@ -33,16 +33,16 @@ CREATE TABLE t1 (c1 INT); --let $assert_cond="[SHOW BINLOG EVENTS FROM 4 LIMIT 1, Pos, 1]" = 4 --source include/assert.inc ---let $assert_text=Verify that the starting offset (127) of an event after the invalid position 14 is displayed. ---let $assert_cond="[SHOW BINLOG EVENTS FROM 14 LIMIT 1, Pos, 1]" = 127 +--let $assert_text=Verify that the starting offset (128) of an event after the invalid position 14 is displayed. +--let $assert_cond="[SHOW BINLOG EVENTS FROM 14 LIMIT 1, Pos, 1]" = 128 --source include/assert.inc ---let $assert_text=Verify that the starting offset (158) of an event after the invalid position 127 is displayed. ---let $assert_cond="[SHOW BINLOG EVENTS FROM 128 LIMIT 1, Pos, 1]" = 158 +--let $assert_text=Verify that the starting offset (159) of an event after the invalid position 129 is displayed. +--let $assert_cond="[SHOW BINLOG EVENTS FROM 129 LIMIT 1, Pos, 1]" = 159 --source include/assert.inc ---let $assert_text=Verify that the starting offset (158) of an event at the valid position 157 is displayed. ---let $assert_cond="[SHOW BINLOG EVENTS FROM 158 LIMIT 1, Pos, 1]" = 158 +--let $assert_text=Verify that the starting offset (159) of an event at the valid position 159 is displayed. +--let $assert_cond="[SHOW BINLOG EVENTS FROM 159 LIMIT 1, Pos, 1]" = 159 --source include/assert.inc DROP TABLE t1; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result index cbd1e142f344..5388c5d6e69f 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result @@ -47,7 +47,7 @@ INSERT INTO aliases(name) VALUES ('slave_parallel_workers'), ('slave_pending_jobs_size_max'), ('pseudo_slave_mode'), ('skip_slave_start'); -include/assert.inc [Expect 110 variables in the table.] +include/assert.inc [Expect 112 variables in the table.] # Test SET PERSIST_ONLY SET PERSIST_ONLY binlog_cache_size = @@GLOBAL.binlog_cache_size; @@ -64,6 +64,8 @@ Warning 1287 '@@binlog_format' is deprecated and will be removed in a future rel SET PERSIST_ONLY binlog_group_commit_sync_delay = @@GLOBAL.binlog_group_commit_sync_delay; SET PERSIST_ONLY binlog_group_commit_sync_no_delay_count = @@GLOBAL.binlog_group_commit_sync_no_delay_count; SET PERSIST_ONLY binlog_gtid_simple_recovery = @@GLOBAL.binlog_gtid_simple_recovery; +SET PERSIST_ONLY binlog_large_transaction_optimization_enabled = @@GLOBAL.binlog_large_transaction_optimization_enabled; +SET PERSIST_ONLY binlog_large_transaction_optimization_threshold = @@GLOBAL.binlog_large_transaction_optimization_threshold; SET PERSIST_ONLY binlog_max_flush_queue_time = @@GLOBAL.binlog_max_flush_queue_time; Warnings: Warning 1287 '@@binlog_max_flush_queue_time' is deprecated and will be removed in a future release. @@ -257,16 +259,16 @@ Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a futu Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a future release. SET PERSIST_ONLY sync_source_info = @@GLOBAL.sync_source_info; -include/assert.inc [Expect 99 persisted variables in persisted_variables table.] +include/assert.inc [Expect 101 persisted variables in persisted_variables table.] ############################################################ # 2. Restart server, it must preserve the persisted variable # settings. Verify persisted configuration. # restart -include/assert.inc [Expect 99 persisted variables in persisted_variables table.] -include/assert.inc [Expect 99 persisted variables shown as PERSISTED in variables_info table.] -include/assert.inc [Expect 99 persisted variables with matching persisted and global values.] +include/assert.inc [Expect 101 persisted variables in persisted_variables table.] +include/assert.inc [Expect 101 persisted variables shown as PERSISTED in variables_info table.] +include/assert.inc [Expect 101 persisted variables with matching persisted and global values.] ############################################################ # 3. Test RESET PERSIST. Verify persisted variable settings @@ -282,6 +284,8 @@ RESET PERSIST binlog_format; RESET PERSIST binlog_group_commit_sync_delay; RESET PERSIST binlog_group_commit_sync_no_delay_count; RESET PERSIST binlog_gtid_simple_recovery; +RESET PERSIST binlog_large_transaction_optimization_enabled; +RESET PERSIST binlog_large_transaction_optimization_threshold; RESET PERSIST binlog_max_flush_queue_time; RESET PERSIST binlog_order_commits; RESET PERSIST binlog_rotate_encryption_master_key_at_startup; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result index 29ada59483ed..ed42893343f3 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result @@ -26,7 +26,7 @@ VARIABLE_NAME LIKE '%source%') AND 'innodb_master_thread_disabled_debug', 'innodb_replication_delay')) ORDER BY VARIABLE_NAME; -include/assert.inc [Expect 110 variables in the table.] +include/assert.inc [Expect 112 variables in the table.] # Test SET PERSIST SET PERSIST binlog_cache_size = @@GLOBAL.binlog_cache_size; @@ -44,6 +44,8 @@ SET PERSIST binlog_group_commit_sync_delay = @@GLOBAL.binlog_group_commit_sync_d SET PERSIST binlog_group_commit_sync_no_delay_count = @@GLOBAL.binlog_group_commit_sync_no_delay_count; SET PERSIST binlog_gtid_simple_recovery = @@GLOBAL.binlog_gtid_simple_recovery; ERROR HY000: Variable 'binlog_gtid_simple_recovery' is a read only variable +SET PERSIST binlog_large_transaction_optimization_enabled = @@GLOBAL.binlog_large_transaction_optimization_enabled; +SET PERSIST binlog_large_transaction_optimization_threshold = @@GLOBAL.binlog_large_transaction_optimization_threshold; SET PERSIST binlog_max_flush_queue_time = @@GLOBAL.binlog_max_flush_queue_time; Warnings: Warning 1287 '@@binlog_max_flush_queue_time' is deprecated and will be removed in a future release. @@ -234,16 +236,16 @@ Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a futu Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a future release. SET PERSIST sync_source_info = @@GLOBAL.sync_source_info; -include/assert.inc [Expect 88 persisted variables in persisted_variables table.] +include/assert.inc [Expect 90 persisted variables in persisted_variables table.] ############################################################ # 2. Restart server, it must preserve the persisted variable # settings. Verify persisted configuration. # restart -include/assert.inc [Expect 88 persisted variables in persisted_variables table.'] -include/assert.inc [Expect 88 persisted variables shown as PERSISTED in variables_info table.'] -include/assert.inc [Expect 88 persisted variables with matching persisted and global values.] +include/assert.inc [Expect 90 persisted variables in persisted_variables table.'] +include/assert.inc [Expect 90 persisted variables shown as PERSISTED in variables_info table.'] +include/assert.inc [Expect 90 persisted variables with matching persisted and global values.] ############################################################ # 3. Test RESET PERSIST IF EXISTS. Verify persisted variable @@ -261,6 +263,8 @@ RESET PERSIST IF EXISTS binlog_group_commit_sync_no_delay_count; RESET PERSIST IF EXISTS binlog_gtid_simple_recovery; Warnings: Warning 3615 Variable binlog_gtid_simple_recovery does not exist in persisted config file +RESET PERSIST IF EXISTS binlog_large_transaction_optimization_enabled; +RESET PERSIST IF EXISTS binlog_large_transaction_optimization_threshold; RESET PERSIST IF EXISTS binlog_max_flush_queue_time; RESET PERSIST IF EXISTS binlog_order_commits; RESET PERSIST IF EXISTS binlog_rotate_encryption_master_key_at_startup; diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test index b7a5237e2ba1..a336e8c81f4a 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test @@ -84,7 +84,7 @@ INSERT INTO aliases(name) VALUES # If this count differs, it means a variable has been added or removed. # In that case, this testcase needs to be updated accordingly. --echo ---let $expected = 110 +--let $expected = 112 --let $assert_text = Expect $expected variables in the table. --let $assert_cond = [SELECT COUNT(*) as count FROM rplvars, count, 1] = $expected --source include/assert.inc @@ -116,7 +116,7 @@ while ( $varid <= $countvars ) } --echo ---let $expected = 99 +--let $expected = 101 --let $assert_text = Expect $expected persisted variables in persisted_variables table. --let $assert_cond = [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected --source include/assert.inc diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test index 225095e31241..1056076de9cd 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test @@ -62,7 +62,7 @@ INSERT INTO rplvars (varname, varvalue) # If this count differs, it means a variable has been added or removed. # In that case, this testcase needs to be updated accordingly. --echo ---let $expected = 110 +--let $expected = 112 --let $assert_text = Expect $expected variables in the table. --let $assert_cond = [SELECT COUNT(*) as count FROM rplvars, count, 1] = $expected --source include/assert.inc @@ -85,7 +85,7 @@ while ( $varid <= $countvars ) } --echo ---let $expected = 88 +--let $expected = 90 --let $assert_text = Expect $expected persisted variables in persisted_variables table. --let $assert_cond = [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected --source include/assert.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica.result b/mysql-test/suite/rpl/r/binlog_bolt_replica.result new file mode 100644 index 000000000000..e7ea3eb7ef82 --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica.result @@ -0,0 +1,37 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +Warnings: +Note 1051 Unknown table 'test.t1' +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +[connection slave] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +include/rpl/start_receiver.inc +[connection master] +# Write a qualifying promoted transaction on the source. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +include/assert.inc [The source transaction was promoted before replication.] +include/rpl/sync_to_replica_received.inc +[connection slave] +include/rpl/start_applier.inc +[connection master] +include/rpl/sync_to_replica.inc +[connection slave] +include/assert.inc [Replication applies every row from the source BOLT transaction.] +include/assert.inc [Replication preserves the full large-transaction payload.] +include/assert.inc [The replica SQL applier promotes the qualifying transaction once.] +include/assert.inc [Replica-local promotion opens a new active binary log.] +[connection master] +DROP TABLE t1; +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica_filter.result b/mysql-test/suite/rpl/r/binlog_bolt_replica_filter.result new file mode 100644 index 000000000000..7c5db1bf942b --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica_filter.result @@ -0,0 +1,50 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE DATABASE bolt_allowed; +CREATE DATABASE bolt_filtered; +CREATE TABLE bolt_allowed.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE bolt_filtered.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +[connection slave] +include/rpl/start_replica.inc +[connection master] +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/stop_replica.inc +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (bolt_filtered);; +include/rpl/start_receiver.inc +[connection master] +# Commit a large allowed subset together with filtered rows. +BEGIN; +INSERT INTO bolt_allowed.t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (3, REPEAT('c', 4000000)); +INSERT INTO bolt_filtered.t1 VALUES (1, 'filtered'); +COMMIT; +include/rpl/sync_to_replica_received.inc +[connection slave] +include/rpl/start_applier.inc +[connection master] +include/rpl/sync_to_replica.inc +[connection slave] +include/assert.inc [The allowed database retains every large transaction row.] +include/assert.inc [The allowed database preserves the full large payload.] +include/assert.inc [The filtered database receives no rows from the transaction.] +include/assert.inc [The replica promotes the retained qualifying subset once.] +include/assert.inc [Replica-local BOLT opens a new active binary log after filtering.] +include/rpl/stop_replica.inc +CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (); +include/rpl/start_replica.inc +[connection master] +DROP DATABASE bolt_filtered; +DROP DATABASE bolt_allowed; +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica_until.result b/mysql-test/suite/rpl/r/binlog_bolt_replica_until.result new file mode 100644 index 000000000000..8ac2b660bb38 --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica_until.result @@ -0,0 +1,42 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +Warnings: +Note 1051 Unknown table 'test.t1' +Warnings: +Note 1051 Unknown table 'test.t2' +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; +[connection slave] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +include/rpl/start_receiver.inc +[connection master] +# Commit the BOLT transaction, record its end position, then add a marker. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +INSERT INTO t2 VALUES (1); +include/rpl/sync_to_replica_received.inc +[connection slave] +START REPLICA SQL_THREAD UNTIL SOURCE_LOG_FILE='SOURCE_LOG_FILE', SOURCE_LOG_POS=SOURCE_LOG_POS;; +include/rpl/wait_for_applier_to_stop.inc +include/assert.inc [START REPLICA UNTIL applies every BOLT transaction row.] +include/assert.inc [START REPLICA UNTIL stops before the later marker transaction.] +include/assert.inc [The replica SQL applier promotes the transaction once before stopping.] +include/assert.inc [Replica-local promotion opens a new active binary log before stopping.] +include/rpl/assert_replica_status.inc [Exec_Source_Log_Pos] +include/rpl/start_applier.inc +[connection master] +DROP TABLE t2; +DROP TABLE t1; +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result b/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result index 5921dd2d0e3f..045b1c9da9e3 100644 --- a/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result +++ b/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result @@ -169,7 +169,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 496, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 497, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#8): @@ -194,7 +194,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 496, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 497, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#8): @@ -219,7 +219,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # BEGIN include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 700, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 701, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#10): @@ -244,7 +244,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 779, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 780, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#11): @@ -269,7 +269,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # BEGIN include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 779, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 780, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#11): @@ -294,7 +294,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # use `test`; CREATE TABLE t1 (c1 INT) ENGINE= InnoDB include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 965, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 966, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#13): @@ -319,7 +319,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # use `test`; CREATE TABLE t1 (c1 INT) ENGINE= InnoDB include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1456, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1457, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#19): @@ -344,7 +344,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 965, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 966, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#13): @@ -369,7 +369,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1456, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1457, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#19): @@ -394,7 +394,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 965, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 966, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#13): @@ -419,7 +419,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1456, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1457, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#19): @@ -444,7 +444,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2045, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2046, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#26): @@ -469,7 +469,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # use `test`; CREATE TABLE t1 (c1 INT) ENGINE= InnoDB include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2045, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2046, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#26): diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica-slave.opt new file mode 100644 index 000000000000..4ed6f297d980 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica-slave.opt @@ -0,0 +1,2 @@ +--log-replica-updates +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica.test b/mysql-test/suite/rpl/t/binlog_bolt_replica.test new file mode 100644 index 000000000000..bc38b6ccd3e5 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica.test @@ -0,0 +1,102 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify a replica re-logging a qualifying source BOLT transaction also uses +# BOLT for its local binary log. +# +# === Requirements === +# R1. A source BOLT transaction replicates successfully to the replica. +# R2. With log_replica_updates enabled, the replica SQL applier promotes its +# local binlog cache for the qualifying ROW transaction. +# +# === Implementation === +# 1. Initialize source/replica replication, BOLT, and a clean source table. +# 2. Commit a qualifying BOLT transaction on the source and wait until the +# replica receiver has it available. +# 3. Enable BOLT before starting the replica applier and verify its local +# promotion and promoted-file header. +# +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc +--let $rpl_skip_start_slave = 1 +--source include/rpl/init_source_replica.inc + +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $source_save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +--source include/rpl/connection_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/start_receiver.inc +--source include/rpl/connection_source.inc + +--echo # Write a qualifying promoted transaction on the source. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +--let $source_optimized_count = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The source transaction was promoted before replication. +--let $assert_cond = $source_optimized_count > 0 +--source include/assert.inc + +--source include/rpl/sync_to_replica_received.inc + +--source include/rpl/connection_replica.inc +--source include/rpl/start_applier.inc +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_replica.inc +--let $replica_rows = query_get_value(SELECT COUNT(*) FROM t1, COUNT(*), 1) +--let $replica_payload_bytes = query_get_value(SELECT SUM(OCTET_LENGTH(data)) FROM t1, SUM(OCTET_LENGTH(data)), 1) +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Replication applies every row from the source BOLT transaction. +--let $assert_cond = $replica_rows = 3 +--source include/assert.inc +--let $assert_text = Replication preserves the full large-transaction payload. +--let $assert_cond = $replica_payload_bytes = 12000000 +--source include/assert.inc +--let $assert_text = The replica SQL applier promotes the qualifying transaction once. +--let $assert_cond = $replica_bolt_after = $replica_bolt_before + 1 +--source include/assert.inc +--let $assert_text = Replica-local promotion opens a new active binary log. +--let $assert_cond = "$replica_binlog_after" != "$replica_binlog_before" +--source include/assert.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_header_file = $replica_binlog_after +--source ../../binlog/inc/validate_bolt_header.inc + +--source include/rpl/connection_source.inc +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--eval SET SESSION binlog_transaction_compression = $source_save_compression +--enable_query_log +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings +--enable_query_log +--let $rpl_skip_sync = 1 +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_filter-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter-slave.opt new file mode 100644 index 000000000000..4ed6f297d980 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter-slave.opt @@ -0,0 +1,2 @@ +--log-replica-updates +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_filter.test b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter.test new file mode 100644 index 000000000000..be96ae6387b4 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter.test @@ -0,0 +1,108 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify a replica filters part of a qualifying source transaction while +# promoting the retained large ROW subset into its local binary log. +# +# === Requirements === +# R1. Rows in the allowed database apply and trigger one local BOLT promotion. +# R2. Rows in the filtered database do not apply. +# +# === Implementation === +# Synchronize both schemas, stop the replica, configure REPLICATE_IGNORE_DB, +# then apply one transaction containing large allowed rows and filtered rows. + +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc +--let $rpl_skip_start_slave = 1 +--source include/rpl/init_source_replica.inc + +--source include/rpl/connection_source.inc +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--disable_query_log +--disable_warnings +DROP DATABASE IF EXISTS bolt_allowed; +DROP DATABASE IF EXISTS bolt_filtered; +--enable_warnings +--enable_query_log +CREATE DATABASE bolt_allowed; +CREATE DATABASE bolt_filtered; +CREATE TABLE bolt_allowed.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE bolt_filtered.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +--source include/rpl/connection_replica.inc +--source include/rpl/start_replica.inc +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--source include/rpl/stop_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--eval CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (bolt_filtered); +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/start_receiver.inc + +--source include/rpl/connection_source.inc +--echo # Commit a large allowed subset together with filtered rows. +BEGIN; +INSERT INTO bolt_allowed.t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (3, REPEAT('c', 4000000)); +INSERT INTO bolt_filtered.t1 VALUES (1, 'filtered'); +COMMIT; +--source include/rpl/sync_to_replica_received.inc + +--source include/rpl/connection_replica.inc +--source include/rpl/start_applier.inc +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_replica.inc +--let $allowed_rows = query_get_value(SELECT COUNT(*) FROM bolt_allowed.t1, COUNT(*), 1) +--let $allowed_payload = query_get_value(SELECT SUM(OCTET_LENGTH(data)) FROM bolt_allowed.t1, SUM(OCTET_LENGTH(data)), 1) +--let $filtered_rows = query_get_value(SELECT COUNT(*) FROM bolt_filtered.t1, COUNT(*), 1) +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The allowed database retains every large transaction row. +--let $assert_cond = $allowed_rows = 3 +--source include/assert.inc +--let $assert_text = The allowed database preserves the full large payload. +--let $assert_cond = $allowed_payload = 12000000 +--source include/assert.inc +--let $assert_text = The filtered database receives no rows from the transaction. +--let $assert_cond = $filtered_rows = 0 +--source include/assert.inc +--let $assert_text = The replica promotes the retained qualifying subset once. +--let $assert_cond = $replica_bolt_after = $replica_bolt_before + 1 +--source include/assert.inc +--let $assert_text = Replica-local BOLT opens a new active binary log after filtering. +--let $assert_cond = "$replica_binlog_after" != "$replica_binlog_before" +--source include/assert.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_header_file = $replica_binlog_after +--source ../../binlog/inc/validate_bolt_header.inc + +--source include/rpl/stop_replica.inc +CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (); +--source include/rpl/start_replica.inc +--source include/rpl/connection_source.inc +DROP DATABASE bolt_filtered; +DROP DATABASE bolt_allowed; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--enable_query_log +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--enable_query_log +--let $rpl_skip_sync = 1 +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_until-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica_until-slave.opt new file mode 100644 index 000000000000..4ed6f297d980 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_until-slave.opt @@ -0,0 +1,2 @@ +--log-replica-updates +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_until.test b/mysql-test/suite/rpl/t/binlog_bolt_replica_until.test new file mode 100644 index 000000000000..777001a8915b --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_until.test @@ -0,0 +1,96 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify START REPLICA UNTIL stops after a BOLT-promoted source transaction +# while the replica SQL applier also promotes its local binary log. +# +# === Requirements === +# R1. Position UNTIL applies the qualifying transaction but not a later marker. +# R2. Applying the qualifying transaction creates one replica-local BOLT file. +# +# === Implementation === +# Stop the applier initially, receive a promoted source transaction and later +# marker, then apply only through the position immediately after BOLT commit. + +--source include/have_binlog_format_row.inc +--source include/have_mta.inc +--source include/not_binlog_transaction_compression_on.inc +--let $rpl_skip_start_slave = 1 +--source include/rpl/init_source_replica.inc + +--source include/rpl/connection_source.inc +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--disable_query_log +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; + +--source include/rpl/connection_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/start_receiver.inc + +--source include/rpl/connection_source.inc +--echo # Commit the BOLT transaction, record its end position, then add a marker. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +--let $until_source_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $until_source_pos = query_get_value(SHOW BINARY LOG STATUS, Position, 1) +INSERT INTO t2 VALUES (1); +--source include/rpl/sync_to_replica_received.inc + +--source include/rpl/connection_replica.inc +--replace_result $until_source_file SOURCE_LOG_FILE $until_source_pos SOURCE_LOG_POS +--eval START REPLICA SQL_THREAD UNTIL SOURCE_LOG_FILE='$until_source_file', SOURCE_LOG_POS=$until_source_pos; +--source include/rpl/wait_for_applier_to_stop.inc +--let $replica_rows = query_get_value(SELECT COUNT(*) FROM t1, COUNT(*), 1) +--let $replica_markers = query_get_value(SELECT COUNT(*) FROM t2, COUNT(*), 1) +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = START REPLICA UNTIL applies every BOLT transaction row. +--let $assert_cond = $replica_rows = 3 +--source include/assert.inc +--let $assert_text = START REPLICA UNTIL stops before the later marker transaction. +--let $assert_cond = $replica_markers = 0 +--source include/assert.inc +--let $assert_text = The replica SQL applier promotes the transaction once before stopping. +--let $assert_cond = $replica_bolt_after = $replica_bolt_before + 1 +--source include/assert.inc +--let $assert_text = Replica-local promotion opens a new active binary log before stopping. +--let $assert_cond = "$replica_binlog_after" != "$replica_binlog_before" +--source include/assert.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_header_file = $replica_binlog_after +--source ../../binlog/inc/validate_bolt_header.inc +--let $slave_param = Exec_Source_Log_Pos +--let $slave_param_value = $until_source_pos +--source include/rpl/assert_replica_status.inc + +--source include/rpl/start_applier.inc +--source include/rpl/connection_source.inc +DROP TABLE t2; +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--enable_query_log +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--enable_query_log +--let $rpl_skip_sync = 1 +--source include/rpl/deinit.inc diff --git a/mysys/mf_cache.cc b/mysys/mf_cache.cc index e83d20addbdb..eadd7b829148 100644 --- a/mysys/mf_cache.cc +++ b/mysys/mf_cache.cc @@ -78,8 +78,17 @@ bool real_open_cached_file(IO_CACHE *cache) { DBUG_TRACE; if ((cache->file = mysql_file_create_temp( cache->file_key, name_buff, cache->dir, cache->prefix, - (O_RDWR | O_TRUNC), UNLINK_FILE, MYF(MY_WME))) >= 0) { + (O_RDWR | O_TRUNC), cache->named_file ? KEEP_FILE : UNLINK_FILE, + MYF(MY_WME))) >= 0) { error = 0; + if (cache->named_file && + (cache->file_name = my_strdup(key_memory_IO_CACHE, name_buff, + MYF(MY_WME))) == nullptr) { + (void)mysql_file_close(cache->file, MYF(0)); + (void)my_delete(name_buff, MYF(0)); + cache->file = -1; + error = 1; + } } return error; } @@ -93,6 +102,12 @@ void close_cached_file(IO_CACHE *cache) { if (file >= 0) { (void)mysql_file_close(file, MYF(0)); } + if (cache->file_name != nullptr) { + /* A named temporary file is deleted when the cache is closed. */ + (void)my_delete(cache->file_name, MYF(0)); + my_free(cache->file_name); + cache->file_name = nullptr; + } my_free(cache->dir); my_free(cache->prefix); } diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index 4afcd3052027..39d0f5304e79 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -11052,6 +11052,9 @@ ER_CSA_CRST_REQUIREMENT_GTID_ONLY ER_DA_CANNOT_REPLICATE_WITHOUT_BINLOG eng "Cannot replicate from source as it does not have logical log enabled. Check source configuration to enable it." +ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING + eng "Variable 'binlog_large_transaction_optimization_threshold' was adjusted from %llu bytes to %llu bytes to match binlog_cache_size." + # # End of "9.7 cal-ver compatibility lineage (starts from 26.7)" error messages (server-to-client). # diff --git a/share/messages_to_error_log.txt b/share/messages_to_error_log.txt index e07442f91299..8f99c473f1d1 100644 --- a/share/messages_to_error_log.txt +++ b/share/messages_to_error_log.txt @@ -13539,6 +13539,34 @@ ER_DD_DOWNGRADE ER_INVALID_SERVER_UPGRADE_SKIPS_LTS_LINEAGE eng "Invalid MySQL server upgrade: Cannot upgrade from %u to %u. Target MySQL server version belongs to a compatibility lineage whose previous LTS is %u, not %u." +ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID + eng "Failed to initialize #binlog_temp_files at '%s': it must be a directory and cannot be a symbolic link." + +ER_BINLOG_BOLT_TEMP_FILES_DIR_UNSAFE_ENTRY + eng "Failed to initialize #binlog_temp_files at '%s': found unsafe entry '%s'. Only regular managed spill files named 'bolt_' are allowed." + +ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED + eng "Failed to initialize #binlog_temp_files at '%s' (OS error %d)." + +ER_BINLOG_BOLT_TEMP_FILES_DIR_CLEANED + eng "Removed %u leftover managed binary log temporary file(s) from '%s'." + +ER_BINLOG_BOLT_LARGE_TRX_FALLBACK + eng "Could not optimize large transaction execution in the binary log because %s; standard binary logging was used instead." + +ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_SKIP + eng "Optimized binlog recovery by avoiding a sequential read of a large transaction body and seeking to its terminating event at offset %llu in binary log file '%s'." + +ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_CHECKSUM_VERIFICATION + eng "Could not optimize binlog recovery of a large transaction because source_verify_checksum is enabled." + +ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER + eng "Could not recover binary log file '%s' because it contains an invalid large transaction header." + +ER_BINLOG_BOLT_THRESHOLD_ADJUSTED + eng "Variable 'binlog_large_transaction_optimization_threshold' was adjusted from %llu bytes to %llu bytes to match binlog_cache_size." + + # # End of "9.7 cal-ver compatibility lineage (starts from 26.7)" messages intended to be written to the server error log. # diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 295ea784a2fe..6d2626b145aa 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -1070,6 +1070,8 @@ SET(BINLOG_SOURCE binlog/binlog_tc_log.cc binlog/thd_backup_and_restore.cc binlog/global.cc + binlog/large_trx_commit.cc + binlog/transaction_commit_helper.cc binlog/log_sanitizer.cc binlog/recovery.cc binlog/group_commit/bgc_ticket_manager.cc diff --git a/sql/binlog.cc b/sql/binlog.cc index 3fa5865903a7..b6ad83d1d386 100644 --- a/sql/binlog.cc +++ b/sql/binlog.cc @@ -51,6 +51,7 @@ #include #endif #include +#include #include #include #include @@ -93,9 +94,13 @@ #include "scope_guard.h" #include "sql/binlog/binlog_ofile.h" // Binlog_ofile #include "sql/binlog/binlog_tc_log.h" +#include "sql/binlog/cache_data.h" // binlog_cache_data #include "sql/binlog/decompressing_event_object_istream.h" #include "sql/binlog/global.h" #include "sql/binlog/group_commit/bgc_ticket_manager.h" // Bgc_ticket_manager +#include "sql/binlog/large_trx_commit.h" // get_cache_to_promote +#include "sql/binlog/transaction_commit_helper.h" +#include "sql/binlog/binlog_ofile.h" // MYSQL_BIN_LOG::Binlog_ofile #include "sql/binlog/recovery.h" // binlog::Binlog_recovery #include "sql/binlog/services/iterator/file_storage.h" #include "sql/binlog/thd_backup_and_restore.h" @@ -159,6 +164,37 @@ using std::max; using std::min; using std::string; +static bool is_valid_large_trx_terminating_event(const char *filename, + my_off_t target, + my_off_t file_size, + my_off_t expected_end, + uint8_t expected_type) { + if (target < BIN_LOG_HEADER_SIZE || target > file_size || + expected_end > file_size || expected_end <= target || + file_size - target < LOG_EVENT_HEADER_LEN) + return false; + + Binlog_file_reader verifier(true /*verify_checksum*/); + if (verifier.open(filename, target)) return false; + + std::unique_ptr event(verifier.read_event_object()); + if (event == nullptr || verifier.has_fatal_error()) return false; + + const auto type = event->get_type_code(); + if (expected_type != 0) { + const bool supported_type = + expected_type == mysql::binlog::event::QUERY_EVENT || + expected_type == mysql::binlog::event::XID_EVENT || + expected_type == mysql::binlog::event::XA_PREPARE_LOG_EVENT; + return supported_type && static_cast(type) == expected_type && + verifier.position() == expected_end; + } + + return (type == mysql::binlog::event::XID_EVENT || + type == mysql::binlog::event::XA_PREPARE_LOG_EVENT) && + verifier.position() == expected_end; +} + #define FLAGSTR(V, F) ((V) & (F) ? #F " " : "") #define YESNO(X) ((X) ? "yes" : "no") @@ -167,8 +203,6 @@ using std::string; @{ */ -#define MY_OFF_T_UNDEF (~(my_off_t)0UL) - /* Constants required for the limit unsafe warnings suppression */ @@ -340,413 +374,6 @@ static bool check_auto_purge_conditions() { @warning The class is not designed to be inherited from. */ -/** - Caches for non-transactional and transactional data before writing - it to the binary log. - - @todo All the access functions for the flags suggest that the - encapsuling is not done correctly, so try to move any logic that - requires access to the flags into the cache. -*/ -class binlog_cache_data { - public: - binlog_cache_data(class binlog_cache_mngr &cache_mngr, bool trx_cache_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : m_cache_mngr(cache_mngr), - m_pending(nullptr), - ptr_binlog_cache_use(ptr_binlog_cache_use_arg), - ptr_binlog_cache_disk_use(ptr_binlog_cache_disk_use_arg) { - flags.transactional = trx_cache_arg; - } - - bool open(my_off_t cache_size, my_off_t max_cache_size) { - return m_cache.open(cache_size, max_cache_size); - } - - Binlog_cache_storage *get_cache() { return &m_cache; } - int finalize(THD *thd, Log_event *end_event); - int finalize(THD *thd, Log_event *end_event, XID_STATE *xs); - int flush(THD *thd, my_off_t *bytes, bool *wrote_xid, - bool parallelization_barrier); - int write_event(Log_event *event); - void set_event_counter(size_t event_counter) { - m_event_counter = event_counter; - } - size_t get_event_counter() const { return m_event_counter; } - size_t get_compressed_size() const { return m_compressed_size; } - size_t get_decompressed_size() const { return m_decompressed_size; } - mysql::binlog::event::compression::type get_compression_type() const { - return m_compression_type; - } - - void set_compressed_size(size_t s) { m_compressed_size = s; } - void set_decompressed_size(size_t s) { m_decompressed_size = s; } - void set_compression_type(mysql::binlog::event::compression::type t) { - m_compression_type = t; - } - - virtual ~binlog_cache_data() { - assert(is_binlog_empty()); - m_cache.close(); - } - - bool is_binlog_empty() const { - DBUG_PRINT("debug", ("%s_cache - pending: 0x%llx, bytes: %llu", - (flags.transactional ? "trx" : "stmt"), - (ulonglong)pending(), (ulonglong)m_cache.length())); - return pending() == nullptr && m_cache.is_empty(); - } - - bool is_finalized() const { return flags.finalized; } - - Rows_log_event *pending() const { return m_pending; } - - void set_pending(Rows_log_event *const pending) { m_pending = pending; } - - /// @see handle_deferred_cache_write_incident - void set_incident( - std::string_view incident_message = - "Non-transactional changes were not written to the binlog."); - - /// @see handle_deferred_cache_write_incident - bool has_incident(void) const; - - bool has_xid() const { - // There should only be an XID event if we are transactional - assert((flags.transactional && flags.with_xid) || !flags.with_xid); - return flags.with_xid; - } - - bool is_trx_cache() const { return flags.transactional; } - - my_off_t get_byte_position() const { return m_cache.length(); } - - void cache_state_checkpoint(my_off_t pos_to_checkpoint) { - // We only need to store the cache state for pos > 0 - if (pos_to_checkpoint) { - cache_state state; - state.with_rbr = flags.with_rbr; - state.with_sbr = flags.with_sbr; - state.with_start = flags.with_start; - state.with_end = flags.with_end; - state.with_content = flags.with_content; - state.event_counter = m_event_counter; - cache_state_map[pos_to_checkpoint] = state; - } - } - - void cache_state_rollback(my_off_t pos_to_rollback) { - if (pos_to_rollback) { - std::map::iterator it; - it = cache_state_map.find(pos_to_rollback); - if (it != cache_state_map.end()) { - flags.with_rbr = it->second.with_rbr; - flags.with_sbr = it->second.with_sbr; - flags.with_start = it->second.with_start; - flags.with_end = it->second.with_end; - flags.with_content = it->second.with_content; - m_event_counter = it->second.event_counter; - } else - assert(it == cache_state_map.end()); - } - // Rolling back to pos == 0 means cleaning up the cache. - else { - flags.with_rbr = false; - flags.with_sbr = false; - flags.with_start = false; - flags.with_end = false; - flags.with_content = false; - m_event_counter = 0; - } - } - - /** - Reset the cache to unused state when the transaction is finished. It - drops all data in the cache and clears the flags of the transaction state. - */ - virtual void reset() { - compute_statistics(); - remove_pending_event(); - - if (m_cache.reset()) { - LogErr(WARNING_LEVEL, ER_BINLOG_CANT_RESIZE_CACHE); - } - - flags.with_xid = false; - flags.immediate = false; - flags.finalized = false; - flags.with_sbr = false; - flags.with_rbr = false; - flags.with_start = false; - flags.with_end = false; - flags.with_content = false; - - /* - The truncate function calls reinit_io_cache that calls my_b_flush_io_cache - which may increase disk_writes. This breaks the disk_writes use by the - binary log which aims to compute the ratio between in-memory cache usage - and disk cache usage. To avoid this undesirable behavior, we reset the - variable after truncating the cache. - */ - cache_state_map.clear(); - m_event_counter = 0; - m_compressed_size = 0; - m_decompressed_size = 0; - m_compression_type = mysql::binlog::event::compression::NONE; - assert(is_binlog_empty()); - } - - /** - Returns information about the cache content with respect to - the binlog_format of the events. - - This will be used to set a flag on GTID_LOG_EVENT stating that the - transaction may have SBR statements or not, but the binlog dump - will show this flag as "rbr_only" when it is not set. That's why - an empty transaction should return true below, or else an empty - transaction would be assumed as "rbr_only" even not having RBR - events. - - When dumping a binary log content using mysqlbinlog client program, - for any transaction assumed as "rbr_only" it will be printed a - statement changing the transaction isolation level to READ COMMITTED. - It doesn't make sense to have an empty transaction "requiring" this - isolation level change. - - @return true The cache have SBR events or is empty. - @return false The cache contains a transaction with no SBR events. - */ - bool may_have_sbr_stmts() { return flags.with_sbr || !flags.with_rbr; } - - /** - Check if the binlog cache contains an empty transaction, which has - two binlog events "BEGIN" and "COMMIT". - - @return true The binlog cache contains an empty transaction. - @return false Otherwise. - */ - bool has_empty_transaction() { - /* - The empty transaction has two events in trx/stmt binlog cache - and no changes: one is a transaction start and other is a transaction - end (there should be no SBR changing content and no RBR events). - */ - if (flags.with_start && // Has transaction start statement - flags.with_end && // Has transaction end statement - !flags.with_content) // Has no other content than START/END - { - assert(m_event_counter == 2); // Two events in the cache only - assert(!flags.with_sbr); // No statements changing content - assert(!flags.with_rbr); // No rows changing content - assert(!flags.immediate); // Not a DDL - assert(!flags.with_xid); // Not a XID trx and not an atomic DDL Query - return true; - } - return false; - } - - /** - Check if the binlog cache is empty or contains an empty transaction, - which has two binlog events "BEGIN" and "COMMIT". - - @return true The binlog cache is empty or contains an empty transaction. - @return false Otherwise. - */ - bool is_empty_or_has_empty_transaction() { - return is_binlog_empty() || has_empty_transaction(); - } - - protected: - /* - This structure should have all cache variables/flags that should be restored - when a ROLLBACK TO SAVEPOINT statement be executed. - */ - struct cache_state { - bool with_sbr; - bool with_rbr; - bool with_start; - bool with_end; - bool with_content; - size_t event_counter; - }; - /* - For every SAVEPOINT used, we will store a cache_state for the current - binlog cache position. So, if a ROLLBACK TO SAVEPOINT is used, we can - restore the cache_state values after truncating the binlog cache. - */ - std::map cache_state_map; - /* - In order to compute the transaction size (because of possible extra checksum - bytes), we need to keep track of how many events are in the binlog cache. - */ - size_t m_event_counter = 0; - - size_t m_compressed_size = 0; - size_t m_decompressed_size = 0; - mysql::binlog::event::compression::type m_compression_type = - mysql::binlog::event::compression::type::NONE; - /* - It truncates the cache to a certain position. This includes deleting the - pending event. It corresponds to rollback statement or rollback to - a savepoint. It doesn't change transaction state. - */ - void truncate(my_off_t pos) { - DBUG_PRINT("info", ("truncating to position %lu", (ulong)pos)); - remove_pending_event(); - - // TODO: check the return value. - (void)m_cache.truncate(pos); - } - - /** - Flush pending event to the cache buffer. - */ - int flush_pending_event(THD *thd) { - if (m_pending) { - m_pending->set_flags(Rows_log_event::STMT_END_F); - if (int error = write_event(m_pending)) return error; - thd->clear_binlog_table_maps(); - } - return 0; - } - - /** - Remove the pending event. - */ - int remove_pending_event() { - delete m_pending; - m_pending = nullptr; - return 0; - } - struct Flags { - /* - Defines if this is either a trx-cache or stmt-cache, respectively, a - transactional or non-transactional cache. - */ - bool transactional : 1; - - /* - This indicates that the cache should be written without BEGIN/END. - */ - bool immediate : 1; - - /* - This flag indicates that the buffer was finalized and has to be - flushed to disk. - */ - bool finalized : 1; - - /* - This indicates that either the cache contain an XID event, or it's - an atomic DDL Query-log-event. In the latter case the flag is set up - on the statement level, namely when the Query-log-event is cached - at time the DDL transaction is not committing. - The flag therefore gets reset when the cache is cleaned due to - the statement rollback, e.g in case of a DDL post-caching execution - error. - Any statement scope flag among other things must consider its - reset policy when the statement is rolled back. - */ - bool with_xid : 1; - - /* - This indicates that the cache contain statements changing content. - */ - bool with_sbr : 1; - - /* - This indicates that the cache contain RBR event changing content. - */ - bool with_rbr : 1; - - /* - This indicates that the cache contain s transaction start statement. - */ - bool with_start : 1; - - /* - This indicates that the cache contain a transaction end event. - */ - bool with_end : 1; - - /* - This indicates that the cache contain content other than START/END. - */ - bool with_content : 1; - } flags; - - /// Compress the current transaction "in-place", if possible - /// - /// This attempts to compress the transaction if it satisfies the - /// necessary pre-conditions. Otherwise it does nothing. - /// - /// @retval true Error: the cache has been corrupted and the - /// transaction must be aborted. - /// - /// @retval false Success: the transaction was either compressed - /// successfully, or compression was not attempted, or compression - /// failed and left the uncompressed transaction intact. - [[nodiscard]] bool compress(THD *thd); - - private: - /* - Reference to the cache_mngr which owns this cache. - */ - class binlog_cache_mngr &m_cache_mngr; - - /* - Storage for byte data. This binlog_cache_data will serialize - events into bytes and put them into m_cache. - */ - Binlog_cache_storage m_cache; - - /* - Pending binrows event. This event is the event where the rows are currently - written. - */ - Rows_log_event *m_pending; - - /** - This function computes binlog cache and disk usage. - */ - void compute_statistics() { - if (!is_binlog_empty()) { - (*ptr_binlog_cache_use)++; - if (m_cache.disk_writes() != 0) (*ptr_binlog_cache_disk_use)++; - } - } - - /* - Stores a pointer to the status variable that keeps track of the in-memory - cache usage. This corresponds to either - . binlog_cache_use or binlog_stmt_cache_use. - */ - ulong *ptr_binlog_cache_use; - - /* - Stores a pointer to the status variable that keeps track of the disk - cache usage. This corresponds to either - . binlog_cache_disk_use or binlog_stmt_cache_disk_use. - */ - ulong *ptr_binlog_cache_disk_use; - - binlog_cache_data &operator=(const binlog_cache_data &info); - binlog_cache_data(const binlog_cache_data &info); -}; - -class binlog_stmt_cache_data : public binlog_cache_data { - public: - binlog_stmt_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, - ptr_binlog_cache_disk_use_arg) {} - - using binlog_cache_data::finalize; - - int finalize(THD *thd); -}; int binlog_stmt_cache_data::finalize(THD *thd) { if (flags.immediate) { @@ -759,216 +386,8 @@ int binlog_stmt_cache_data::finalize(THD *thd) { return 0; } -class binlog_trx_cache_data : public binlog_cache_data { - public: - binlog_trx_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, - ptr_binlog_cache_disk_use_arg), - m_cannot_rollback(false), - before_stmt_pos(MY_OFF_T_UNDEF) {} - - void reset() override { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - m_cannot_rollback = false; - before_stmt_pos = MY_OFF_T_UNDEF; - binlog_cache_data::reset(); - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - bool cannot_rollback() const { return m_cannot_rollback; } - - void set_cannot_rollback() { m_cannot_rollback = true; } - - my_off_t get_prev_position() const { return before_stmt_pos; } - - void set_prev_position(my_off_t pos) { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - before_stmt_pos = pos; - cache_state_checkpoint(before_stmt_pos); - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - void restore_prev_position() { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - binlog_cache_data::truncate(before_stmt_pos); - cache_state_rollback(before_stmt_pos); - before_stmt_pos = MY_OFF_T_UNDEF; - /* - Binlog statement rollback clears with_xid now as the atomic DDL statement - marker which can be set as early as at event creation and caching. - */ - flags.with_xid = false; - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - void restore_savepoint(my_off_t pos) { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - binlog_cache_data::truncate(pos); - if (pos <= before_stmt_pos) before_stmt_pos = MY_OFF_T_UNDEF; - cache_state_rollback(pos); - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - using binlog_cache_data::truncate; - - void truncate(THD *thd, bool all); - - private: - /* - It will be set true if any statement which cannot be rolled back safely - is put in trx_cache. - */ - bool m_cannot_rollback; - /* - Binlog position before the start of the current statement. - */ - my_off_t before_stmt_pos; - - binlog_trx_cache_data &operator=(const binlog_trx_cache_data &info); - binlog_trx_cache_data(const binlog_trx_cache_data &info); -}; - -class binlog_cache_mngr { - /// Indicates that some events did not get into the cache(s) and most - /// likely it is incomplete. @see handle_deferred_cache_write_incident - std::string m_incident; - - public: -#ifndef NDEBUG - /// The number of times that the incident status has been set due to the - /// debug symbol binlog_inject_incident. - int m_injected_incident_count{0}; -#endif - - binlog_cache_mngr(ulong *ptr_binlog_stmt_cache_use_arg, - ulong *ptr_binlog_stmt_cache_disk_use_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : stmt_cache(*this, false, ptr_binlog_stmt_cache_use_arg, - ptr_binlog_stmt_cache_disk_use_arg), - trx_cache(*this, true, ptr_binlog_cache_use_arg, - ptr_binlog_cache_disk_use_arg) {} - - bool init() { - return stmt_cache.open(binlog_stmt_cache_size, - max_binlog_stmt_cache_size) || - trx_cache.open(binlog_cache_size, max_binlog_cache_size); - } - - binlog_cache_data *get_binlog_cache_data(bool is_transactional) { - if (is_transactional) - return &trx_cache; - else - return &stmt_cache; - } - - Binlog_cache_storage *get_stmt_cache() { return stmt_cache.get_cache(); } - Binlog_cache_storage *get_trx_cache() { return trx_cache.get_cache(); } - /** - Convenience method to check if both caches are empty. - */ - bool is_binlog_empty() const { - return stmt_cache.is_binlog_empty() && trx_cache.is_binlog_empty(); - } - - int handle_deferred_cache_write_incident(THD *thd); - - /// Check if either of the caches have an incident - /// @see handle_deferred_cache_write_incident - bool has_incident() const { return !m_incident.empty(); } - - void set_incident(std::string_view incident_message) { - assert(!incident_message.empty()); - m_incident = incident_message; - } - - /* - clear stmt_cache and trx_cache if they are not empty - */ - void reset() { - if (!stmt_cache.is_binlog_empty()) stmt_cache.reset(); - if (!trx_cache.is_binlog_empty()) trx_cache.reset(); - } - -#ifndef NDEBUG - bool dbug_any_finalized() const { - return stmt_cache.is_finalized() || trx_cache.is_finalized(); - } -#endif - - /* - Convenience method to flush both caches to the binary log. - - @param bytes_written Pointer to variable that will be set to the - number of bytes written for the flush. - @param wrote_xid Pointer to variable that will be set to @c - true if any XID event was written to the - binary log. Otherwise, the variable will not - be touched. - @return Error code on error, zero if no error. - */ - int flush(THD *thd, my_off_t *bytes_written, bool *wrote_xid) { - my_off_t stmt_bytes = 0; - my_off_t trx_bytes = 0; - assert(stmt_cache.has_xid() == 0); - - bool parallelization_barrier = false; - if (has_incident()) { - if (int error = handle_deferred_cache_write_incident(thd)) return error; - // Request force rotate - thd->rpl_thd_ctx.binlog_group_commit_ctx().set_force_rotate(); - // Set as parallelization_barrier so that dependency tracker marks all - // subsequent transactions to depend on it. - parallelization_barrier = true; - } - - int error = - stmt_cache.flush(thd, &stmt_bytes, wrote_xid, parallelization_barrier); - if (error) return error; - DEBUG_SYNC(thd, "after_flush_stm_cache_before_flush_trx_cache"); - error = - trx_cache.flush(thd, &trx_bytes, wrote_xid, parallelization_barrier); - if (error) return error; - *bytes_written = stmt_bytes + trx_bytes; - return 0; - } - - /** - Check if at least one of transactions and statement binlog caches - contains an empty transaction, other one is empty or contains an - empty transaction. - - @return true At least one of transactions and statement binlog - caches an empty transaction, other one is empty - or contains an empty transaction. - @return false Otherwise. - */ - bool has_empty_transaction() { - return (trx_cache.is_empty_or_has_empty_transaction() && - stmt_cache.is_empty_or_has_empty_transaction() && - !is_binlog_empty()); - } - - binlog_stmt_cache_data stmt_cache; - binlog_trx_cache_data trx_cache; - - private: - binlog_cache_mngr &operator=(const binlog_cache_mngr &info); - binlog_cache_mngr(const binlog_cache_mngr &info); -}; - -static binlog_cache_mngr *thd_get_cache_mngr(const THD *thd) { +binlog_cache_mngr *thd_get_cache_mngr(const THD *thd) { /* If opt_bin_log is not set, binlog_hton->slot == -1 and hence thd_get_ha_data(thd, hton) segfaults. @@ -1071,12 +490,33 @@ static int binlog_dummy_recover(handlerton *, XA_recover_txn *, uint, class Binlog_event_writer : public Basic_ostream { MYSQL_BIN_LOG::Binlog_ofile *m_binlog_file; bool have_checksum; + /* + Checksum algorithm the incoming cache events were serialized with, captured + once at the transaction's first event. Used to detect each event's stale + checksum (stale because log_pos changes on copy) so it can be skipped and + recomputed for the binlog. UNDEF means the events carry no checksum (e.g. + events written directly rather than from a cache). + */ + enum_binlog_checksum_alg m_checksum_trx_start = + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + + /** + Returns true when the events streaming in carry a checksum. + */ + bool is_checksum_computed() const { + return m_checksum_trx_start != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF && + m_checksum_trx_start != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF; + } ha_checksum initial_checksum; ha_checksum checksum; uint32 end_log_pos; uchar header[LOG_EVENT_HEADER_LEN]; my_off_t header_len = 0; uint32 event_len = 0; + /* Stale checksum bytes of the finished event left to skip. */ + uint32 skip_len = 0; public: /** @@ -1098,27 +538,58 @@ class Binlog_event_writer : public Basic_ostream { } void update_header() { - event_len = uint4korr(header + EVENT_LEN_OFFSET); + /* Length of the incoming event, its stale checksum included when + it carries one. */ + uint32 event_len_incoming = uint4korr(header + EVENT_LEN_OFFSET); - // Increase end_log_pos - end_log_pos += event_len; + /* Length the event will have in the binlog file. */ + uint32 event_len_on_disk = event_len_incoming; + if (have_checksum && !is_checksum_computed()) + event_len_on_disk += BINLOG_CHECKSUM_LEN; + else if (!have_checksum && is_checksum_computed()) + event_len_on_disk -= BINLOG_CHECKSUM_LEN; - // Update event length if it has checksum - if (have_checksum) { - int4store(header + EVENT_LEN_OFFSET, event_len + BINLOG_CHECKSUM_LEN); - end_log_pos += BINLOG_CHECKSUM_LEN; - } + end_log_pos += event_len_on_disk; - // Store end_log_pos + int4store(header + EVENT_LEN_OFFSET, event_len_on_disk); int4store(header + LOG_POS_OFFSET, end_log_pos); // update the checksum if (have_checksum) checksum = my_checksum(checksum, header, header_len); + + /* The event's bytes to copy; the stale checksum is not copied, it + is skipped after them (see skip_len). */ + event_len = event_len_incoming; + if (is_checksum_computed()) event_len -= BINLOG_CHECKSUM_LEN; + } + + /** + Write the computed checksum after the event, and restart the + checksum for the next event. + + @retval false Success + @retval true Error + */ + bool write_checksum() { + uchar checksum_buf[BINLOG_CHECKSUM_LEN]; + int4store(checksum_buf, checksum); + if (m_binlog_file->write(checksum_buf, BINLOG_CHECKSUM_LEN)) return true; + checksum = initial_checksum; + return false; } bool write(const unsigned char *buffer, my_off_t length) override { DBUG_TRACE; while (length > 0) { + /* Skip the previous event's stale checksum */ + if (skip_len > 0) { + uint32 skip = std::min(skip_len, length); + buffer += skip; + length -= skip; + skip_len -= skip; + continue; + } + /* Write event header into binlog */ if (event_len == 0) { /* data in the buf may be smaller than header size.*/ @@ -1150,14 +621,11 @@ class Binlog_event_writer : public Basic_ostream { length -= write_bytes; buffer += write_bytes; - // The whole event is copied, now add the checksum - if (have_checksum && event_len == 0) { - uchar checksum_buf[BINLOG_CHECKSUM_LEN]; - - int4store(checksum_buf, checksum); - if (m_binlog_file->write(checksum_buf, BINLOG_CHECKSUM_LEN)) - return true; - checksum = initial_checksum; + // The whole event is copied: write its checksum, skip its + // stale one. + if (event_len == 0) { + if (have_checksum && write_checksum()) return true; + if (is_checksum_computed()) skip_len = BINLOG_CHECKSUM_LEN; } } } @@ -1167,6 +635,16 @@ class Binlog_event_writer : public Basic_ostream { Returns true if per event checksum is enabled. */ bool is_checksum_enabled() { return have_checksum; } + + /** + Tell the writer which checksum algorithm the events about to + stream in were serialized with at their transaction's start. Set + before copying a cache (the Gtid event written before the cache + carries no checksum). + */ + void set_checksum_trx_start(enum_binlog_checksum_alg alg) { + m_checksum_trx_start = alg; + } }; /* @@ -1217,6 +695,7 @@ int binlog_cache_data::write_event(Log_event *ev) { DBUG_TRACE; if (ev != nullptr) { + if (is_trx_cache()) latch_large_trx_optimization(); DBUG_EXECUTE_IF("binlog_inject_incident", { // Set the incident status only once per session. Without this limitation, // it usually gets sets first for the transaction cache and then, when @@ -1231,6 +710,33 @@ int binlog_cache_data::write_event(Log_event *ev) { DBUG_EXECUTE_IF("simulate_disk_full_at_flush_pending", { DBUG_SET("+d,simulate_file_write_error"); }); + /* + Set the event's log_pos assuming this cache becomes the start of a new + binlog file: offset = reserved header region + bytes written so far. + - Small / non-promoted transaction: at commit the cache is copied into + the active binlog, and Binlog_event_writer overwrites log_pos with the + real destination offset. + - Large / promoted transaction: the spilled file becomes the next binlog + file with this transaction at its start, so this log_pos is already + correct and is used as is. + */ + ev->common_header->log_pos = m_cache.reserved_bytes() + m_cache.length(); + + /* + One checksum algorithm for the whole transaction, captured at its first + event, so a mid-transaction binlog_checksum change cannot mix algorithms + within one cache. Compressible transactions record + BINLOG_CHECKSUM_ALG_OFF, because events inside a compressed payload carry + no checksum. + */ + if (m_checksum_trx_start == + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF) + m_checksum_trx_start = + (ev->thd != nullptr && ev->thd->variables.binlog_trx_compression) + ? mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF + : static_cast(binlog_checksum_options); + ev->common_footer->checksum_alg = m_checksum_trx_start; + if (binary_event_serialize(ev, &m_cache)) { DBUG_EXECUTE_IF("simulate_disk_full_at_flush_pending", { DBUG_SET("-d,simulate_file_write_error"); @@ -1351,115 +857,20 @@ bool MYSQL_BIN_LOG::write_transaction(THD *thd, binlog_cache_data *cache_data, assert(thd->owned_gtid.sidno == THD::OWNED_SIDNO_ANONYMOUS || thd->owned_gtid.sidno > 0); - int64 sequence_number, last_committed; - /* Generate logical timestamps for MTS */ - m_dependency_tracker.get_dependency(thd, parallelization_barrier, - sequence_number, last_committed); - - /* - In case both the transaction cache and the statement cache are - non-empty, both will be flushed in sequence and logged as - different transactions. Then the second transaction must only - be executed after the first one has committed. Therefore, we - need to set last_committed for the second transaction equal to - last_committed for the first transaction. This is done in - binlog_cache_data::flush. binlog_cache_data::flush uses the - condition trn_ctx->last_committed==SEQ_UNINIT to detect this - situation, hence the need to set it here. - */ - thd->get_transaction()->last_committed = SEQ_UNINIT; - - /* - For delayed replication and also for the purpose of lag monitoring, - we assume that the commit timestamp of the transaction is the time of - executing this code (the time of writing the Gtid_log_event to the binary - log). - */ - ulonglong immediate_commit_timestamp = my_micro_time(); + Transaction_gtid_header metadata{thd, parallelization_barrier, + &m_dependency_tracker}; - /* - When the original_commit_timestamp session variable is set to a value - other than UNDEFINED_COMMIT_TIMESTAMP, it means that either the timestamp - is known ( > 0 ) or the timestamp is not known ( == 0 ). - */ - ulonglong original_commit_timestamp = - thd->variables.original_commit_timestamp; - /* - When original_commit_timestamp == UNDEFINED_COMMIT_TIMESTAMP, we assume - that: - a) it is not known if this thread is a slave applier ( = 0 ); - b) this is a new transaction ( = immediate_commit_timestamp); - */ - if (original_commit_timestamp == UNDEFINED_COMMIT_TIMESTAMP) { - /* - When applying a transaction using replication, assume that the - original commit timestamp is not known (the transaction wasn't - originated on the current server). - */ - if (thd->slave_thread || thd->is_binlog_applier()) { - original_commit_timestamp = 0; - } else - /* Assume that this transaction is original from this server */ - { - DBUG_EXECUTE_IF("rpl_invalid_gtid_timestamp", - // add one our to the commit timestamps - immediate_commit_timestamp += 3600000000;); - original_commit_timestamp = immediate_commit_timestamp; - } - } else { - // Clear the session variable to have cleared states for next transaction. - thd->variables.original_commit_timestamp = UNDEFINED_COMMIT_TIMESTAMP; - } - - uint32_t trx_immediate_server_version = - do_server_version_int(::server_version); - // Clear the session variable to have cleared states for next transaction. - thd->variables.immediate_server_version = UNDEFINED_SERVER_VERSION; - DBUG_EXECUTE_IF("fixed_server_version", - trx_immediate_server_version = 888888;); - DBUG_EXECUTE_IF("gr_fixed_server_version", - trx_immediate_server_version = 777777;); - - /* - When the original_server_version session variable is set to a value - other than UNDEFINED_SERVER_VERSION, it means that either the - server version is known or the server_version is not known - (UNKNOWN_SERVER_VERSION). - */ - uint32_t trx_original_server_version = thd->variables.original_server_version; - - /* - When original_server_version == UNDEFINED_SERVER_VERSION, we assume - that: - a) it is not known if this thread is a slave applier ( = 0 ); - b) this is a new transaction ( = ::server_version); - */ - if (trx_original_server_version == UNDEFINED_SERVER_VERSION) { - /* - When applying a transaction using replication, assume that the - original server version is not known (the transaction wasn't - originated on the current server). - */ - if (thd->slave_thread || thd->is_binlog_applier()) { - trx_original_server_version = UNKNOWN_SERVER_VERSION; - } else - /* Assume that this transaction is original from this server */ - { - trx_original_server_version = trx_immediate_server_version; - } - } else { - // Clear the session variable to have cleared states for next transaction. - thd->variables.original_server_version = UNDEFINED_SERVER_VERSION; - } Gtid_log_event gtid_event( - thd, cache_data->is_trx_cache(), last_committed, sequence_number, - cache_data->may_have_sbr_stmts(), original_commit_timestamp, - immediate_commit_timestamp, trx_original_server_version, - trx_immediate_server_version); + thd, cache_data->is_trx_cache(), metadata.last_committed(), + metadata.sequence_number(), cache_data->may_have_sbr_stmts(), + metadata.original_commit_timestamp(), + metadata.immediate_commit_timestamp(), metadata.original_server_version(), + metadata.immediate_server_version()); // Set the transaction length, based on cache info gtid_event.set_trx_length_by_cache_size(cache_data->get_byte_position(), writer->is_checksum_enabled(), + cache_data->is_checksum_computed(), cache_data->get_event_counter()); DBUG_PRINT("debug", ("cache_data->get_byte_position()= %llu", @@ -1476,6 +887,9 @@ bool MYSQL_BIN_LOG::write_transaction(THD *thd, binlog_cache_data *cache_data, gtid_event.write(writer)); if (ret) goto end; + /* The Gtid event above carried no checksum; the cache's events may. */ + writer->set_checksum_trx_start(cache_data->checksum_trx_start()); + /* finally write the transaction data, if it was not compressed and written as part of the gtid event already @@ -1818,6 +1232,16 @@ class Binlog_cache_compressor { DBUG_PRINT("info", ("fallback to uncompressed: may have SBR events")); return false; } + /* + Do not compress events that carry a checksum: events + inside a payload must not have one. This only happens when + binlog_transaction_compression was enabled after the + transaction's first event (see binlog_cache_data::write_event). + */ + if (m_cache.is_checksum_computed()) { + DBUG_PRINT("info", ("fallback to uncompressed: events have checksum")); + return false; + } // nothing can stop us now! return true; } @@ -1979,6 +1403,18 @@ int binlog_cache_data::finalize(THD *thd, Log_event *end_event) { if (!is_binlog_empty()) { assert(!flags.finalized); if (int error = flush_pending_event(thd)) return error; + /* + Record where the transaction's terminating event will sit in a promoted + file, so recovery can seek to it. Only a real end_event + (COMMIT / XID / XA_PREPARE) is a terminating event; immediately-logged + statements (e.g. CREATE TABLE) finalize with end_event == nullptr and + record no terminating metadata. + */ + if (end_event != nullptr) { + m_terminating_event_offset = + m_cache.reserved_bytes() + m_cache.length(); + m_terminating_event_type = end_event->get_type_code(); + } if (int error = write_event(end_event)) return error; if (int error = this->compress(thd)) return error; DBUG_PRINT("debug", ("flags.finalized: %s", YESNO(flags.finalized))); @@ -2060,6 +1496,36 @@ int binlog_cache_mngr::handle_deferred_cache_write_incident(THD *thd) { return 0; } +/* + Cache reservation only needs an estimate. The exact header size is checked + again during promotion, so a stale value merely causes fallback to standard + commit rather than an unsafe promotion. +*/ +static std::atomic binlog_temp_file_previous_gtids_size{0}; + +my_off_t get_binlog_temp_file_reserved_bytes() { + const my_off_t previous_gtids_size = + binlog_temp_file_previous_gtids_size.load(std::memory_order_relaxed); + const my_off_t required_size = + previous_gtids_size + kBinlogTempFilePreviousGtidsHeadroomBytes; + const my_off_t remainder = + required_size % kBinlogTempFileReservedBytes; + return remainder == 0 ? required_size + : required_size + kBinlogTempFileReservedBytes - + remainder; +} + +void update_binlog_temp_file_previous_gtids_size_estimate( + my_off_t previous_gtids_size) { + binlog_temp_file_previous_gtids_size.store(previous_gtids_size, + std::memory_order_relaxed); +} + +bool binlog_cache_data::open(my_off_t cache_size, my_off_t max_cache_size) { + return m_cache.open(cache_size, max_cache_size, + get_binlog_temp_file_reserved_bytes()); +} + /** Flush caches to the binary log. @@ -3496,12 +2962,19 @@ bool MYSQL_BIN_LOG::init_and_set_log_file_name(const char *log_name, @param new_index_number The binary log file index number to start from after the RESET BINARY LOGS AND GTIDS command is called. + @param existing When true, the file already exists on disk with + its contents in place (a promoted BOLT temp file): + it is opened for append (positioned at its end) + rather than created fresh, and no new file header + or encryption header is written. False for a + normal, newly created log file. @return true if error, false otherwise. */ bool MYSQL_BIN_LOG::open(PSI_file_key log_file_key, const char *log_name, - const char *new_name, uint32 new_index_number) { + const char *new_name, uint32 new_index_number, + bool existing) { DBUG_TRACE; bool ret = false; @@ -3529,7 +3002,19 @@ bool MYSQL_BIN_LOG::open(PSI_file_key log_file_key, const char *log_name, */ if (!is_relay_log) mysql_mutex_lock(&LOCK_sync); - ret = m_binlog_file->open(log_file_key, log_file_name, flags); + ret = m_binlog_file->open(log_file_key, log_file_name, flags, existing); + + /* + A promoted binary log file already contains its header events and the + transaction; position at its end, so subsequent transactions append. + */ + if (!ret && existing) { + MY_STAT info; + if (mysql_file_stat(log_file_key, log_file_name, &info, MYF(MY_WME)) == + nullptr || + m_binlog_file->position_at(info.st_size)) + ret = true; + } if (!is_relay_log) mysql_mutex_unlock(&LOCK_sync); @@ -3891,6 +3376,8 @@ static enum_read_gtids_from_binlog_status read_gtids_from_binlog( enum_read_gtids_from_binlog_status ret = NO_GTIDS; bool done = false; bool seen_first_gtid = false; + my_off_t large_trx_end_offset = 0; + uint8_t large_trx_terminating_event_type = 0; while (!done && (ev = binlog_file_reader.read_event_object()) != nullptr) { #ifndef NDEBUG event_counter++; @@ -3901,6 +3388,23 @@ static enum_read_gtids_from_binlog_status read_gtids_from_binlog( case mysql::binlog::event::ROTATE_EVENT: // do nothing; just accept this event and go to next break; + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: { + /* + Large transaction optimization: record the offset and type of the + transaction's terminating event. After the Gtid event that follows + is read, the scan seeks past the transaction body straight to that + offset (see the seek logic further below). The offset is only + meaningful in a binary log; in a relay log it refers to the source's + file, so skip it there. + */ + if (!is_relay_log) { + const auto <h = + static_cast(*ev); + large_trx_end_offset = lth.get_terminating_event_offset(); + large_trx_terminating_event_type = lth.get_terminating_event_type(); + } + break; + } case mysql::binlog::event::PREVIOUS_GTIDS_LOG_EVENT: { ret = GOT_PREVIOUS_GTIDS; // add events to sets @@ -4030,7 +3534,50 @@ static enum_read_gtids_from_binlog_status read_gtids_from_binlog( if (ret == GOT_PREVIOUS_GTIDS && is_relay_log) done = true; break; } + const bool is_large_trx_header = + ev->get_type_code() == + mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT; + const bool is_gtid_event = + ev->get_type_code() == mysql::binlog::event::GTID_LOG_EVENT || + ev->get_type_code() == mysql::binlog::event::GTID_TAGGED_LOG_EVENT; + const my_off_t gtid_start = + is_gtid_event ? binlog_file_reader.event_start_pos() : 0; + const ulonglong trx_length = + is_gtid_event ? static_cast(ev)->get_trx_length() + : 0; delete ev; + /* + The header's offset is usable only after its GTID event. Verify that it + resolves to the exact end of that transaction before seeking; otherwise + discard the hint and continue scanning sequentially. + */ + if (large_trx_end_offset != 0 && is_gtid_event && !done) { + const my_off_t target = large_trx_end_offset; + const uint8_t expected_type = large_trx_terminating_event_type; + large_trx_end_offset = 0; + large_trx_terminating_event_type = 0; + const bool trx_length_fits = + trx_length <= static_cast( + std::numeric_limits::max() - + gtid_start); + if (trx_length_fits) { + const my_off_t expected_end = + gtid_start + static_cast(trx_length); + // ERROR here is from seek() failing (returns true on error), not from + // the event being valid; the seek only runs once the event is valid. + if (target > binlog_file_reader.position() && + is_valid_large_trx_terminating_event( + filename, target, binlog_file_reader.ifile()->length(), + expected_end, expected_type) && + binlog_file_reader.seek(target)) { + ret = ERROR; + done = true; + } + } + } else if (large_trx_end_offset != 0 && !is_large_trx_header) { + large_trx_end_offset = 0; + large_trx_terminating_event_type = 0; + } DBUG_PRINT("info", ("done=%d", done)); } @@ -4492,15 +4039,21 @@ bool MYSQL_BIN_LOG::open_binlog( const char *log_name, const char *new_name, ulong max_size_arg, bool null_created_arg, bool need_lock_index, bool need_tsid_lock, Format_description_log_event *extra_description_event, - uint32 new_index_number) { + uint32 new_index_number, const char *promoted_log_name, + bool promoted_file_is_renamed) { // lock_index must be acquired *before* tsid_lock. assert(need_tsid_lock || !need_lock_index); + assert(!promoted_file_is_renamed || promoted_log_name != nullptr); DBUG_TRACE; DBUG_PRINT("enter", ("base filename: %s", log_name)); mysql_mutex_assert_owner(get_log_lock()); if (init_and_set_log_file_name(log_name, new_name, new_index_number)) { + if (promoted_file_is_renamed) { + (void)purge_index_entry(nullptr, nullptr, need_lock_index); + m_binlog_index_monitor.close_purge_index_file(); + } LogErr(ERROR_LEVEL, ER_BINLOG_CANT_GENERATE_NEW_FILE_NAME); return true; } @@ -4509,10 +4062,11 @@ bool MYSQL_BIN_LOG::open_binlog( DEBUG_SYNC(current_thd, "after_log_file_name_initialized"); - if (m_binlog_index_monitor.open_purge_index_file(true) || - m_binlog_index_monitor.register_create_index_entry(log_file_name) || - m_binlog_index_monitor.sync_purge_index_file() || - DBUG_EVALUATE_IF("fault_injection_registering_index", 1, 0)) { + if (!promoted_file_is_renamed && + (m_binlog_index_monitor.open_purge_index_file(true) || + m_binlog_index_monitor.register_create_index_entry(log_file_name) || + m_binlog_index_monitor.sync_purge_index_file() || + DBUG_EVALUATE_IF("fault_injection_registering_index", 1, 0))) { /** @todo: although this was introduced to appease valgrind when injecting emulated faults using fault_injection_registering_index @@ -4528,14 +4082,40 @@ bool MYSQL_BIN_LOG::open_binlog( LogErr(ERROR_LEVEL, ER_BINLOG_FAILED_TO_SYNC_INDEX_FILE_IN_OPEN); return true; } + if (promoted_log_name != nullptr && !promoted_file_is_renamed) + DBUG_EXECUTE_IF("crash_bolt_after_purge_index_sync", DBUG_SUICIDE();); + if (promoted_file_is_renamed) + assert(m_binlog_index_monitor.is_inited_purge_index_file()); DBUG_EXECUTE_IF("crash_create_non_critical_before_update_index", DBUG_SUICIDE();); write_error = false; + bool promoted_file_renamed = promoted_file_is_renamed; + + /* + Promote a spilled large transaction's temporary file into the binary + log sequence. The BOLT path registers and renames the file before its + Rotate event; legacy callers do both operations here. + */ + if (promoted_log_name != nullptr && !promoted_file_is_renamed) { + assert(!is_relay_log); + if (my_rename(promoted_log_name, log_file_name, MYF(MY_WME))) { + LogErr(ERROR_LEVEL, ER_BINLOG_CANT_USE_FOR_LOGGING, log_file_name, + errno); + m_binlog_index_monitor.close_purge_index_file(); + return true; + } + promoted_file_renamed = true; + DBUG_EXECUTE_IF("crash_bolt_after_promote_rename", DBUG_SUICIDE();); + } /* open the main log file */ - if (open(m_key_file_log, log_name, new_name, new_index_number)) { + if (open(m_key_file_log, log_name, new_name, new_index_number, + promoted_log_name != nullptr /*existing*/)) { + if (m_binlog_index_monitor.is_inited_purge_index_file()) + (void)purge_index_entry(nullptr, nullptr, need_lock_index); m_binlog_index_monitor.close_purge_index_file(); + if (promoted_file_renamed) my_delete(log_file_name, MYF(0)); return true; /* all warnings issued */ } @@ -4545,6 +4125,16 @@ bool MYSQL_BIN_LOG::open_binlog( Format_description_log_event s; + /* + A promoted binary log file already carries its header events (magic, + Format_description, Previous_gtids, Large_transaction_header, Gtid); + skip writing them and just register the file in the index. + */ + if (promoted_log_name != nullptr) { + write_file_name_to_index_file = true; + goto promoted_add_to_index; + } + if (m_binlog_file->is_empty()) { /* The binary log file was empty (probably newly created) @@ -4626,6 +4216,9 @@ bool MYSQL_BIN_LOG::open_binlog( if (is_relay_log) prev_gtids_ev.set_relay_log_event(); if (need_tsid_lock) tsid_lock->unlock(); if (write_event_to_binlog(&prev_gtids_ev)) goto err; + if (!is_relay_log) + update_binlog_temp_file_previous_gtids_size_estimate( + prev_gtids_ev.common_header->data_written); } else // !(current_thd) { /* @@ -4695,6 +4288,7 @@ bool MYSQL_BIN_LOG::open_binlog( goto err; bytes_written += extra_description_event->common_header->data_written; } +promoted_add_to_index: if (m_binlog_file->flush_and_sync()) goto err; if (write_file_name_to_index_file) { @@ -4721,16 +4315,27 @@ bool MYSQL_BIN_LOG::open_binlog( } DBUG_EXECUTE_IF("crash_create_after_update_index", DBUG_SUICIDE();); + if (promoted_log_name != nullptr) + DBUG_EXECUTE_IF("crash_bolt_after_main_index_update", { + (void)ha_flush_logs(true); + DBUG_SUICIDE(); + }); } atomic_log_state = LOG_OPENED; /* At every rotate memorize the last transaction counter state to use it as - offset at logging the transaction logical timestamps. + offset at logging the transaction logical timestamps. For a promoted + file the tracker was rotated before its Gtid event was written. */ - m_dependency_tracker.rotate(); + if (promoted_log_name == nullptr) m_dependency_tracker.rotate(); m_binlog_index_monitor.close_purge_index_file(); + if (promoted_log_name != nullptr) + DBUG_EXECUTE_IF("crash_bolt_after_purge_index_remove", { + (void)ha_flush_logs(true); + DBUG_SUICIDE(); + }); update_binlog_end_pos(); return false; @@ -4748,6 +4353,8 @@ bool MYSQL_BIN_LOG::open_binlog( LogErr(ERROR_LEVEL, ER_BINLOG_CANT_USE_FOR_LOGGING, (new_name) ? new_name : name, errno); close(LOG_CLOSE_INDEX, false, need_lock_index); + /* Undo the promotion rename before the caller relinquishes ownership. */ + if (promoted_file_renamed) my_delete(log_file_name, MYF(0)); } return true; } @@ -5372,6 +4979,32 @@ int MYSQL_BIN_LOG::new_file_without_locking( return new_file_impl(false /*need_lock_log=false*/, extra_description_event); } +int MYSQL_BIN_LOG::persist_gtids_on_rotate(bool use_dedicated_thd, + bool *keep_current_binlog) { + assert(keep_current_binlog != nullptr); + mysql_mutex_assert_owner(&LOCK_log); + m_binlog_index_monitor.assert_owner(); + + *keep_current_binlog = false; + const bool use_client_thd_for_debug_readonly = + DBUG_EVALUATE_IF("gtid_executed_readonly", true, false); + THD *saved_current_thd = nullptr; + if (use_dedicated_thd && !use_client_thd_for_debug_readonly) { + saved_current_thd = current_thd; + current_thd = nullptr; + } + + const int error = gtid_state->save_gtids_of_last_binlog_into_table(); + if (saved_current_thd != nullptr) saved_current_thd->store_globals(); + + if (error == ER_RPL_GTID_TABLE_CANNOT_OPEN) { + *keep_current_binlog = + m_binlog_file->get_real_file_size() < static_cast(max_size) && + !DBUG_EVALUATE_IF("simulate_max_binlog_size", true, false); + } + return error; +} + /** Start writing to a new log file or reopen the old file. @@ -5430,15 +5063,14 @@ int MYSQL_BIN_LOG::new_file_impl( } if (!is_relay_log) { - /* Save set of GTIDs of the last binlog into table on binlog rotation */ - if ((error = gtid_state->save_gtids_of_last_binlog_into_table())) { + /* Save the last binlog's GTID set before rotation. */ + bool keep_current_binlog = false; + if ((error = persist_gtids_on_rotate(false /*use_dedicated_thd*/, + &keep_current_binlog))) { if (error == ER_RPL_GTID_TABLE_CANNOT_OPEN) { - close_on_error = - m_binlog_file->get_real_file_size() >= - static_cast(max_size) || - DBUG_EVALUATE_IF("simulate_max_binlog_size", true, false); + close_on_error = !keep_current_binlog; - if (!close_on_error) { + if (keep_current_binlog) { LogErr(ERROR_LEVEL, ER_BINLOG_UNABLE_TO_ROTATE_GTID_TABLE_READONLY, "Current binlog file was flushed to disk and will be kept in " "use."); @@ -6762,7 +6394,11 @@ void MYSQL_BIN_LOG::close() {} */ int MYSQL_BIN_LOG::prepare(THD *thd, bool all) { DBUG_TRACE; - return m_tc_log_processing->prepare(this, thd, all); + DBUG_EXECUTE_IF("crash_bolt_before_tc_prepare", DBUG_SUICIDE();); + const int error = m_tc_log_processing->prepare(this, thd, all); + if (!error) + DBUG_EXECUTE_IF("crash_bolt_after_tc_prepare", DBUG_SUICIDE();); + return error; } /** @@ -7034,14 +6670,22 @@ TC_LOG::enum_result MYSQL_BIN_LOG::commit(THD *thd, bool all) { return RESULT_ABORTED; } + binlog_cache_data *cache_to_promote = get_cache_to_promote(cache_mngr); + const bool used_bolt_promotion = cache_to_promote != nullptr; if (DBUG_EVALUATE_IF("simulate_xa_commit_log_inconsistency", true, false) || - ordered_commit(thd, all, skip_commit)) { + (used_bolt_promotion + ? commit_large_transaction(thd, all, skip_commit, + cache_to_promote) + : ordered_commit(thd, all, skip_commit))) { thd_get_cache_mngr(thd)->reset(); if (thd->get_stmt_da()->is_ok()) thd->get_stmt_da()->reset_diagnostics_area(); return RESULT_INCONSISTENT; } + if (used_bolt_promotion) + DBUG_EXECUTE_IF("crash_bolt_after_binlog_commit", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("ensure_binlog_cache_is_reset", { /* Assert that binlog cache is reset at commit time. */ assert(binlog_cache_is_reset); @@ -7107,41 +6751,6 @@ void MYSQL_BIN_LOG::reset_thread_caches(THD *thd) { return cache_mngr->reset(); } -void MYSQL_BIN_LOG::init_thd_variables(THD *thd, bool all, bool skip_commit) { - /* - These values are used while committing a transaction, so clear - everything. - - Notes: - - - It would be good if we could keep transaction coordinator - log-specific data out of the THD structure, but that is not the - case right now. - - - Everything in the transaction structure is reset when calling - ha_commit_low since that calls Transaction_ctx::cleanup. - */ - thd->tx_commit_pending = true; - thd->commit_error = THD::CE_NONE; - thd->next_to_commit = nullptr; - thd->durability_property = HA_IGNORE_DURABILITY; - thd->get_transaction()->m_flags.real_commit = all; - thd->get_transaction()->m_flags.xid_written = false; - thd->get_transaction()->m_flags.commit_low = !skip_commit; - thd->get_transaction()->m_flags.run_hooks = !skip_commit; -#ifndef NDEBUG - /* - The group commit Leader may have to wait for follower whose transaction - is not ready to be preempted. Initially the status is pessimistic. - Preemption guarding logics is necessary only when !NDEBUG is set. - It won't be required for the dbug-off case as long as the follower won't - execute any thread-specific write access code in this method, which is - the case as of current. - */ - thd->get_transaction()->m_flags.ready_preempt = false; -#endif -} - /** Commit a sequence of sessions. @@ -8068,10 +7677,10 @@ bool THD::binlog_configure_trx_cache_size(ulong new_size) { return true; } - // Close and reopen with new value - Binlog_cache_storage *const cache = cache_mngr->get_trx_cache(); - cache->close(); - return cache->open(new_size, max_binlog_cache_size); + // Close and reopen with new value and a fresh Previous_gtids snapshot. + binlog_cache_data *const cache_data = &cache_mngr->trx_cache; + cache_data->get_cache()->close(); + return cache_data->open(new_size, max_binlog_cache_size); } /** diff --git a/sql/binlog.h b/sql/binlog.h index 37885fa582fd..ca50022beac5 100644 --- a/sql/binlog.h +++ b/sql/binlog.h @@ -67,9 +67,12 @@ class Tsid_map; class THD; class Transaction_boundary_parser; class binlog_cache_data; +class binlog_cache_mngr; class user_var_entry; class Binlog_cache_storage; +binlog_cache_mngr *thd_get_cache_mngr(const THD *thd); + struct Gtid; typedef int64 query_id_t; @@ -249,8 +252,49 @@ class MYSQL_BIN_LOG : public TC_LOG { int new_file_impl(bool need_lock, Format_description_log_event *extra_description_event); + /** + Persist the current binary log's GTIDs before starting a new binary log. + The caller must hold LOCK_log and the binlog-index lock. + + @param use_dedicated_thd Force GTID-table persistence to use a temporary + THD instead of current_thd. + @param[out] keep_current_binlog Set when a read-only GTID table permits + continuing with the current log. + @return 0 on success, otherwise the GTID persistence error. + */ + int persist_gtids_on_rotate(bool use_dedicated_thd, + bool *keep_current_binlog); + bool open(PSI_file_key log_file_key, const char *log_name, - const char *new_name, uint32 new_index_number); + const char *new_name, uint32 new_index_number, + bool existing = false); + + /** + Writes the file header of a promoted binary log file into the reserved + region at the head of a spilled temporary file: the binlog magic, a + Format_description event, a Previous_gtids event, a + Large_transaction_header event sized to fill the region exactly, and + the transaction's Gtid event, which ends precisely where the + transaction's first event was placed at spill time. + + Rotates the dependency tracker (the promoted file starts a new binlog + file) and assigns the transaction's GTID and logical timestamps, but + only after the reserved region is known to fit the header events. + Called with LOCK_log held. + + @param thd The committing session. + @param cache_data The transaction's (spilled) binlog cache. + @param file Descriptor of the temporary file. + @param[out] fits Set to false when the reserved region cannot + fit the header events, in which case nothing + was written or assigned and the caller must + fall back; set to true otherwise. + + @retval false Success. + @retval true Error (only when *fits is true). + */ + bool write_promoted_binlog_header(THD *thd, binlog_cache_data *cache_data, + File file, bool *fits); bool init_and_set_log_file_name(const char *log_name, const char *new_name, uint32 new_index_number); int generate_new_name(char *new_name, const char *log_name, @@ -511,19 +555,6 @@ class MYSQL_BIN_LOG : public TC_LOG { bool change_stage(THD *thd, Commit_stage_manager::StageID stage, THD *queue, mysql_mutex_t *leave_mutex, mysql_mutex_t *enter_mutex); - /** - Set thread variables used while flushing a transaction. - - @param[in] thd thread whose variables need to be set - @param[in] all This is @c true if this is a real transaction commit, and - @c false otherwise. - @param[in] skip_commit - This is @c true if the call to @c ha_commit_low should - be skipped (it is handled by the caller somehow) and @c - false otherwise (the normal case). - */ - void init_thd_variables(THD *thd, bool all, bool skip_commit); - [[nodiscard]] int flush_cache_to_file(my_off_t *flush_end_pos); [[nodiscard]] std::pair flush_thread_caches(THD *thd); void handle_binlog_flush_or_sync_error(THD *thd, bool need_lock_log, @@ -716,12 +747,22 @@ class MYSQL_BIN_LOG : public TC_LOG { binary log files. @param new_index_number The binary log file index number to start from after the RESET BINARY LOGS AND GTIDS command is called. + @param promoted_log_name When a spilled large transaction commits by + promoting its temporary file into the binary log sequence, the name of + that file. It is opened at its end instead of a fresh file being created + because its header events are already in place. This should be NULL + otherwise. + @param promoted_file_is_renamed True when the caller has already renamed + and registered the promoted file in the durable purge index. This keeps + the recovery record in place until the main-index update completes. */ bool open_binlog(const char *log_name, const char *new_name, ulong max_size_arg, bool null_created_arg, bool need_lock_index, bool need_tsid_lock, Format_description_log_event *extra_description_event, - uint32 new_index_number = 0); + uint32 new_index_number = 0, + const char *promoted_log_name = nullptr, + bool promoted_file_is_renamed = false); bool open_index_file(const char *index_file_name_arg, const char *log_name, bool need_lock_index); /* Use this to start writing a new log file */ @@ -742,6 +783,32 @@ class MYSQL_BIN_LOG : public TC_LOG { Binlog_event_writer *writer, bool parallelization_barrier); + /** + Commit a large transaction by promoting its spilled temporary file into + the binary log sequence: the file header events are written into the + reserved region at the head of the file, the file is renamed to become + the next binary log file, and the transaction is committed in the + engines. The commit work is constant regardless of the transaction + size: the transaction body, already in final binary log form in the + file, is never copied. + + If the reserved region cannot fit the header events, the transaction + falls back to ordered_commit(). + + @param thd The committing session. + @param all Is set in case of explicit commit + (COMMIT statement), or implicit commit issued by + a DDL. + @param skip_commit Is set in case of XA PREPARE, in which case the + commit in the engines is skipped. + @param cache_data The transaction's (spilled) binlog cache. + + @retval 0 Success. + @retval !=0 Error. + */ + int commit_large_transaction(THD *thd, bool all, bool skip_commit, + binlog_cache_data *cache_data); + /** Write a dml into statement cache and then flush it into binlog. It writes Gtid_log_event and BEGIN, COMMIT automatically. @@ -828,6 +895,7 @@ class MYSQL_BIN_LOG : public TC_LOG { int remove_logs_outside_range_from_index(const std::string &first, const std::string &last); int rotate(bool force_rotate, bool *check_purge); + int rotate_if_needed(); /** @brief This function runs automatic purge if the conditions to meet @@ -1036,6 +1104,16 @@ struct LOAD_FILE_INFO { extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log; +/** + Return the lock-free reservation based on the most recently serialized + Previous_gtids event size. +*/ +my_off_t get_binlog_temp_file_reserved_bytes(); + +/** Publish a newly serialized Previous_gtids event size for cache opens. */ +void update_binlog_temp_file_previous_gtids_size_estimate( + my_off_t previous_gtids_size); + /** Check if the the transaction is empty. diff --git a/sql/binlog/binlog_ofile.cc b/sql/binlog/binlog_ofile.cc index d76a426e8991..cfc69a5e4204 100644 --- a/sql/binlog/binlog_ofile.cc +++ b/sql/binlog/binlog_ofile.cc @@ -153,6 +153,21 @@ bool MYSQL_BIN_LOG::Binlog_ofile::truncate(my_off_t offset) { return false; } +bool MYSQL_BIN_LOG::Binlog_ofile::position_at(my_off_t offset) { + assert(m_pipeline_head != nullptr); + /* + The caller passes a physical file offset (e.g. the promoted file's size) as + the logical position. That equality only holds for an unencrypted file; an + encrypted file's physical size includes the encryption header. The only + caller (opening a promoted binary log file) never encrypts, so require it. + */ + assert(!is_encrypted()); + + if (m_pipeline_head->seek(offset)) return true; + m_position = offset; + return false; +} + bool MYSQL_BIN_LOG::Binlog_ofile::flush() { return m_pipeline_head->flush(); } bool MYSQL_BIN_LOG::Binlog_ofile::sync() { return m_pipeline_head->sync(); } bool MYSQL_BIN_LOG::Binlog_ofile::flush_and_sync() { return flush() || sync(); } diff --git a/sql/binlog/binlog_ofile.h b/sql/binlog/binlog_ofile.h index f0680bb19970..cb0775d11e6d 100644 --- a/sql/binlog/binlog_ofile.h +++ b/sql/binlog/binlog_ofile.h @@ -109,6 +109,16 @@ class MYSQL_BIN_LOG::Binlog_ofile : public Basic_ostream { */ [[nodiscard]] virtual bool truncate(my_off_t offset); + /** + Seeks to an existing binlog offset so the next write appends there. + + @param[in] offset Logical offset for the next write. + + @retval false Success + @retval true Error + */ + [[nodiscard]] virtual bool position_at(my_off_t offset); + [[nodiscard]] virtual bool flush(); [[nodiscard]] virtual bool sync(); [[nodiscard]] virtual bool flush_and_sync(); diff --git a/sql/binlog/binlog_tc_log.cc b/sql/binlog/binlog_tc_log.cc index 3b0990d16e2e..cccd561855c8 100644 --- a/sql/binlog/binlog_tc_log.cc +++ b/sql/binlog/binlog_tc_log.cc @@ -25,7 +25,9 @@ #include "mysql/components/services/log_builtins.h" #include "sql/binlog.h" #include "sql/binlog/binlog_ofile.h" +#include "sql/binlog/cache_data.h" #include "sql/binlog/group_commit/bgc_ticket_manager.h" +#include "sql/binlog/large_trx_commit.h" #include "sql/binlog/thd_backup_and_restore.h" #include "sql/clone_handler.h" #include "sql/debug_sync.h" @@ -46,16 +48,25 @@ int Binlog_tc_log::prepare(MYSQL_BIN_LOG *binlog, THD *thd, bool all) { assert(opt_bin_log); + binlog_cache_mngr *const cache_mngr = thd_get_cache_mngr(thd); + const bool large_trx_promotion = + cache_mngr != nullptr && is_large_trx_promotion_eligible(cache_mngr); + /* - Set HA_IGNORE_DURABILITY to not flush the prepared record of the - transaction to the log of storage engine (for example, InnoDB - redo log) during the prepare phase. So that we can flush prepared - records of transactions to the log of storage engine in a group - right before flushing them to binary log during binlog group - commit flush stage. Reset to HA_REGULAR_DURABILITY at the - beginning of parsing next command. + Non-BOLT transactions use HA_IGNORE_DURABILITY so the prepared record is + not flushed to the storage engine log (e.g. InnoDB redo) during prepare. + Instead, prepared records are flushed to the engine log in a group right + before flushing them to the binary log during the binlog group commit + flush stage. (Reset to HA_REGULAR_DURABILITY at the start of parsing the + next command.) + + BOLT bypasses that group-commit flush stage, so there is no group flush to + make the prepared engine state durable before the binary log decision. A + qualifying BOLT transaction therefore prepares with HA_REGULAR_DURABILITY + so its prepared record is durable on its own. */ - thd->durability_property = HA_IGNORE_DURABILITY; + thd->durability_property = + large_trx_promotion ? HA_REGULAR_DURABILITY : HA_IGNORE_DURABILITY; CONDITIONAL_SYNC_POINT_FOR_TIMESTAMP("before_prepare_in_engines"); diff --git a/sql/binlog/cache_data.h b/sql/binlog/cache_data.h new file mode 100644 index 000000000000..fa9df865ad7a --- /dev/null +++ b/sql/binlog/cache_data.h @@ -0,0 +1,722 @@ +#ifndef BINLOG_CACHE_DATA_H_INCLUDED +#define BINLOG_CACHE_DATA_H_INCLUDED + +#include +#include +#include + +#include "my_dbug.h" +#include "my_inttypes.h" +#include "my_sys.h" +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/components/services/log_builtins.h" +#include "mysqld_error.h" +#include "sql/binlog_ostream.h" +#include "sql/debug_sync.h" +#include "sql/log_event.h" +#include "sql/mysqld.h" // binlog_cache_size +#include "sql/sql_class.h" +#include "sql/xa.h" + +#define MY_OFF_T_UNDEF (~(my_off_t)0UL) + +/** + Caches for non-transactional and transactional data before writing + it to the binary log. + + @todo All the access functions for the flags suggest that the + encapsuling is not done correctly, so try to move any logic that + requires access to the flags into the cache. +*/ +class binlog_cache_data { + public: + binlog_cache_data(class binlog_cache_mngr &cache_mngr, bool trx_cache_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : m_cache_mngr(cache_mngr), + m_pending(nullptr), + ptr_binlog_cache_use(ptr_binlog_cache_use_arg), + ptr_binlog_cache_disk_use(ptr_binlog_cache_disk_use_arg) { + flags.transactional = trx_cache_arg; + } + + bool open(my_off_t cache_size, my_off_t max_cache_size); + + Binlog_cache_storage *get_cache() { return &m_cache; } + int finalize(THD *thd, Log_event *end_event); + int finalize(THD *thd, Log_event *end_event, XID_STATE *xs); + int flush(THD *thd, my_off_t *bytes, bool *wrote_xid, + bool parallelization_barrier); + int write_event(Log_event *event); + void set_event_counter(size_t event_counter) { + m_event_counter = event_counter; + } + size_t get_event_counter() const { return m_event_counter; } + size_t get_compressed_size() const { return m_compressed_size; } + size_t get_decompressed_size() const { return m_decompressed_size; } + mysql::binlog::event::compression::type get_compression_type() const { + return m_compression_type; + } + + void set_compressed_size(size_t s) { m_compressed_size = s; } + void set_decompressed_size(size_t s) { m_decompressed_size = s; } + void set_compression_type(mysql::binlog::event::compression::type t) { + m_compression_type = t; + } + + virtual ~binlog_cache_data() { + assert(is_binlog_empty()); + m_cache.close(); + } + + bool is_binlog_empty() const { + DBUG_PRINT("debug", ("%s_cache - pending: 0x%llx, bytes: %llu", + (flags.transactional ? "trx" : "stmt"), + (ulonglong)pending(), (ulonglong)m_cache.length())); + return pending() == nullptr && m_cache.is_empty(); + } + + bool is_finalized() const { return flags.finalized; } + + Rows_log_event *pending() const { return m_pending; } + + void set_pending(Rows_log_event *const pending) { m_pending = pending; } + + /// @see handle_deferred_cache_write_incident + void set_incident( + std::string_view incident_message = + "Non-transactional changes were not written to the binlog."); + + /// @see handle_deferred_cache_write_incident + bool has_incident(void) const; + + bool has_xid() const { + // There should only be an XID event if we are transactional + assert((flags.transactional && flags.with_xid) || !flags.with_xid); + return flags.with_xid; + } + + bool is_trx_cache() const { return flags.transactional; } + + /** + Returns the checksum algorithm the events of this cache are + serialized with, recorded at the transaction's first event (see + write_event). + */ + mysql::binlog::event::enum_binlog_checksum_alg checksum_trx_start() const { + return m_checksum_trx_start; + } + + /** + Returns true when the events in this cache carry a checksum + (see write_event). + */ + bool is_checksum_computed() const { + return m_checksum_trx_start != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF && + m_checksum_trx_start != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF; + } + + void latch_large_trx_optimization() { + if (!m_large_trx_optimization_latched) { + m_large_trx_optimization_enabled = + opt_binlog_large_transaction_optimization_enabled; + m_large_trx_optimization_threshold = + opt_binlog_large_transaction_optimization_threshold; + m_large_trx_optimization_latched = true; + /* + Give the spill file a promotable (named) form only when the + optimization is enabled for this transaction, so a disabled knob leaves + no visible bolt_ files. This uses the same knob value captured here for + the promotion decision, so naming and eligibility always agree. The + reserved header region is applied regardless (see + IO_CACHE_binlog_cache_storage::open); only the naming is gated. + */ + m_cache.set_named_file(m_large_trx_optimization_enabled); + } + } + + bool large_trx_optimization_enabled() const { + return m_large_trx_optimization_enabled; + } + + ulonglong large_trx_optimization_threshold() const { + return m_large_trx_optimization_threshold; + } + + /** + Returns the offset and type of the transaction's terminating event in a + promoted binary log file. Both values are recorded by finalize(). + */ + my_off_t terminating_event_offset() const { + return m_terminating_event_offset; + } + mysql::binlog::event::Log_event_type terminating_event_type() const { + return m_terminating_event_type; + } + + my_off_t get_byte_position() const { return m_cache.length(); } + + void cache_state_checkpoint(my_off_t pos_to_checkpoint) { + // We only need to store the cache state for pos > 0 + if (pos_to_checkpoint) { + cache_state state; + state.with_rbr = flags.with_rbr; + state.with_sbr = flags.with_sbr; + state.with_start = flags.with_start; + state.with_end = flags.with_end; + state.with_content = flags.with_content; + state.event_counter = m_event_counter; + cache_state_map[pos_to_checkpoint] = state; + } + } + + void cache_state_rollback(my_off_t pos_to_rollback) { + if (pos_to_rollback) { + std::map::iterator it; + it = cache_state_map.find(pos_to_rollback); + if (it != cache_state_map.end()) { + flags.with_rbr = it->second.with_rbr; + flags.with_sbr = it->second.with_sbr; + flags.with_start = it->second.with_start; + flags.with_end = it->second.with_end; + flags.with_content = it->second.with_content; + m_event_counter = it->second.event_counter; + } else + assert(it == cache_state_map.end()); + } + // Rolling back to pos == 0 means cleaning up the cache. + else { + flags.with_rbr = false; + flags.with_sbr = false; + flags.with_start = false; + flags.with_end = false; + flags.with_content = false; + m_event_counter = 0; + } + } + + /** + Reset the cache to unused state when the transaction is finished. It + drops all data and clears the transaction flags. If the caller has + promoted a spilled file, preserve_spilled_file retains that file while + resetting the cache. + */ + virtual void reset(bool preserve_spilled_file = false) { + compute_statistics(); + remove_pending_event(); + + if (m_cache.reset(preserve_spilled_file)) { + LogErr(WARNING_LEVEL, ER_BINLOG_CANT_RESIZE_CACHE); + } + + flags.with_xid = false; + flags.immediate = false; + flags.finalized = false; + flags.with_sbr = false; + flags.with_rbr = false; + flags.with_start = false; + flags.with_end = false; + flags.with_content = false; + + /* + The truncate function calls reinit_io_cache that calls my_b_flush_io_cache + which may increase disk_writes. This breaks the disk_writes use by the + binary log which aims to compute the ratio between in-memory cache usage + and disk cache usage. To avoid this undesirable behavior, we reset the + variable after truncating the cache. + */ + cache_state_map.clear(); + m_event_counter = 0; + m_checksum_trx_start = mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + m_large_trx_optimization_latched = false; + m_large_trx_optimization_enabled = false; + m_large_trx_optimization_threshold = 0; + m_terminating_event_offset = 0; + m_terminating_event_type = mysql::binlog::event::UNKNOWN_EVENT; + m_compressed_size = 0; + m_decompressed_size = 0; + m_compression_type = mysql::binlog::event::compression::NONE; + assert(is_binlog_empty()); + } + + /** + Returns information about the cache content with respect to + the binlog_format of the events. + + This will be used to set a flag on GTID_LOG_EVENT stating that the + transaction may have SBR statements or not, but the binlog dump + will show this flag as "rbr_only" when it is not set. That's why + an empty transaction should return true below, or else an empty + transaction would be assumed as "rbr_only" even not having RBR + events. + + When dumping a binary log content using mysqlbinlog client program, + for any transaction assumed as "rbr_only" it will be printed a + statement changing the transaction isolation level to READ COMMITTED. + It doesn't make sense to have an empty transaction "requiring" this + isolation level change. + + @return true The cache have SBR events or is empty. + @return false The cache contains a transaction with no SBR events. + */ + bool may_have_sbr_stmts() { return flags.with_sbr || !flags.with_rbr; } + + /** + Check if the binlog cache contains an empty transaction, which has + two binlog events "BEGIN" and "COMMIT". + + @return true The binlog cache contains an empty transaction. + @return false Otherwise. + */ + bool has_empty_transaction() { + /* + The empty transaction has two events in trx/stmt binlog cache + and no changes: one is a transaction start and other is a transaction + end (there should be no SBR changing content and no RBR events). + */ + if (flags.with_start && // Has transaction start statement + flags.with_end && // Has transaction end statement + !flags.with_content) // Has no other content than START/END + { + assert(m_event_counter == 2); // Two events in the cache only + assert(!flags.with_sbr); // No statements changing content + assert(!flags.with_rbr); // No rows changing content + assert(!flags.immediate); // Not a DDL + assert(!flags.with_xid); // Not a XID trx and not an atomic DDL Query + return true; + } + return false; + } + + /** + Check if the binlog cache is empty or contains an empty transaction, + which has two binlog events "BEGIN" and "COMMIT". + + @return true The binlog cache is empty or contains an empty transaction. + @return false Otherwise. + */ + bool is_empty_or_has_empty_transaction() { + return is_binlog_empty() || has_empty_transaction(); + } + + protected: + /* + This structure should have all cache variables/flags that should be restored + when a ROLLBACK TO SAVEPOINT statement be executed. + */ + struct cache_state { + bool with_sbr; + bool with_rbr; + bool with_start; + bool with_end; + bool with_content; + size_t event_counter; + }; + /* + For every SAVEPOINT used, we will store a cache_state for the current + binlog cache position. So, if a ROLLBACK TO SAVEPOINT is used, we can + restore the cache_state values after truncating the binlog cache. + */ + std::map cache_state_map; + /* + In order to compute the transaction size (because of possible extra checksum + bytes), we need to keep track of how many events are in the binlog cache. + */ + size_t m_event_counter = 0; + + /* + Checksum algorithm the events of this cache are serialized with, + recorded at the transaction's first event (see write_event). + */ + mysql::binlog::event::enum_binlog_checksum_alg m_checksum_trx_start = + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + + bool m_large_trx_optimization_latched = false; + bool m_large_trx_optimization_enabled = false; + ulonglong m_large_trx_optimization_threshold = 0; + + /* + Offset and type of the transaction's terminating event in a promoted + binary log file, recorded together at finalize(). + */ + my_off_t m_terminating_event_offset = 0; + mysql::binlog::event::Log_event_type m_terminating_event_type = + mysql::binlog::event::UNKNOWN_EVENT; + + size_t m_compressed_size = 0; + size_t m_decompressed_size = 0; + mysql::binlog::event::compression::type m_compression_type = + mysql::binlog::event::compression::type::NONE; + /* + It truncates the cache to a certain position. This includes deleting the + pending event. It corresponds to rollback statement or rollback to + a savepoint. It doesn't change transaction state. + */ + void truncate(my_off_t pos) { + DBUG_PRINT("info", ("truncating to position %lu", (ulong)pos)); + remove_pending_event(); + + // TODO: check the return value. + (void)m_cache.truncate(pos); + } + + /** + Flush pending event to the cache buffer. + */ + int flush_pending_event(THD *thd) { + if (m_pending) { + m_pending->set_flags(Rows_log_event::STMT_END_F); + if (int error = write_event(m_pending)) return error; + thd->clear_binlog_table_maps(); + } + return 0; + } + + /** + Remove the pending event. + */ + int remove_pending_event() { + delete m_pending; + m_pending = nullptr; + return 0; + } + struct Flags { + /* + Defines if this is either a trx-cache or stmt-cache, respectively, a + transactional or non-transactional cache. + */ + bool transactional : 1; + + /* + This indicates that the cache should be written without BEGIN/END. + */ + bool immediate : 1; + + /* + This flag indicates that the buffer was finalized and has to be + flushed to disk. + */ + bool finalized : 1; + + /* + This indicates that either the cache contain an XID event, or it's + an atomic DDL Query-log-event. In the latter case the flag is set up + on the statement level, namely when the Query-log-event is cached + at time the DDL transaction is not committing. + The flag therefore gets reset when the cache is cleaned due to + the statement rollback, e.g in case of a DDL post-caching execution + error. + Any statement scope flag among other things must consider its + reset policy when the statement is rolled back. + */ + bool with_xid : 1; + + /* + This indicates that the cache contain statements changing content. + */ + bool with_sbr : 1; + + /* + This indicates that the cache contain RBR event changing content. + */ + bool with_rbr : 1; + + /* + This indicates that the cache contain s transaction start statement. + */ + bool with_start : 1; + + /* + This indicates that the cache contain a transaction end event. + */ + bool with_end : 1; + + /* + This indicates that the cache contain content other than START/END. + */ + bool with_content : 1; + } flags; + + /// Compress the current transaction "in-place", if possible + /// + /// This attempts to compress the transaction if it satisfies the + /// necessary pre-conditions. Otherwise it does nothing. + /// + /// @retval true Error: the cache has been corrupted and the + /// transaction must be aborted. + /// + /// @retval false Success: the transaction was either compressed + /// successfully, or compression was not attempted, or compression + /// failed and left the uncompressed transaction intact. + [[nodiscard]] bool compress(THD *thd); + + private: + /* + Reference to the cache_mngr which owns this cache. + */ + class binlog_cache_mngr &m_cache_mngr; + + /* + Storage for byte data. This binlog_cache_data will serialize + events into bytes and put them into m_cache. + */ + Binlog_cache_storage m_cache; + + /* + Pending binrows event. This event is the event where the rows are currently + written. + */ + Rows_log_event *m_pending; + + /** + This function computes binlog cache and disk usage. + */ + void compute_statistics() { + if (!is_binlog_empty()) { + (*ptr_binlog_cache_use)++; + if (m_cache.disk_writes() != 0) (*ptr_binlog_cache_disk_use)++; + } + } + + /* + Stores a pointer to the status variable that keeps track of the in-memory + cache usage. This corresponds to either + . binlog_cache_use or binlog_stmt_cache_use. + */ + ulong *ptr_binlog_cache_use; + + /* + Stores a pointer to the status variable that keeps track of the disk + cache usage. This corresponds to either + . binlog_cache_disk_use or binlog_stmt_cache_disk_use. + */ + ulong *ptr_binlog_cache_disk_use; + + binlog_cache_data &operator=(const binlog_cache_data &info); + binlog_cache_data(const binlog_cache_data &info); +}; + +class binlog_stmt_cache_data : public binlog_cache_data { + public: + binlog_stmt_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, + ptr_binlog_cache_disk_use_arg) {} + + using binlog_cache_data::finalize; + + int finalize(THD *thd); +}; +class binlog_trx_cache_data : public binlog_cache_data { + public: + binlog_trx_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, + ptr_binlog_cache_disk_use_arg), + m_cannot_rollback(false), + before_stmt_pos(MY_OFF_T_UNDEF) {} + + void reset(bool preserve_spilled_file = false) override { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + m_cannot_rollback = false; + before_stmt_pos = MY_OFF_T_UNDEF; + binlog_cache_data::reset(preserve_spilled_file); + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; + } + + bool cannot_rollback() const { return m_cannot_rollback; } + + void set_cannot_rollback() { m_cannot_rollback = true; } + + my_off_t get_prev_position() const { return before_stmt_pos; } + + void set_prev_position(my_off_t pos) { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + before_stmt_pos = pos; + cache_state_checkpoint(before_stmt_pos); + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; + } + + void restore_prev_position() { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + binlog_cache_data::truncate(before_stmt_pos); + cache_state_rollback(before_stmt_pos); + before_stmt_pos = MY_OFF_T_UNDEF; + /* + Binlog statement rollback clears with_xid now as the atomic DDL statement + marker which can be set as early as at event creation and caching. + */ + flags.with_xid = false; + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; + } + + void restore_savepoint(my_off_t pos) { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + binlog_cache_data::truncate(pos); + if (pos <= before_stmt_pos) before_stmt_pos = MY_OFF_T_UNDEF; + cache_state_rollback(pos); + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; + } + + using binlog_cache_data::truncate; + + void truncate(THD *thd, bool all); + + private: + /* + It will be set true if any statement which cannot be rolled back safely + is put in trx_cache. + */ + bool m_cannot_rollback; + + /* + Binlog position before the start of the current statement. + */ + my_off_t before_stmt_pos; + + binlog_trx_cache_data &operator=(const binlog_trx_cache_data &info); + binlog_trx_cache_data(const binlog_trx_cache_data &info); +}; + +class binlog_cache_mngr { + /// Indicates that some events did not get into the cache(s) and most + /// likely it is incomplete. @see handle_deferred_cache_write_incident + std::string m_incident; + + public: +#ifndef NDEBUG + /// The number of times that the incident status has been set due to the + /// debug symbol binlog_inject_incident. + int m_injected_incident_count{0}; +#endif + + binlog_cache_mngr(ulong *ptr_binlog_stmt_cache_use_arg, + ulong *ptr_binlog_stmt_cache_disk_use_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : stmt_cache(*this, false, ptr_binlog_stmt_cache_use_arg, + ptr_binlog_stmt_cache_disk_use_arg), + trx_cache(*this, true, ptr_binlog_cache_use_arg, + ptr_binlog_cache_disk_use_arg) {} + + bool init() { + return stmt_cache.open(binlog_stmt_cache_size, + max_binlog_stmt_cache_size) || + trx_cache.open(binlog_cache_size, max_binlog_cache_size); + } + + binlog_cache_data *get_binlog_cache_data(bool is_transactional) { + if (is_transactional) + return &trx_cache; + else + return &stmt_cache; + } + + Binlog_cache_storage *get_stmt_cache() { return stmt_cache.get_cache(); } + Binlog_cache_storage *get_trx_cache() { return trx_cache.get_cache(); } + /** + Convenience method to check if both caches are empty. + */ + bool is_binlog_empty() const { + return stmt_cache.is_binlog_empty() && trx_cache.is_binlog_empty(); + } + + int handle_deferred_cache_write_incident(THD *thd); + + /// Check if either of the caches have an incident + /// @see handle_deferred_cache_write_incident + bool has_incident() const { return !m_incident.empty(); } + + void set_incident(std::string_view incident_message) { + assert(!incident_message.empty()); + m_incident = incident_message; + } + + /* + clear stmt_cache and trx_cache if they are not empty + */ + void reset() { + if (!stmt_cache.is_binlog_empty()) stmt_cache.reset(); + if (!trx_cache.is_binlog_empty()) trx_cache.reset(); + } + +#ifndef NDEBUG + bool dbug_any_finalized() const { + return stmt_cache.is_finalized() || trx_cache.is_finalized(); + } +#endif + + /* + Convenience method to flush both caches to the binary log. + + @param bytes_written Pointer to variable that will be set to the + number of bytes written for the flush. + @param wrote_xid Pointer to variable that will be set to @c + true if any XID event was written to the + binary log. Otherwise, the variable will not + be touched. + @return Error code on error, zero if no error. + */ + int flush(THD *thd, my_off_t *bytes_written, bool *wrote_xid) { + my_off_t stmt_bytes = 0; + my_off_t trx_bytes = 0; + assert(stmt_cache.has_xid() == 0); + + bool parallelization_barrier = false; + if (has_incident()) { + if (int error = handle_deferred_cache_write_incident(thd)) return error; + // Request force rotate + thd->rpl_thd_ctx.binlog_group_commit_ctx().set_force_rotate(); + // Set as parallelization_barrier so that dependency tracker marks all + // subsequent transactions to depend on it. + parallelization_barrier = true; + } + + int error = + stmt_cache.flush(thd, &stmt_bytes, wrote_xid, parallelization_barrier); + if (error) return error; + DEBUG_SYNC(thd, "after_flush_stm_cache_before_flush_trx_cache"); + error = + trx_cache.flush(thd, &trx_bytes, wrote_xid, parallelization_barrier); + if (error) return error; + *bytes_written = stmt_bytes + trx_bytes; + return 0; + } + + /** + Check if at least one of transactions and statement binlog caches + contains an empty transaction, other one is empty or contains an + empty transaction. + + @return true At least one of transactions and statement binlog + caches an empty transaction, other one is empty + or contains an empty transaction. + @return false Otherwise. + */ + bool has_empty_transaction() { + return (trx_cache.is_empty_or_has_empty_transaction() && + stmt_cache.is_empty_or_has_empty_transaction() && + !is_binlog_empty()); + } + + binlog_stmt_cache_data stmt_cache; + binlog_trx_cache_data trx_cache; + + private: + binlog_cache_mngr &operator=(const binlog_cache_mngr &info); + binlog_cache_mngr(const binlog_cache_mngr &info); +}; + +#endif // BINLOG_CACHE_DATA_H_INCLUDED diff --git a/sql/binlog/large_trx_commit.cc b/sql/binlog/large_trx_commit.cc new file mode 100644 index 000000000000..f2eafaaf3543 --- /dev/null +++ b/sql/binlog/large_trx_commit.cc @@ -0,0 +1,510 @@ +#include "sql/binlog/large_trx_commit.h" +#include "sql/binlog/transaction_commit_helper.h" + +#include +#include + +#include "my_dbug.h" +#include "my_dir.h" +#include "my_sys.h" +#include "my_systime.h" // my_micro_time +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/binlog/event/control_events.h" +#include "mysql/psi/mysql_file.h" +#include "sql/basic_ostream.h" // StringBuffer_ostream +#include "sql/binlog.h" +#include "sql/current_thd.h" +#include "sql/binlog/cache_data.h" // binlog_cache_data +#include "sql/binlog/binlog_ofile.h" // MYSQL_BIN_LOG::Binlog_ofile +#include "sql/binlog_ostream.h" +#include "sql/handler.h" // ha_flush_logs +#include "sql/log_event.h" +#include "sql/mysqld.h" +#include "sql/rpl_gtid.h" // gtid_state +#include "sql/rpl_handler.h" // RUN_HOOK +#include "sql/rpl_trx_tracking.h" // Transaction_dependency_tracker +#include "sql/sql_class.h" +#include "sql/transaction_info.h" + +using mysql::binlog::event::enum_binlog_checksum_alg; + +void record_large_trx_fallback(Large_trx_fallback_reason reason) { + const char *detail = nullptr; + switch (reason) { + case Large_trx_fallback_reason::statement_cache: + detail = "the statement cache is nonempty"; + break; + case Large_trx_fallback_reason::non_row_format: + detail = "the transaction contains non-ROW events"; + break; + case Large_trx_fallback_reason::encryption: + detail = "binary log encryption is enabled"; + break; + case Large_trx_fallback_reason::compression: + detail = "binary log transaction compression is enabled"; + break; + case Large_trx_fallback_reason::checksum_change: + detail = "binlog_checksum changed during the transaction"; + break; + case Large_trx_fallback_reason::reserved_header_space: + detail = "the reserved header region is too small"; + break; + case Large_trx_fallback_reason::gtid_persistence: + detail = + "GTID persistence prevented promotion; attempting the standard " + "commit path"; + break; + case Large_trx_fallback_reason::incident: + detail = "the transaction has a logging incident to report"; + break; + } + /* Atomic: record_large_trx_fallback runs in the commit path without + LOCK_log, so concurrent sessions can increment this counter. */ + binlog_large_transaction_optimization_missed_count.fetch_add( + 1, std::memory_order_relaxed); + LogErr(WARNING_LEVEL, ER_BINLOG_BOLT_LARGE_TRX_FALLBACK, detail); +} + +bool is_large_trx_promotion_eligible(binlog_cache_mngr *cache_mngr) { + binlog_cache_data *trx_cache = &cache_mngr->trx_cache; + return trx_cache->large_trx_optimization_enabled() && + trx_cache->get_cache()->is_spilled() && + trx_cache->get_byte_position() > + trx_cache->large_trx_optimization_threshold() && + cache_mngr->stmt_cache.is_binlog_empty() && + !cache_mngr->has_incident() && !trx_cache->may_have_sbr_stmts() && + !trx_cache->get_cache()->is_encrypted() && + trx_cache->get_compression_type() == + mysql::binlog::event::compression::NONE && + trx_cache->checksum_trx_start() == + static_cast(binlog_checksum_options); +} + +binlog_cache_data *get_cache_to_promote(binlog_cache_mngr *cache_mngr) { + binlog_cache_data *trx_cache = &cache_mngr->trx_cache; + if (!trx_cache->large_trx_optimization_enabled()) return nullptr; + + DBUG_EXECUTE_IF("force_large_trx_compression_fallback", { + record_large_trx_fallback(Large_trx_fallback_reason::compression); + return nullptr; + }); + + if (!trx_cache->get_cache()->is_spilled() || + trx_cache->get_byte_position() <= + trx_cache->large_trx_optimization_threshold()) + return nullptr; + + DBUG_EXECUTE_IF("force_large_trx_statement_cache_fallback", { + record_large_trx_fallback(Large_trx_fallback_reason::statement_cache); + return nullptr; + }); + if (!cache_mngr->stmt_cache.is_binlog_empty()) { + record_large_trx_fallback(Large_trx_fallback_reason::statement_cache); + return nullptr; + } + /* + A pending incident is normally materialized and force-rotated by the + group-commit flush, which the BOLT path bypasses. Fall back to the standard + commit path so the incident is written and the replica is notified. + */ + if (cache_mngr->has_incident()) { + record_large_trx_fallback(Large_trx_fallback_reason::incident); + return nullptr; + } + if (trx_cache->may_have_sbr_stmts()) { + record_large_trx_fallback(Large_trx_fallback_reason::non_row_format); + return nullptr; + } + /* + Consult the spilled file's actual encryption, not the live global: a file + that spilled while binlog_encryption was ON stays encrypted even if the + global was turned OFF before commit, and must never be promoted (its + plaintext header would front an encrypted body). It commits through the + standard path, which decrypts on read. + */ + if (trx_cache->get_cache()->is_encrypted()) { + record_large_trx_fallback(Large_trx_fallback_reason::encryption); + return nullptr; + } + if (trx_cache->get_compression_type() != + mysql::binlog::event::compression::NONE) { + record_large_trx_fallback(Large_trx_fallback_reason::compression); + return nullptr; + } + if (trx_cache->checksum_trx_start() != + static_cast(binlog_checksum_options)) { + record_large_trx_fallback(Large_trx_fallback_reason::checksum_change); + return nullptr; + } + return trx_cache; +} + +bool MYSQL_BIN_LOG::write_promoted_binlog_header(THD *thd, + binlog_cache_data *cache_data, + File file, bool *fits) { + DBUG_TRACE; + mysql_mutex_assert_owner(&LOCK_log); + *fits = true; + + const enum_binlog_checksum_alg checksum_alg = + cache_data->checksum_trx_start(); + assert(checksum_alg != mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF); + const my_off_t checksum_len = + checksum_alg != mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF + ? BINLOG_CHECKSUM_LEN + : 0; + const my_off_t reserved = cache_data->get_cache()->reserved_bytes(); + + /* The header events are serialized into memory first. */ + StringBuffer_ostream<1024> block; + + if (block.write(pointer_cast(BINLOG_MAGIC), + BIN_LOG_HEADER_SIZE)) + return true; + + /* The Format_description event, with rotation semantics (created = 0), + so replicas do not treat the promoted file as a server restart. */ + Format_description_log_event fde; + fde.common_header->flags |= LOG_EVENT_BINLOG_IN_USE_F; + fde.dont_set_created = true; + if (!fde.is_valid()) return true; + fde.common_footer->checksum_alg = checksum_alg; + fde.common_header->log_pos = block.length(); + if (binary_event_serialize(&fde, &block)) return true; + + /* The Previous_gtids event. The snapshot is stable: transactions commit + under LOCK_commit, which the caller holds. */ + { + Gtid_set logged_gtids_binlog(global_tsid_map, global_tsid_lock); + global_tsid_lock->wrlock(); + const Gtid_set *executed_gtids = gtid_state->get_executed_gtids(); + const Gtid_set *gtids_only_in_table = + gtid_state->get_gtids_only_in_table(); + if (logged_gtids_binlog.add_gtid_set(executed_gtids) != RETURN_STATUS_OK) { + global_tsid_lock->unlock(); + return true; + } + logged_gtids_binlog.remove_gtid_set(gtids_only_in_table); + Previous_gtids_log_event prev_gtids_ev(&logged_gtids_binlog); + global_tsid_lock->unlock(); + const my_off_t previous_gtids_start = block.length(); + prev_gtids_ev.common_footer->checksum_alg = checksum_alg; + prev_gtids_ev.common_header->log_pos = previous_gtids_start; + if (binary_event_serialize(&prev_gtids_ev, &block)) return true; + update_binlog_temp_file_previous_gtids_size_estimate( + block.length() - previous_gtids_start); + } + + /* + Check that the reserved region fits the remaining header events: the + Large_transaction_header event at its minimum size and the Gtid event + at its maximum. If not, nothing has been assigned or written yet and + the transaction can still fall back to the standard commit path. + */ + const my_off_t lth_min_length = LOG_EVENT_HEADER_LEN + + mysql::binlog::event::Large_transaction_header_event::kFixedBodyLength + + checksum_len; + const bool header_events_fit = + block.length() + lth_min_length + + mysql::binlog::event::Gtid_event::get_max_event_length() + + checksum_len <= reserved; + /* + Debug hook to force the "reserved region too small" fallback in tests. + Placed here, before any GTID assignment or file I/O, so it takes the same + no-side-effects fallback path as the real check just below. + */ + DBUG_EXECUTE_IF("force_large_trx_reserved_header_fallback", { + *fits = false; + return false; + }); + if (!header_events_fit) { + *fits = false; + return false; + } + + /* + Commit point of the promotion. The promoted file starts a new binary + log file: rotate the dependency tracker before generating the + transaction's logical timestamps, so they restart for the new file + (open_binlog() must then not rotate it again). The transaction is a + parallelization barrier for the replica's parallel applier. + */ + m_dependency_tracker.rotate(); + thd->get_transaction()->sequence_number = m_dependency_tracker.step(); + + assert(thd->next_to_commit == nullptr); + if (assign_automatic_gtids_to_flush_group(thd)) return true; + + Transaction_gtid_header metadata{thd, true /*parallelization_barrier*/, + &m_dependency_tracker}; + + Gtid_log_event gtid_event( + thd, cache_data->is_trx_cache(), metadata.last_committed(), + metadata.sequence_number(), cache_data->may_have_sbr_stmts(), + metadata.original_commit_timestamp(), + metadata.immediate_commit_timestamp(), metadata.original_server_version(), + metadata.immediate_server_version()); + gtid_event.set_trx_length_by_cache_size( + cache_data->get_byte_position(), checksum_len != 0, + cache_data->is_checksum_computed(), cache_data->get_event_counter()); + + /* + The Large_transaction_header event's padding is sized so the Gtid + event ends exactly at the reserved offset, where the transaction's + first event was placed at spill time. + */ + const my_off_t gtid_length = gtid_event.get_event_length() + checksum_len; + const my_off_t lth_length = reserved - gtid_length - block.length(); + assert(lth_length >= lth_min_length); + + Large_transaction_header_log_event lth_event( + thd, cache_data->terminating_event_offset(), + cache_data->terminating_event_type(), + lth_length - lth_min_length /*padding_size*/); + lth_event.common_footer->checksum_alg = checksum_alg; + lth_event.common_header->log_pos = block.length(); + if (binary_event_serialize(<h_event, &block)) return true; + + gtid_event.common_footer->checksum_alg = checksum_alg; + gtid_event.common_header->log_pos = block.length(); + if (binary_event_serialize(>id_event, &block)) return true; + assert(block.length() == reserved); + + /* Fill the reserved region and make the complete file durable. */ + if (mysql_file_pwrite(file, pointer_cast(block.ptr()), + block.length(), 0, MYF(MY_WME + MY_NABP)) != 0) + return true; + if (mysql_file_sync(file, MYF(MY_WME)) != 0) return true; + DBUG_EXECUTE_IF("crash_bolt_after_header_sync", DBUG_SUICIDE();); + return false; +} + +int MYSQL_BIN_LOG::commit_large_transaction(THD *thd, bool all, + bool skip_commit, + binlog_cache_data *cache_data) { + DBUG_TRACE; + int error = 0; + bool fits = true; + bool lock_index_acquired = false; + bool purge_index_opened = false; + bool purge_index_registered = false; + bool promoted_file_renamed = false; + char new_name[FN_REFLEN], *old_name; + const char *temp_file_name = nullptr; + bool keep_current_binlog = false; + Large_trx_fallback_reason fallback_reason; + + /* Make the spilled file durable before entering the critical section. */ + if (cache_data->get_cache()->flush_and_sync_spilled_file() || + DBUG_EVALUATE_IF("fail_bolt_spill_file_sync", true, false)) { + thd->commit_error = THD::CE_FLUSH_ERROR; + cache_data->reset(); + return finish_commit(thd); + } + + /* Initialize private THD state before entering the global lock section. */ + init_thd_variables(thd, all, skip_commit, true /*ready_preempt*/); + + /* Critical section */ + mysql_mutex_lock(&LOCK_log); + wait_for_prep_xids(); + mysql_mutex_lock(&LOCK_commit); + + /* + Stage #0: Ensure new binlog file is complete and previous binlog + file is durable. + */ + if ((error = generate_new_name(new_name, name))) goto err; + + /* + Persist the current binary log's GTIDs into mysql.gtid_executed, exactly + as a normal rotation does. If that table is temporarily read-only, + keep_current_binlog is set and we fall back to the standard commit path. + */ + m_binlog_index_monitor.lock(); + lock_index_acquired = true; + if ((error = persist_gtids_on_rotate(true /*use_dedicated_thd*/, + &keep_current_binlog))) { + if (keep_current_binlog) { + DBUG_EXECUTE_IF("gtid_executed_readonly", { DBUG_SET("-d,gtid_executed_readonly"); }); + fallback_reason = Large_trx_fallback_reason::gtid_persistence; + goto fallback_to_ordered_commit; + } + thd->commit_error = THD::CE_FLUSH_ERROR; + goto err; + } + + /* + Write the promoted file's header events into the reserved region at the + front of the spilled file. The transaction body lives past that region, + so this does not disturb the cache's data; if the header does not fit, + the fit check reports it (fits == false) and we fall back. + */ + if (write_promoted_binlog_header( + thd, cache_data, cache_data->get_cache()->spilled_file(), &fits)) + goto err; + if (!fits) { + /* + The header events no longer fit the reserved region + (gtid_executed grew since the spill): fall back to the standard + commit path. + */ + fallback_reason = Large_trx_fallback_reason::reserved_header_space; + goto fallback_to_ordered_commit; + } + + /* + Register the new binary log name in the purge index and sync it, before + the rename. This is the file's crash-recovery record until it is added to + the main index below: if we crash after this point but before that, + startup uses the purge index to delete the not-yet-published file. + */ + if (m_binlog_index_monitor.open_purge_index_file(true)) goto err; + purge_index_opened = true; + if (m_binlog_index_monitor.register_create_index_entry(new_name) || + m_binlog_index_monitor.sync_purge_index_file()) { + LogErr(ERROR_LEVEL, ER_BINLOG_FAILED_TO_SYNC_INDEX_FILE_IN_OPEN); + goto err; + } + purge_index_registered = true; + DBUG_EXECUTE_IF("crash_bolt_after_purge_index_sync", DBUG_SUICIDE();); + + /* The cache keeps the open descriptor across the rename; sync the renamed + file so it is durable under its new binary-log name. */ + temp_file_name = cache_data->get_cache()->tmp_file_name(); + if (temp_file_name == nullptr) goto err; + if (my_rename(temp_file_name, new_name, MYF(MY_WME))) { + LogErr(ERROR_LEVEL, ER_BINLOG_CANT_USE_FOR_LOGGING, new_name, errno); + goto err; + } + promoted_file_renamed = true; + if (mysql_file_sync(cache_data->get_cache()->spilled_file(), MYF(MY_WME)) != + 0) + goto err; + DBUG_EXECUTE_IF("crash_bolt_after_promote_rename", DBUG_SUICIDE();); + + /* + Stage 2 (sync): chain the current binary log to the promoted file and + sync that rotate event before opening the promoted file as active. + */ + { + Rotate_log_event r(new_name + dirname_length(new_name), 0, + LOG_EVENT_OFFSET, 0 /*flags*/); + if ((error = write_event_to_binlog(&r) ? 1 : 0)) goto err; + } + if ((error = m_binlog_file->flush_and_sync() ? 1 : 0)) goto err; + + /* + The promoted file is durable and the old binlog's Rotate event points to + it. Close the old log file but keep the index open: this stops startup + recovery from acting on the purge record before open_binlog() adds the + promoted file to the main index, just below. + */ + old_name = name; + name = nullptr; // Don't free name; open_binlog reassigns it. + close(LOG_CLOSE_TO_BE_OPENED, false /*need_lock_log*/, + false /*need_lock_index*/); + + error = open_binlog(old_name, new_name, max_size, + true /*null_created_arg*/, false /*need_lock_index*/, + true /*need_tsid_lock*/, + nullptr /*extra_description_event*/, + 0 /*new_index_number*/, new_name, + true /*promoted_file_is_renamed*/) ? 1 : 0; + my_free(old_name); + /* open_binlog() now owns purge-index cleanup on success and failure. */ + purge_index_opened = false; + purge_index_registered = false; + if (error) goto err; + + /* The renamed file is now indexed; reset can release the cache descriptor. */ + m_binlog_index_monitor.unlock(); + lock_index_acquired = false; + + /* Atomic for consistency with the missed counter; not strictly required + here since this increment always runs under LOCK_log. */ + binlog_large_transaction_optimization_count.fetch_add( + 1, std::memory_order_relaxed); + + { + const my_off_t end_pos = m_binlog_file->position(); + thd->set_trans_pos(log_file_name, end_pos); + thd->set_next_event_pos(log_file_name, end_pos); + } + if (cache_data->has_xid() && thd->get_transaction()->m_flags.commit_low) + inc_prep_xids(thd); + + if (RUN_HOOK(binlog_storage, after_flush, + (thd, log_file_name + dirname_length(log_file_name), + m_binlog_file->position()))) { + error = 1; + thd->commit_error = THD::CE_FLUSH_ERROR; + goto err; + } + + /* + Stage 3 (commit): the binary log is durable and visible, so commit the + transaction in the storage engines before releasing LOCK_commit. + */ + DBUG_EXECUTE_IF("crash_bolt_before_engine_commit", DBUG_SUICIDE();); + mysql_mutex_unlock(&LOCK_log); + + // The cache retains the renamed file while resetting its transaction state. + cache_data->reset(true /*preserve_spilled_file*/); + (void)finish_commit(thd); + mysql_mutex_unlock(&LOCK_commit); + + /* Post-commit: rotate when the promoted active file exceeds max_size. */ + if (rotate_if_needed()) thd->commit_error = THD::CE_COMMIT_ERROR; + return thd->commit_error == THD::CE_COMMIT_ERROR; + +fallback_to_ordered_commit: + m_binlog_index_monitor.unlock(); + lock_index_acquired = false; + mysql_mutex_unlock(&LOCK_commit); + mysql_mutex_unlock(&LOCK_log); + record_large_trx_fallback(fallback_reason); + return ordered_commit(thd, all, skip_commit); + +err: + /* + Until the file is in the main index, the purge index owns it. On an + ordinary error, delete the renamed file and its purge-index entry here; + after a crash, startup performs the same cleanup from the purge index. + */ + if (purge_index_opened) { + if (purge_index_registered && promoted_file_renamed) + (void)purge_index_entry(nullptr, nullptr, false /*need_lock_index*/); + m_binlog_index_monitor.close_purge_index_file(); + } + if (lock_index_acquired) m_binlog_index_monitor.unlock(); + if (thd->commit_error == THD::CE_NONE) + thd->commit_error = THD::CE_FLUSH_ERROR; + cache_data->reset(promoted_file_renamed /*preserve_spilled_file*/); + /* + Apply binlog_error_action semantics: either the server aborts, or + binary logging is disabled and the commit proceeds in the engines. + */ + handle_binlog_flush_or_sync_error(thd, false /*need_lock_log*/, nullptr); + mysql_mutex_unlock(&LOCK_commit); + mysql_mutex_unlock(&LOCK_log); + return finish_commit(thd); +} + +int MYSQL_BIN_LOG::rotate_if_needed() { + if (m_binlog_file->get_real_file_size() <= + static_cast(max_size)) + return 0; + + bool check_purge = false; + mysql_mutex_lock(&LOCK_log); + DBUG_EXECUTE_IF("crash_bolt_before_max_size_rotate", DBUG_SUICIDE();); + int error = rotate(false /*force_rotate*/, &check_purge); + /* Match the normal group-commit rotation boundary. */ + if (!error) + DBUG_EXECUTE_IF("crash_bolt_after_max_size_rotate", DBUG_SUICIDE();); + mysql_mutex_unlock(&LOCK_log); + + if (!error && check_purge) auto_purge(); + return error; +} diff --git a/sql/binlog/large_trx_commit.h b/sql/binlog/large_trx_commit.h new file mode 100644 index 000000000000..9921f33bc8d9 --- /dev/null +++ b/sql/binlog/large_trx_commit.h @@ -0,0 +1,56 @@ +#ifndef BINLOG_LARGE_TRX_COMMIT_H_INCLUDED +#define BINLOG_LARGE_TRX_COMMIT_H_INCLUDED + +/** + @file + + The binlog large transaction optimization's commit path: a transaction + whose spilled binlog cache exceeds + binlog_large_transaction_optimization_threshold commits by promoting + its temporary file into the binary log sequence as the next binary log + file (MYSQL_BIN_LOG::commit_large_transaction), instead of copying + the cache into the active binary log. +*/ + +class THD; +class binlog_cache_data; +class binlog_cache_mngr; + +/** + Returns the transaction cache to commit through the large transaction + optimization — promotion of its spilled temporary file into the binary + log sequence — or nullptr when the transaction commits through the + standard path. A transaction qualifies when its spilled size exceeds + binlog_large_transaction_optimization_threshold and no condition forces + the standard path: non-ROW events in the cache, binary log encryption, + transaction compression, a binlog_checksum change since the + transaction's first event, or a non-empty statement cache. + + @param cache_mngr The session's binlog cache manager. + + @return the cache to promote, or nullptr. +*/ +binlog_cache_data *get_cache_to_promote(binlog_cache_mngr *cache_mngr); + +enum class Large_trx_fallback_reason { + statement_cache, + non_row_format, + encryption, + compression, + checksum_change, + reserved_header_space, + gtid_persistence, + incident, +}; + +/** + * Returns whether a transaction is eligible for BOLT promotion without + * recording fallback telemetry. Use before prepare when the durability policy + * must be selected without changing commit-path accounting. + */ +bool is_large_trx_promotion_eligible(binlog_cache_mngr *cache_mngr); + +/** Record an attempted promotion that must use the standard commit path. */ +void record_large_trx_fallback(Large_trx_fallback_reason reason); + +#endif // BINLOG_LARGE_TRX_COMMIT_H_INCLUDED diff --git a/sql/binlog/log_sanitizer.cc b/sql/binlog/log_sanitizer.cc index 848bbea59e8c..c31f076f9ccb 100644 --- a/sql/binlog/log_sanitizer.cc +++ b/sql/binlog/log_sanitizer.cc @@ -22,6 +22,8 @@ // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. #include "sql/binlog/log_sanitizer.h" +#include "mysql/components/services/log_builtins.h" // LogErr +#include "mysqld_error.h" #include "sql/binlog.h" #include "sql/binlog/decompressing_event_object_istream.h" // Decompressing_event_object_istream #include "sql/psi_memory_key.h" @@ -62,6 +64,78 @@ bool Log_sanitizer::is_log_truncation_needed() const { return m_is_log_truncation_needed; } +void Log_sanitizer::process_large_trx_header_event( + Large_transaction_header_log_event const &ev, + IBasic_binlog_file_reader &reader) { + const my_off_t xid_offset = + static_cast(ev.get_terminating_event_offset()); + const uint8_t xid_type = ev.get_terminating_event_type(); + + const bool valid_xid_type = + xid_type == mysql::binlog::event::QUERY_EVENT || + xid_type == mysql::binlog::event::XID_EVENT || + xid_type == mysql::binlog::event::XA_PREPARE_LOG_EVENT; + + const bool valid_xid_offset = + xid_offset != 0 && xid_offset >= BIN_LOG_HEADER_SIZE && + xid_offset <= m_last_file_size && xid_offset > reader.position() && + m_last_file_size - xid_offset >= LOG_EVENT_HEADER_LEN; + + if (!valid_xid_offset || !valid_xid_type) { + m_is_malformed = true; + m_failure_message.assign( + "Large_transaction_header_log_event holds an invalid terminal event"); + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER, + m_valid_file.c_str()); + return; + } + + if (reader.is_checksum_verification_enabled()) { + // The BOLT recovery optimization that skips the transaction body is not + // applicable when source checksum verification is enabled. + LogErr(WARNING_LEVEL, + ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_CHECKSUM_VERIFICATION); + return; + } + + m_large_trx_xid_offset = xid_offset; + m_large_trx_xid_type = xid_type; + if (reader.seek(xid_offset)) { + m_is_malformed = true; + m_failure_message.assign( + "Large_transaction_header_log_event holds an invalid terminal event"); + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER, + m_valid_file.c_str()); + return; + } + + LogErr(INFORMATION_LEVEL, ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_SKIP, + static_cast(xid_offset), m_valid_file.c_str()); + m_in_transaction = true; +} + +bool Log_sanitizer::validate_large_trx_terminal_event(Log_event const &ev) { + const my_off_t event_start_pos = + static_cast(ev.common_header->log_pos - + ev.common_header->data_written); + if (event_start_pos != m_large_trx_xid_offset) return true; + + if (static_cast(ev.get_type_code()) == m_large_trx_xid_type) { + // Terminal event validated. Clear the recorded metadata so a later event + // can never be re-matched against this (already-consumed) transaction. + m_large_trx_xid_offset = 0; + m_large_trx_xid_type = 0; + return true; + } + + m_is_malformed = true; + m_failure_message.assign( + "Large_transaction_header_log_event holds an invalid terminal event"); + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER, + m_valid_file.c_str()); + return false; +} + void Log_sanitizer::process_query_event(Query_log_event const &ev) { std::string query{ev.query}; diff --git a/sql/binlog/log_sanitizer.h b/sql/binlog/log_sanitizer.h index 5cf6e9e21791..42e1be61dc51 100644 --- a/sql/binlog/log_sanitizer.h +++ b/sql/binlog/log_sanitizer.h @@ -115,6 +115,15 @@ class Log_sanitizer { /// @returns Reference to a memory key virtual PSI_memory_key &get_memory_key() const = 0; + /// @brief Whether this sanitizer is recovering a relay log rather than a + /// binary log. A Large_transaction_header_log_event's recorded offset + /// refers to the source's binary log, so it is only actionable during + /// binary-log recovery; relay-log recovery must ignore it. This cannot be + /// derived from the event itself, because a header relayed from the source + /// does not carry LOG_EVENT_RELAY_LOG_F. + /// @returns true for relay-log recovery, false for binary-log recovery. + virtual bool is_relay_log_recovery() const { return false; } + /// @brief This function goes through the opened file and searches for /// a valid position in a binary log file. It also gathers /// information about XA transactions which will be used during the @@ -229,6 +238,22 @@ class Log_sanitizer { /// Last opened file size my_off_t m_last_file_size{0}; + /// Metadata for the large transaction's terminal event. + my_off_t m_large_trx_xid_offset{0}; + uint8_t m_large_trx_xid_type{0}; + + /// @brief Invoked when a `Large_transaction_header_log_event` is read from + /// the reader. + /// @details Validates the terminal event's offset and type, seeks to that + /// event, and marks the transaction as open. + /// @param ev The `Large_transaction_header_log_event` to process. + /// @param reader Reader for the current binary log. + void process_large_trx_header_event( + Large_transaction_header_log_event const &ev, + IBasic_binlog_file_reader &reader); + + bool validate_large_trx_terminal_event(Log_event const &ev); + /// @brief Invoked when a `Query_log_event` is read from the binary log file /// reader. /// @details The underlying query string is inspected to determine if the diff --git a/sql/binlog/log_sanitizer_impl.hpp b/sql/binlog/log_sanitizer_impl.hpp index 26aaffde906c..97a9ac8112de 100644 --- a/sql/binlog/log_sanitizer_impl.hpp +++ b/sql/binlog/log_sanitizer_impl.hpp @@ -123,6 +123,8 @@ bool Log_sanitizer::process_one_log(Type_reader &reader, this->m_is_malformed = false; while (istream >> ev) { + if (!this->validate_large_trx_terminal_event(*ev)) break; + bool is_source_event = !ev->is_relay_log_event() || (ev->server_id && ::server_id != ev->server_id); switch (ev->get_type_code()) { @@ -139,6 +141,18 @@ bool Log_sanitizer::process_one_log(Type_reader &reader, dynamic_cast(*ev)); break; } + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: { + // The header's offset refers to the source's binary log, so it is only + // actionable during binary-log recovery. Relay-log recovery must skip + // it: a header relayed from the source does not carry + // LOG_EVENT_RELAY_LOG_F, so is_relay_log_recovery() (not the event + // flag) is the correct discriminator. + if (!this->is_relay_log_recovery()) { + this->process_large_trx_header_event( + dynamic_cast(*ev), reader); + } + break; + } case mysql::binlog::event::ROTATE_EVENT: { if (is_source_event) { m_validation_started = true; diff --git a/sql/binlog/transaction_commit_helper.cc b/sql/binlog/transaction_commit_helper.cc new file mode 100644 index 000000000000..5304b42fb972 --- /dev/null +++ b/sql/binlog/transaction_commit_helper.cc @@ -0,0 +1,67 @@ +#include "sql/binlog/transaction_commit_helper.h" + +#include "dur_prop.h" +#include "my_dbug.h" +#include "my_systime.h" +#include "sql/mysqld.h" +#include "sql/rpl_gtid.h" +#include "sql/rpl_trx_tracking.h" +#include "sql/sql_class.h" +#include "sql/transaction_info.h" + +void init_thd_variables(THD *thd, bool all, bool skip_commit, + [[maybe_unused]] bool ready_preempt) { + /* These values are reset before a transaction enters commit processing. */ + thd->tx_commit_pending = true; + thd->commit_error = THD::CE_NONE; + thd->next_to_commit = nullptr; + thd->durability_property = HA_IGNORE_DURABILITY; + thd->get_transaction()->m_flags.real_commit = all; + thd->get_transaction()->m_flags.xid_written = false; + thd->get_transaction()->m_flags.commit_low = !skip_commit; + thd->get_transaction()->m_flags.run_hooks = !skip_commit; +#ifndef NDEBUG + thd->get_transaction()->m_flags.ready_preempt = ready_preempt; +#endif +} + +Transaction_gtid_header::Transaction_gtid_header( + THD *thd, bool parallelization_barrier, + Transaction_dependency_tracker *dependency_tracker) { + dependency_tracker->get_dependency(thd, parallelization_barrier, + m_sequence_number, m_last_committed); + + /* Preserve commit ordering when the statement cache follows this cache. */ + thd->get_transaction()->last_committed = SEQ_UNINIT; + + m_immediate_commit_timestamp = my_micro_time(); + m_original_commit_timestamp = thd->variables.original_commit_timestamp; + if (m_original_commit_timestamp == UNDEFINED_COMMIT_TIMESTAMP) { + if (thd->slave_thread || thd->is_binlog_applier()) { + m_original_commit_timestamp = 0; + } else { + DBUG_EXECUTE_IF("rpl_invalid_gtid_timestamp", + m_immediate_commit_timestamp += 3600000000;); + m_original_commit_timestamp = m_immediate_commit_timestamp; + } + } else { + thd->variables.original_commit_timestamp = UNDEFINED_COMMIT_TIMESTAMP; + } + + m_immediate_server_version = do_server_version_int(::server_version); + thd->variables.immediate_server_version = UNDEFINED_SERVER_VERSION; + DBUG_EXECUTE_IF("fixed_server_version", m_immediate_server_version = 888888;); + DBUG_EXECUTE_IF("gr_fixed_server_version", + m_immediate_server_version = 777777;); + + m_original_server_version = thd->variables.original_server_version; + if (m_original_server_version == UNDEFINED_SERVER_VERSION) { + if (thd->slave_thread || thd->is_binlog_applier()) { + m_original_server_version = UNKNOWN_SERVER_VERSION; + } else { + m_original_server_version = m_immediate_server_version; + } + } else { + thd->variables.original_server_version = UNDEFINED_SERVER_VERSION; + } +} diff --git a/sql/binlog/transaction_commit_helper.h b/sql/binlog/transaction_commit_helper.h new file mode 100644 index 000000000000..1a9b90a2496b --- /dev/null +++ b/sql/binlog/transaction_commit_helper.h @@ -0,0 +1,47 @@ +#ifndef BINLOG_TRANSACTION_COMMIT_HELPER_H_INCLUDED +#define BINLOG_TRANSACTION_COMMIT_HELPER_H_INCLUDED + +#include + +#include "my_inttypes.h" + +class THD; +class Transaction_dependency_tracker; + +/** Initializes THD state shared by group commit and BOLT. */ +void init_thd_variables(THD *thd, bool all, bool skip_commit, + bool ready_preempt = false); + +/** + This class collects the fields used to create a transaction's GTID event. + It is used by both group and large transaction commit codepaths. +*/ +class Transaction_gtid_header { + public: + Transaction_gtid_header( + THD *thd, bool parallelization_barrier, + Transaction_dependency_tracker *dependency_tracker); + + int64 last_committed() const { return m_last_committed; } + int64 sequence_number() const { return m_sequence_number; } + ulonglong original_commit_timestamp() const { + return m_original_commit_timestamp; + } + ulonglong immediate_commit_timestamp() const { + return m_immediate_commit_timestamp; + } + uint32_t original_server_version() const { return m_original_server_version; } + uint32_t immediate_server_version() const { + return m_immediate_server_version; + } + + private: + int64 m_last_committed; + int64 m_sequence_number; + ulonglong m_original_commit_timestamp; + ulonglong m_immediate_commit_timestamp; + uint32_t m_original_server_version; + uint32_t m_immediate_server_version; +}; + +#endif // BINLOG_TRANSACTION_COMMIT_HELPER_H_INCLUDED diff --git a/sql/binlog_ostream.cc b/sql/binlog_ostream.cc index 9c6b4680a93b..2bca32b88820 100644 --- a/sql/binlog_ostream.cc +++ b/sql/binlog_ostream.cc @@ -23,10 +23,18 @@ #include "sql/binlog_ostream.h" #include +#include +#include +#include +#include +#include +#include #include "my_aes.h" +#include "my_dir.h" #include "my_inttypes.h" #include "my_rnd.h" #include "my_sys.h" +#include "my_thread_local.h" // my_errno #include "mysql/components/services/log_builtins.h" #include "mysql/psi/mysql_file.h" #include "mysqld_error.h" @@ -38,21 +46,100 @@ bool binlog_cache_is_reset = false; #endif +namespace { + +constexpr char kBinlogTempFilePrefix[] = "bolt_"; + +/* + Returns true if 'name' is a temp file created by this feature, i.e. matches + the bolt_ pattern. Note: a true result does NOT mean the file + was (or will be) promoted into the binary log sequence. Every binlog-cache + spill file uses this name, including transactions that commit through the + standard path (below the threshold, encrypted, compressed, etc.). This is + purely an ownership check so startup cleanup only deletes files this feature + created. +*/ +bool is_bolt_temp_file(const char *name) { + const size_t prefix_length = strlen(kBinlogTempFilePrefix); + if (strncmp(name, kBinlogTempFilePrefix, prefix_length) != 0 || + name[prefix_length] == '\0') + return false; + + for (const char *cursor = name + prefix_length; *cursor != '\0'; ++cursor) { + if (!(*cursor >= 'a' && *cursor <= 'z') && + !(*cursor >= '0' && *cursor <= '9') && *cursor != '_') + return false; + } + return true; +} + +ulong binlog_temp_file_permissions() { + ulong permissions = 0; + if (my_umask & 0400) permissions |= USER_READ; + if (my_umask & 0200) permissions |= USER_WRITE; + if (my_umask & 0100) permissions |= USER_EXECUTE; + if (my_umask & 0040) permissions |= GROUP_READ; + if (my_umask & 0020) permissions |= GROUP_WRITE; + if (my_umask & 0010) permissions |= GROUP_EXECUTE; + if (my_umask & 0004) permissions |= OTHERS_READ; + if (my_umask & 0002) permissions |= OTHERS_WRITE; + if (my_umask & 0001) permissions |= OTHERS_EXECUTE; + return permissions; +} + +} // namespace + IO_CACHE_binlog_cache_storage::IO_CACHE_binlog_cache_storage() = default; IO_CACHE_binlog_cache_storage::~IO_CACHE_binlog_cache_storage() { close(); } bool IO_CACHE_binlog_cache_storage::open(const char *dir, const char *prefix, my_off_t cache_size, - my_off_t max_cache_size) { + my_off_t max_cache_size, + my_off_t reserved_bytes) { DBUG_TRACE; if (open_cached_file(&m_io_cache, dir, prefix, cache_size, MYF(MY_WME))) return true; + m_spilled_file_is_managed = false; + + /* + Default to an anonymous spill file. Whether it is instead a named file that + can be promoted into the binary log sequence is decided per transaction + from the optimization knob, at transaction start, via set_named_file(). + A named file is kept in the filesystem namespace (promotable, and cleaned + up at server startup if left behind); an anonymous file is unlinked at + creation and is never visible. So when the optimization is disabled for a + transaction, its spill file leaves no visible bolt_ file. + */ + m_io_cache.named_file = false; + /* Keep the arguments: reset re-opens the cache after a spill. */ + m_dir = dir; + m_prefix = prefix; + m_cache_size = cache_size; + m_max_cache_size_arg = max_cache_size; + + /* + The cache's content is placed after the reserved bytes: physical + positions start there, and the first flush into the lazily created + temporary file seeks there (the file's offset is 0 at creation). + + The reserved region is applied in all cases, even when the optimization is + disabled for this transaction. A non-promoted transaction never fills it + with header events and never copies it into the binary log (begin() starts + the read cursor past it), so it costs only transient temp-file space and is + invisible in the binary log. Only the file naming is gated on the knob. + */ + m_reserved_bytes = reserved_bytes; + m_io_cache.pos_in_file = reserved_bytes; + m_io_cache.seek_not_done = true; if (rpl_encryption.is_enabled()) enable_encryption(); m_max_cache_size = max_cache_size; + /* The max cache size caps physical positions: shift it too. */ + if (m_max_cache_size <= ~(my_off_t)0 - reserved_bytes) + m_max_cache_size += reserved_bytes; /* Set the max cache size for IO_CACHE */ - m_io_cache.end_of_file = max_cache_size; + m_io_cache.end_of_file = m_max_cache_size; return false; } @@ -87,10 +174,25 @@ bool IO_CACHE_binlog_cache_storage::write(const unsigned char *buffer, } } - return my_b_safe_write(&m_io_cache, buffer, length); + if (my_b_safe_write(&m_io_cache, buffer, length)) return true; + + /* + open_cached_file creates the physical backing file lazily. For a named + (promotable) spill file, immediately replace its generic mkstemp name with + the managed bolt_ form before callers can observe or promote it. An + anonymous spill file (optimization disabled for this transaction) has no + name and is left as-is. + */ + if (is_spilled() && m_io_cache.named_file && !m_spilled_file_is_managed) { + if (rename_spilled_file()) return true; + m_spilled_file_is_managed = true; + } + return false; } bool IO_CACHE_binlog_cache_storage::truncate(my_off_t offset) { + /* Translate the zero-based data offset to a physical position. */ + offset += m_reserved_bytes; /* It is not really necessary to flush the data will be truncated into temporary file before truncating . And it may cause write failure. So set @@ -105,18 +207,72 @@ bool IO_CACHE_binlog_cache_storage::truncate(my_off_t offset) { return false; } -bool IO_CACHE_binlog_cache_storage::reset() { - if (truncate(0)) return true; +bool IO_CACHE_binlog_cache_storage::rename_spilled_file() { + DBUG_TRACE; + assert(is_spilled()); + if (m_io_cache.file_name == nullptr) return true; + + const char *const old_name = m_io_cache.file_name; + + /* + The name is bolt__, both in hex. It is unique + within a server run: server_start_time is constant for the run and the + serial is a monotonic atomic counter, so no probe or retry is needed. + Leftover files from earlier runs are removed at startup; if that cleanup + fails the optimization is disabled and no new files are created, so a + name cannot collide with an earlier run's file either. + */ + static std::atomic serial_counter{0}; + const uint64_t serial = + serial_counter.fetch_add(1, std::memory_order_relaxed); + + char new_name[FN_REFLEN]; + int length; + if (m_dir != nullptr) + length = snprintf(new_name, sizeof(new_name), "%s%c%s%llx_%llx", m_dir, + FN_LIBCHAR, kBinlogTempFilePrefix, + static_cast(server_start_time), + static_cast(serial)); + else + length = snprintf(new_name, sizeof(new_name), "%s%llx_%llx", + kBinlogTempFilePrefix, + static_cast(server_start_time), + static_cast(serial)); + if (length < 0 || static_cast(length) >= sizeof(new_name)) + return true; + + char *replacement = my_strdup(PSI_NOT_INSTRUMENTED, new_name, MYF(MY_WME)); + if (replacement == nullptr) return true; + if (mysql_file_rename(m_io_cache.file_key, old_name, new_name, + MYF(MY_WME))) { + my_free(replacement); + return true; + } - /* Truncate the temporary file if there is one. */ - if (m_io_cache.file != -1) { - if (my_chsize(m_io_cache.file, 0, 0, MYF(MY_WME))) return true; + my_free(m_io_cache.file_name); + m_io_cache.file_name = replacement; + return my_chmod(new_name, binlog_temp_file_permissions(), MYF(MY_WME)); +} - DBUG_EXECUTE_IF("show_io_cache_size", { - my_off_t file_size = - my_seek(m_io_cache.file, 0L, MY_SEEK_END, MYF(MY_WME + MY_FAE)); - assert(file_size == 0); - }); +bool IO_CACHE_binlog_cache_storage::reset(bool preserve_spilled_file) { + assert(!preserve_spilled_file || is_spilled()); + if (is_spilled()) { + /* + A finished transaction leaves no trace in #binlog_temp_files unless BOLT + has promoted the file into the binary log sequence. + */ + disable_encryption(); + if (preserve_spilled_file) { + /* Prevent close_cached_file from deleting the promoted file. */ + my_free(m_io_cache.file_name); + m_io_cache.file_name = nullptr; + } + close(); + if (open(m_dir, m_prefix, m_cache_size, m_max_cache_size_arg, + m_reserved_bytes)) + return true; + } else if (truncate(0)) { + return true; } DBUG_EXECUTE_IF("ensure_binlog_cache_temporary_file_is_encrypted", { @@ -144,8 +300,26 @@ size_t IO_CACHE_binlog_cache_storage::disk_writes() const { return m_io_cache.disk_writes; } +bool IO_CACHE_binlog_cache_storage::is_spilled() const { + return m_io_cache.file != -1; +} + +bool IO_CACHE_binlog_cache_storage::flush_and_sync_spilled_file() { + DBUG_TRACE; + assert(is_spilled()); + if (flush_io_cache(&m_io_cache)) return true; + /* + A ROLLBACK TO SAVEPOINT truncation repositions the cache but does not + shrink the file: cut any stale bytes past the logical end, so the + promoted file ends exactly at the transaction's terminating event. + */ + if (my_chsize(m_io_cache.file, my_b_tell(&m_io_cache), 0, MYF(MY_WME))) + return true; + return mysql_file_sync(m_io_cache.file, MYF(MY_WME)) != 0; +} + const char *IO_CACHE_binlog_cache_storage::tmp_file_name() const { - return my_filename(m_io_cache.file); + return m_io_cache.file_name; } bool IO_CACHE_binlog_cache_storage::begin(unsigned char **buffer, @@ -166,7 +340,9 @@ bool IO_CACHE_binlog_cache_storage::begin(unsigned char **buffer, m_io_cache.m_decryptor == nullptr); };); - if (reinit_io_cache(&m_io_cache, READ_CACHE, 0, false, false)) { + /* The data starts after the reserved bytes. */ + if (reinit_io_cache(&m_io_cache, READ_CACHE, m_reserved_bytes, false, + false)) { DBUG_EXECUTE_IF("simulate_tmpdir_partition_full", { DBUG_SET("-d,simulate_file_write_error"); }); @@ -194,8 +370,11 @@ bool IO_CACHE_binlog_cache_storage::next(unsigned char **buffer, } my_off_t IO_CACHE_binlog_cache_storage::length() const { - if (m_io_cache.type == WRITE_CACHE) return my_b_tell(&m_io_cache); - return m_io_cache.end_of_file; + /* Physical positions include the reserved bytes; report the data + length. */ + if (m_io_cache.type == WRITE_CACHE) + return my_b_tell(&m_io_cache) - m_reserved_bytes; + return m_io_cache.end_of_file - m_reserved_bytes; } bool IO_CACHE_binlog_cache_storage::enable_encryption() { @@ -250,10 +429,10 @@ bool IO_CACHE_binlog_cache_storage::setup_ciphers_password() { return false; } -bool Binlog_cache_storage::open(my_off_t cache_size, my_off_t max_cache_size) { - const char *LOG_PREFIX = "ML"; - - if (m_file.open(mysql_tmpdir, LOG_PREFIX, cache_size, max_cache_size)) +bool Binlog_cache_storage::open(my_off_t cache_size, my_off_t max_cache_size, + my_off_t reserved_bytes) { + if (m_file.open(binlog_temp_files_dir.path(), kBinlogTempFilePrefix, + cache_size, max_cache_size, reserved_bytes)) return true; m_pipeline_head = &m_file; return false; @@ -411,3 +590,93 @@ bool Binlog_encryption_ostream::sync() { return m_down_ostream->sync(); } int Binlog_encryption_ostream::get_header_size() { return m_header->get_header_size(); } + +Binlog_temp_files_dir binlog_temp_files_dir; + +// Clears the temp files directory. +static bool temp_files_dir_clear_files(const char *path) { + MY_DIR *dir_info = my_dir(path, MYF(MY_WANT_STAT)); + if (dir_info == nullptr) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED, path, + my_errno()); + return true; + } + + uint removed = 0; + bool failed = false; + for (uint i = 0; i < dir_info->number_off_files && !failed; i++) { + const fileinfo *file = dir_info->dir_entry + i; + /* Skip "." and "..". */ + if (file->name[0] == '.' && + (!file->name[1] || (file->name[1] == '.' && !file->name[2]))) + continue; + char file_path[FN_REFLEN]; + if (snprintf(file_path, sizeof(file_path), "%s%c%s", path, FN_LIBCHAR, + file->name) >= static_cast(sizeof(file_path))) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID, path); + failed = true; + break; + } + /* Do not follow links or delete entries the server did not create. */ + if (file->mystat == nullptr || !MY_S_ISREG(file->mystat->st_mode) || + my_is_symlink(file_path, nullptr) || !is_bolt_temp_file(file->name)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_UNSAFE_ENTRY, path, + file->name); + failed = true; + break; + } + if (my_delete(file_path, MYF(0))) { + LogErr(ERROR_LEVEL, ER_BINLOG_CANT_DELETE_FILE, file_path); + failed = true; + break; + } + removed++; + } + my_dirend(dir_info); + if (failed) return true; + + if (removed > 0) + LogErr(INFORMATION_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_CLEANED, removed, + path); + return false; +} + +bool Binlog_temp_files_dir::init(const char *log_basename) { + DBUG_TRACE; + assert(log_basename != nullptr && !m_initialized); + + // Build /#binlog_temp_files + char dir_part[FN_REFLEN]; + size_t dir_len; + dirname_part(dir_part, log_basename, &dir_len); + if (dir_len + strlen(kBinlogTempFilesDirName) + 1 > sizeof(m_path)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED, log_basename, + ENAMETOOLONG); + return true; + } + snprintf(m_path, sizeof(m_path), "%s%s", dir_part, kBinlogTempFilesDirName); + const char *path = m_path; + + /* A symlink is rejected even if it points to a directory: files in + this directory must be on the same filesystem as the binlog files. */ + if (my_is_symlink(path, nullptr)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID, path); + return true; + } + + MY_STAT stat_area; + if (my_stat(path, &stat_area, MYF(0)) != nullptr) { + if (!MY_S_ISDIR(stat_area.st_mode)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID, path); + return true; + } + if (temp_files_dir_clear_files(path)) return true; + } else if (my_mkdir(path, my_umask_dir, MYF(0)) != 0) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED, path, + my_errno()); + return true; + } + + m_initialized = true; + return false; +} diff --git a/sql/binlog_ostream.h b/sql/binlog_ostream.h index 41dba074fd3a..c9660e6f99fa 100644 --- a/sql/binlog_ostream.h +++ b/sql/binlog_ostream.h @@ -25,6 +25,8 @@ #define BINLOG_OSTREAM_INCLUDED #include +#include +#include "my_io.h" // FN_REFLEN #include "sql/basic_ostream.h" #include "sql/rpl_log_encryption.h" @@ -84,11 +86,13 @@ class IO_CACHE_binlog_cache_storage : public Truncatable_ostream { @param[in] prefix Prefix of the temporary file name @param[in] cache_size Size of the memory buffer. @param[in] max_cache_size Maximum size of the memory buffer + @param[in] reserved_bytes Bytes reserved for header events + @retval false Success @retval true Error */ bool open(const char *dir, const char *prefix, my_off_t cache_size, - my_off_t max_cache_size); + my_off_t max_cache_size, my_off_t reserved_bytes); void close(); bool write(const unsigned char *buffer, my_off_t length) override; @@ -97,10 +101,11 @@ class IO_CACHE_binlog_cache_storage : public Truncatable_ostream { /* binlog cache doesn't need seek operation. Setting true to return error */ bool seek(my_off_t offset [[maybe_unused]]) override { return true; } /** - Reset status and drop all data. It looks like a cache never was used after - reset. + Reset status and drop all data. When preserve_spilled_file is true, the + caller has promoted the spilled file and reset closes the cache without + deleting that file. */ - bool reset(); + bool reset(bool preserve_spilled_file = false); /** Returns the file name if a temporary file is opened, otherwise nullptr is returned. @@ -135,12 +140,53 @@ class IO_CACHE_binlog_cache_storage : public Truncatable_ostream { */ bool next(unsigned char **buffer, my_off_t *length); my_off_t length() const; + my_off_t reserved_bytes() const { return m_reserved_bytes; } bool flush() override { return false; } bool sync() override { return false; } + /** + Returns true once the cache overflowed into its temporary file. + */ + bool is_spilled() const; + /** + Returns the spilled temporary file's descriptor (-1 when not + spilled). The cache retains ownership until reset. + */ + File spilled_file() const { return m_io_cache.file; } + /** + Returns true if the cache's events are encrypted. This reflects the + temporary file's actual encryption, which is fixed at spill time, and is + independent of the current global binlog_encryption setting (which may + have changed since the transaction spilled). + */ + bool is_encrypted() const { return m_io_cache.m_encryptor != nullptr; } + /** + Flushes buffered bytes into the spilled temporary file and syncs the + file to disk. The cache must be spilled. + @retval false Success + @retval true Error + */ + bool flush_and_sync_spilled_file(); + /** + Selects whether the next lazily-created spill file is a named file (kept + in the filesystem namespace, and therefore promotable) or an anonymous + file. Set per transaction, before the first spill. + */ + void set_named_file(bool named) { m_io_cache.named_file = named; } private: + /** Rename a newly spilled generic cache file to a managed bolt_ name. */ + bool rename_spilled_file(); + IO_CACHE m_io_cache; my_off_t m_max_cache_size = 0; + my_off_t m_reserved_bytes = 0; + /* True after the lazily-created spill file has a generated bolt_ name. */ + bool m_spilled_file_is_managed = false; + /* The open() arguments, kept for re-opening after reset. */ + const char *m_dir = nullptr; + const char *m_prefix = nullptr; + my_off_t m_cache_size = 0; + my_off_t m_max_cache_size_arg = 0; /** Enable IO Cache temporary file encryption. @@ -175,7 +221,8 @@ class Binlog_cache_storage : public Basic_ostream { public: ~Binlog_cache_storage() override; - bool open(my_off_t cache_size, my_off_t max_cache_size); + bool open(my_off_t cache_size, my_off_t max_cache_size, + my_off_t reserved_bytes); void close(); bool write(const unsigned char *buffer, my_off_t length) override { @@ -192,19 +239,36 @@ class Binlog_cache_storage : public Basic_ostream { bool truncate(my_off_t offset) { return m_pipeline_head->truncate(offset); } /** - Reset status and drop all data. It looks like a cache was never used - after reset. + Reset status and drop all data. When preserve_spilled_file is true, the + cache closes without deleting a file BOLT has already promoted. */ - bool reset() { return m_file.reset(); } + bool reset(bool preserve_spilled_file = false) { + return m_file.reset(preserve_spilled_file); + } /** Returns the count of disk writes */ size_t disk_writes() const { return m_file.disk_writes(); } + /** + Returns the bytes reserved at the beginning of the temp file. + */ + my_off_t reserved_bytes() const { return m_file.reserved_bytes(); } /** Returns the name of the temporary file. */ const char *tmp_file_name() const { return m_file.tmp_file_name(); } - + /// @see IO_CACHE_binlog_cache_storage::is_spilled + bool is_spilled() const { return m_file.is_spilled(); } + /// @see IO_CACHE_binlog_cache_storage::spilled_file + File spilled_file() const { return m_file.spilled_file(); } + /// @see IO_CACHE_binlog_cache_storage::is_encrypted + bool is_encrypted() const { return m_file.is_encrypted(); } + /// @see IO_CACHE_binlog_cache_storage::flush_and_sync_spilled_file + bool flush_and_sync_spilled_file() { + return m_file.flush_and_sync_spilled_file(); + } + /// @see IO_CACHE_binlog_cache_storage::set_named_file + void set_named_file(bool named) { m_file.set_named_file(named); } /** Copy all data to a output stream. This function hides the internal implementation of storage detail. So it will not disturb the callers @@ -301,4 +365,48 @@ class Binlog_encryption_ostream : public Truncatable_ostream { std::unique_ptr m_header; std::unique_ptr m_encryptor; }; + +// Directory where the temp binlog files (spilled from cache), live. +// This directory lives in the binlog directory. +inline constexpr const char *kBinlogTempFilesDirName = "#binlog_temp_files"; + +// Allocation quantum for the header reservation at the beginning of every +// binlog temp file. +inline constexpr my_off_t kBinlogTempFileReservedBytes = 64 * 1024; + +// Minimum space left after the Previous_gtids payload in a temp-file header. +inline constexpr my_off_t kBinlogTempFilePreviousGtidsHeadroomBytes = + 32 * 1024; + +// Managed large-transaction spill files are named bolt__ +// (both hex), which is unique within a server run and lets startup cleanup +// recognize only files created by this feature. + +class Binlog_temp_files_dir { + public: + /** + Init directory at server startup. + + @param log_basename The log basename; the directory is created in + its directory part + + @retval false Success. + @retval true Failure; an error has been logged. + */ + bool init(const char *log_basename); + + // @return full path of the directory. + const char *path() const { + assert(m_initialized); + return m_path; + } + + private: + char m_path[FN_REFLEN]; + bool m_initialized{false}; +}; + +// The binary log's temp files directory (#binlog_temp_files). +extern Binlog_temp_files_dir binlog_temp_files_dir; + #endif // BINLOG_OSTREAM_INCLUDED diff --git a/sql/binlog_reader.cc b/sql/binlog_reader.cc index 82fe6bd84f8c..d758673857ae 100644 --- a/sql/binlog_reader.cc +++ b/sql/binlog_reader.cc @@ -306,6 +306,9 @@ Binlog_read_error::Error_type binlog_event_deserialize( case mysql::binlog::event::TRANSACTION_PAYLOAD_EVENT: ev = new Transaction_payload_log_event(buf, fde); break; + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: + ev = new Large_transaction_header_log_event(buf, fde); + break; default: /* Create an object of Ignorable_log_event for unrecognized sub-class. diff --git a/sql/binlog_reader.h b/sql/binlog_reader.h index eb61dfe3d92d..f4d1407cc4ff 100644 --- a/sql/binlog_reader.h +++ b/sql/binlog_reader.h @@ -396,6 +396,9 @@ class IBasic_binlog_file_reader { /// The return value is static memory that is never deallocated. virtual const char *get_error_str() const = 0; + /// Return whether checksum verification is enabled. + virtual bool is_checksum_verification_enabled() const = 0; + /// Return the current position in bytes, relative to the beginning /// of the file. virtual my_off_t position() const = 0; @@ -529,6 +532,9 @@ class Basic_binlog_file_reader : public IBasic_binlog_file_reader { } bool is_open() const { return m_ifile.is_open(); } + bool is_checksum_verification_enabled() const override { + return m_verify_checksum; + } my_off_t position() const override { return m_ifile.position(); } bool seek(my_off_t pos) override { return m_ifile.seek(pos); } diff --git a/sql/log_event.cc b/sql/log_event.cc index a181de89dbe0..89757e485fb7 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1155,6 +1155,12 @@ bool Log_event::need_checksum() { assert(!ret || ((common_footer->checksum_alg == static_cast(binlog_checksum_options) || + /* + Cached events carry the algorithm recorded at their + transaction's first event, which a concurrent change of + binlog_checksum may outdate. + */ + event_cache_type != Log_event::EVENT_NO_CACHE || /* Stop event closes the relay-log and its checksum alg preference is set by the caller can be different @@ -12993,6 +12999,81 @@ void Ignorable_log_event::print(FILE *, } #endif +Large_transaction_header_log_event:: + Large_transaction_header_log_event( + const char *buf, const Format_description_event *descr_event) + : mysql::binlog::event::Large_transaction_header_event( + buf, descr_event), + Log_event(header(), footer()) { + DBUG_TRACE; +} + +void Large_transaction_header_log_event::claim_memory_ownership( + bool claim) { + my_claim(temp_buf, claim); + my_claim(this, claim); +} + +#ifdef MYSQL_SERVER +int Large_transaction_header_log_event::pack_info( + Protocol *protocol) { + char buf[256]; + const size_t bytes = + snprintf(buf, sizeof(buf), + "# Large transaction header " + "(terminating event offset %llu, type %u)", + static_cast(m_terminating_event_offset), + static_cast(m_terminating_event_type)); + protocol->store_string(buf, bytes, &my_charset_bin); + return 0; +} + +bool Large_transaction_header_log_event::write_data_body( + Basic_ostream *ostream) { + DBUG_TRACE; + uchar fixed[Large_transaction_header_event::kFixedBodyLength]; + fixed[0] = m_version; + int8store(fixed + 1, m_terminating_event_offset); + fixed[1 + sizeof(m_terminating_event_offset)] = + m_terminating_event_type; + if (wrapper_my_b_safe_write(ostream, fixed, sizeof(fixed))) return true; + + /* Write the padding in bounded chunks; its contents are undefined and + ignored on read, zeros keep the file deterministic. */ + const uchar zeros[4096] = {0}; + for (uint64_t left = m_padding_size; left > 0;) { + const size_t chunk = std::min(left, sizeof(zeros)); + if (wrapper_my_b_safe_write(ostream, zeros, chunk)) return true; + left -= chunk; + } + return false; +} + +int Large_transaction_header_log_event::do_apply_event( + Relay_log_info const *) { + DBUG_TRACE; + /* Nothing to apply: the event only carries recovery metadata for the + file it was written into. */ + return 0; +} +#endif + +#ifndef MYSQL_SERVER +void Large_transaction_header_log_event::print( + FILE *, PRINT_EVENT_INFO *print_event_info) const { + if (print_event_info->short_form) return; + + print_header(&print_event_info->head_cache, print_event_info, false); + my_b_printf(&print_event_info->head_cache, + "\tLarge transaction header\tIgnorable\n"); + my_b_printf(&print_event_info->head_cache, + "# Terminating event offset %llu, type %u, padding %llu bytes\n", + static_cast(m_terminating_event_offset), + static_cast(m_terminating_event_type), + static_cast(m_padding_size)); +} +#endif + Rows_query_log_event::Rows_query_log_event( const char *buf, const Format_description_event *descr_event) : mysql::binlog::event::Ignorable_event(buf, descr_event), @@ -13721,32 +13802,58 @@ Log_event::enum_skip_reason Gtid_log_event::do_shall_skip(Relay_log_info *rli) { } #endif // MYSQL_SERVER -void Gtid_log_event::set_trx_length_by_cache_size_tagged( - ulonglong cache_size, bool is_checksum_enabled, int event_counter) { - auto transaction_length_overhead = cache_size; +/** + Compute the transaction's on-disk length from its cache size, correcting for + any checksum bytes that will be added or removed relative to the cache. + + @param cache_size Byte size of the transaction's cached events. + @param is_checksum_enabled Whether events will carry a checksum on disk. + @param is_checksum_computed Whether the cached events already include a + checksum (so it is not added again per event). + @param event_counter Number of events in the transaction. + @return The adjusted transaction length. +*/ +static ulonglong adjust_trx_length_to_checksum_changes(ulonglong cache_size, + bool is_checksum_enabled, + bool is_checksum_computed, + int event_counter) { + ulonglong length = cache_size; if (is_checksum_enabled) { - transaction_length_overhead += (event_counter + 1) * BINLOG_CHECKSUM_LEN; + length += BINLOG_CHECKSUM_LEN; + if (!is_checksum_computed) + length += event_counter * BINLOG_CHECKSUM_LEN; + } else if (is_checksum_computed) { + length -= event_counter * BINLOG_CHECKSUM_LEN; } - transaction_length_overhead += LOG_EVENT_HEADER_LEN; + return length; +} + +void Gtid_log_event::set_trx_length_by_cache_size_tagged( + ulonglong cache_size, bool is_checksum_enabled, bool is_checksum_computed, + int event_counter) { + auto transaction_length_overhead = + adjust_trx_length_to_checksum_changes(cache_size, is_checksum_enabled, + is_checksum_computed, + event_counter) + + LOG_EVENT_HEADER_LEN; update_tagged_transaction_length(transaction_length_overhead); } void Gtid_log_event::set_trx_length_by_cache_size(ulonglong cache_size, bool is_checksum_enabled, + bool is_checksum_computed, int event_counter) { if (is_tagged()) { - return set_trx_length_by_cache_size_tagged(cache_size, is_checksum_enabled, - event_counter); + return set_trx_length_by_cache_size_tagged( + cache_size, is_checksum_enabled, is_checksum_computed, event_counter); } - // Transaction content length - transaction_length = cache_size; - if (is_checksum_enabled) - transaction_length += event_counter * BINLOG_CHECKSUM_LEN; + // Transaction content length, including all checksums + transaction_length = adjust_trx_length_to_checksum_changes( + cache_size, is_checksum_enabled, is_checksum_computed, event_counter); // GTID length transaction_length += LOG_EVENT_HEADER_LEN; transaction_length += POST_HEADER_LENGTH; - transaction_length += is_checksum_enabled ? BINLOG_CHECKSUM_LEN : 0; transaction_length += get_commit_timestamp_length(); transaction_length += get_server_version_length(); return update_untagged_transaction_length(); diff --git a/sql/log_event.h b/sql/log_event.h index f8715432ac02..31a6a079c5a4 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -55,6 +55,7 @@ #include "my_thread_local.h" #include "mysql/binlog/event/binlog_event.h" #include "mysql/binlog/event/control_events.h" +#include "mysql/binlog/event/large_transaction_header_event.h" #include "mysql/binlog/event/load_data_events.h" #include "mysql/binlog/event/rows_event.h" #include "mysql/binlog/event/statement_events.h" @@ -3763,6 +3764,95 @@ class Ignorable_log_event } }; +/** + @class Large_transaction_header_log_event + + Server class of the binlog large transaction optimization header event + (see mysql::binlog::event::Large_transaction_header_event + for the wire format and purpose). Written by the server into the + reserved header region of a promoted binary log file; on the applier + side it is a no-op (and, being flagged ignorable, servers that do not + know the type skip it entirely). + + @internal + The inheritance structure is as follows + + Binary_log_event + ^ + | + B_l:Large_transaction_header_event Log_event + \ / + \ / + Large_transaction_header_log_event + + B_l: namespace mysql::binlog::event + @endinternal +*/ +class Large_transaction_header_log_event + : public mysql::binlog::event::Large_transaction_header_event, + public Log_event { + public: + // disable copy-move semantics + Large_transaction_header_log_event( + Large_transaction_header_log_event &&) noexcept = delete; + Large_transaction_header_log_event &operator=( + Large_transaction_header_log_event &&) noexcept = delete; + Large_transaction_header_log_event( + const Large_transaction_header_log_event &) = delete; + Large_transaction_header_log_event &operator=( + const Large_transaction_header_log_event &) = delete; + +#ifdef MYSQL_SERVER + /** + Creates the event for writing into a promoted binary log file's + reserved header region. + + @param thd_arg THD of the committing session. + @param terminating_event_offset Offset of the transaction's + terminating event in the file. + @param terminating_event_type Type of the transaction's terminating + event in the file. + @param padding_size Filler bytes occupying the remainder + of the reserved region. + */ + Large_transaction_header_log_event( + THD *thd_arg, uint64_t terminating_event_offset, + mysql::binlog::event::Log_event_type terminating_event_type, + uint64_t padding_size) + : mysql::binlog::event::Large_transaction_header_event( + terminating_event_offset, + static_cast(terminating_event_type), padding_size), + Log_event(thd_arg, LOG_EVENT_IGNORABLE_F, Log_event::EVENT_STMT_CACHE, + Log_event::EVENT_NORMAL_LOGGING, header(), footer()) { + DBUG_TRACE; + common_header->set_is_valid(true); + } + + int pack_info(Protocol *protocol) override; + bool write_data_body(Basic_ostream *ostream) override; +#endif + + Large_transaction_header_log_event( + const char *buf, + const mysql::binlog::event::Format_description_event *descr_event); + + ~Large_transaction_header_log_event() override = default; + + void claim_memory_ownership(bool claim) override; + + size_t get_data_size() override { + return kFixedBodyLength + m_padding_size; + } + +#ifndef MYSQL_SERVER + void print(FILE *file, PRINT_EVENT_INFO *print_event_info) const override; +#endif + +#if defined(MYSQL_SERVER) + int do_apply_event(Relay_log_info const *rli) override; +#endif +}; + /** @class Rows_query_log_event It is used to record the original query for the rows @@ -4189,16 +4279,20 @@ class Gtid_log_event : public mysql::binlog::event::Gtid_event, @param cache_size The size of the binlog cache in bytes. @param is_checksum_enabled If checksum will be added to events on flush. + @param is_checksum_computed If the events in the cache already carry + their checksum. @param event_counter The amount of events in the cache. */ void set_trx_length_by_cache_size(ulonglong cache_size, bool is_checksum_enabled = false, + bool is_checksum_computed = false, int event_counter = 0); /// @copydoc set_trx_length_by_cache_size /// @detail tagged version of event void set_trx_length_by_cache_size_tagged(ulonglong cache_size, bool is_checksum_enabled = false, + bool is_checksum_computed = false, int event_counter = 0); }; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8e264455c53a..5327b421eb15 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -795,7 +795,8 @@ MySQL clients support the protocol: #include "sql/auth/sql_authentication.h" // init_rsa_keys #include "sql/auth/sql_security_ctx.h" #include "sql/auto_thd.h" // Auto_THD -#include "sql/binlog.h" // mysql_bin_log +#include "sql/binlog.h" // mysql_bin_log +#include "sql/binlog_ostream.h" // binlog_temp_files_dir #include "sql/bootstrap.h" // bootstrap #include "sql/check_stack.h" #include "sql/conn_handler/connection_acceptor.h" // Connection_acceptor @@ -1399,6 +1400,8 @@ ulong binlog_stmt_cache_size = 0; int32 opt_binlog_max_flush_queue_time = 0; long opt_binlog_group_commit_sync_delay = 0; ulong opt_binlog_group_commit_sync_no_delay_count = 0; +bool opt_binlog_large_transaction_optimization_enabled = true; +ulonglong opt_binlog_large_transaction_optimization_threshold = 0; ulonglong max_binlog_stmt_cache_size = 0; ulong refresh_version; /* Increments on each reload */ std::atomic atomic_global_query_id{1}; @@ -1411,6 +1414,8 @@ ulong binlog_cache_use = 0, binlog_cache_disk_use = 0; ulong binlog_stmt_cache_use = 0, binlog_stmt_cache_disk_use = 0; ulong max_connections, max_connect_errors; ulong rpl_stop_replica_timeout = LONG_TIMEOUT; +std::atomic binlog_large_transaction_optimization_count{0}; +std::atomic binlog_large_transaction_optimization_missed_count{0}; bool thread_cache_size_specified = false; bool host_cache_size_specified = false; bool table_definition_cache_specified = false; @@ -6848,6 +6853,7 @@ int init_common_variables() { } } update_parser_max_mem_size(); + update_binlog_large_transaction_optimization_threshold(); update_optimizer_switch(); set_server_version(); @@ -8396,6 +8402,14 @@ static int init_server_components() { unireg_abort(MYSQLD_ABORT_EXIT); } + /* + Initialize the #binlog_temp_files directory for spilled files; if the + directory already exists, clear it. + */ + if (opt_bin_log && !is_help_or_validate_option() && + binlog_temp_files_dir.init(log_bin_basename)) + unireg_abort(MYSQLD_ABORT_EXIT); + if (global_system_variables.binlog_row_value_options != 0) { const char *msg = nullptr; longlong err = ER_BINLOG_ROW_VALUE_OPTION_IGNORED; @@ -9973,6 +9987,8 @@ int mysqld_main(int argc, char **argv) if (mysql_bin_log.write_event_to_binlog_and_sync(&prev_gtids_ev)) unireg_abort(MYSQLD_ABORT_EXIT); + update_binlog_temp_file_previous_gtids_size_estimate( + prev_gtids_ev.common_header->data_written); // run auto purge member function. It will evaluate auto purge controls // and configuration, calculate which log files are to be purged, and @@ -11469,6 +11485,26 @@ static int show_count_hit_query_past_global_conn_mem_status_limit(THD *, return 0; } +static int show_binlog_large_transaction_optimization_count(THD *, + SHOW_VAR *var, + char *buf) { + var->type = SHOW_LONG; + var->value = buf; + *((long *)buf) = (long)(binlog_large_transaction_optimization_count.load( + std::memory_order_relaxed)); + return 0; +} + +static int show_binlog_large_transaction_optimization_missed_count( + THD *, SHOW_VAR *var, char *buf) { + var->type = SHOW_LONG; + var->value = buf; + *((long *)buf) = + (long)(binlog_large_transaction_optimization_missed_count.load( + std::memory_order_relaxed)); + return 0; +} + static int show_count_hit_query_past_conn_mem_status_limit(THD *, SHOW_VAR *var, char *buf) { var->type = SHOW_LONG; @@ -11747,6 +11783,12 @@ SHOW_VAR status_vars[] = { SHOW_SCOPE_GLOBAL}, {"Binlog_cache_use", (char *)&binlog_cache_use, SHOW_LONG, SHOW_SCOPE_GLOBAL}, + {"Binlog_large_transaction_optimization_count", + (char *)&show_binlog_large_transaction_optimization_count, SHOW_FUNC, + SHOW_SCOPE_GLOBAL}, + {"Binlog_large_transaction_optimization_missed_count", + (char *)&show_binlog_large_transaction_optimization_missed_count, + SHOW_FUNC, SHOW_SCOPE_GLOBAL}, {"Binlog_stmt_cache_disk_use", (char *)&binlog_stmt_cache_disk_use, SHOW_LONG, SHOW_SCOPE_GLOBAL}, {"Binlog_stmt_cache_use", (char *)&binlog_stmt_cache_use, SHOW_LONG, diff --git a/sql/mysqld.h b/sql/mysqld.h index e2c30a3fe260..407bde956d4e 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -293,6 +293,8 @@ extern const char *server_build_id_ptr; #endif extern const double log_10[309]; extern ulong binlog_cache_use, binlog_cache_disk_use; +extern std::atomic binlog_large_transaction_optimization_count; +extern std::atomic binlog_large_transaction_optimization_missed_count; extern ulong binlog_stmt_cache_use, binlog_stmt_cache_disk_use; extern ulong aborted_threads; extern ulong delayed_insert_timeout; @@ -323,6 +325,8 @@ extern ulonglong max_binlog_cache_size, max_binlog_stmt_cache_size; extern int32 opt_binlog_max_flush_queue_time; extern long opt_binlog_group_commit_sync_delay; extern ulong opt_binlog_group_commit_sync_no_delay_count; +extern bool opt_binlog_large_transaction_optimization_enabled; +extern ulonglong opt_binlog_large_transaction_optimization_threshold; extern ulong max_binlog_size, max_relay_log_size; extern ulong replica_max_allowed_packet; extern ulong binlog_row_event_max_size; diff --git a/sql/rpl_relay_log_sanitizer.h b/sql/rpl_relay_log_sanitizer.h index da1d7bd5ded4..bf8f9c4caac7 100644 --- a/sql/rpl_relay_log_sanitizer.h +++ b/sql/rpl_relay_log_sanitizer.h @@ -80,6 +80,8 @@ class Relay_log_sanitizer : public binlog::Log_sanitizer { PSI_memory_key &get_memory_key() const override { return key_memory_relaylog_recovery; } + + bool is_relay_log_recovery() const override { return true; } }; } // namespace rpl diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 09edc90041a3..a454b5604b02 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1283,8 +1283,57 @@ static Sys_var_bool Sys_partial_revokes( ON_CHECK(check_partial_revokes), ON_UPDATE(partial_revokes_update), nullptr, sys_var::PARSE_EARLY); +static void warn_binlog_large_transaction_optimization_threshold_adjusted( + THD *thd, ulonglong previous_threshold, ulonglong adjusted_threshold) { + LogErr(WARNING_LEVEL, ER_BINLOG_BOLT_THRESHOLD_ADJUSTED, + previous_threshold, adjusted_threshold); + if (thd != nullptr) + push_warning_printf( + thd, Sql_condition::SL_WARNING, + ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING, + ER_THD(thd, ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING), + previous_threshold, adjusted_threshold); +} + +static bool adjust_binlog_large_transaction_optimization_threshold(THD *thd) { + if (opt_binlog_large_transaction_optimization_threshold == 0 || + opt_binlog_large_transaction_optimization_threshold >= + static_cast(binlog_cache_size)) + return false; + + const ulonglong previous_threshold = + opt_binlog_large_transaction_optimization_threshold; + opt_binlog_large_transaction_optimization_threshold = binlog_cache_size; + if (opt_binlog_large_transaction_optimization_enabled) + warn_binlog_large_transaction_optimization_threshold_adjusted( + thd, previous_threshold, + opt_binlog_large_transaction_optimization_threshold); + return true; +} + +static bool check_binlog_large_transaction_optimization_threshold( + sys_var *, THD *thd, set_var *var) { + if (var->save_result.ulonglong_value >= + static_cast(binlog_cache_size)) + return false; + + const ulonglong requested_threshold = var->save_result.ulonglong_value; + var->save_result.ulonglong_value = binlog_cache_size; + if (opt_binlog_large_transaction_optimization_enabled) + warn_binlog_large_transaction_optimization_threshold_adjusted( + thd, requested_threshold, var->save_result.ulonglong_value); + return false; +} + +static bool fix_binlog_large_transaction_optimization_enabled( + sys_var *, THD *thd, enum_var_type) { + adjust_binlog_large_transaction_optimization_threshold(thd); + return false; +} + static bool fix_binlog_cache_size(sys_var *, THD *thd, enum_var_type) { check_binlog_cache_size(thd); + adjust_binlog_large_transaction_optimization_threshold(thd); return false; } @@ -1315,6 +1364,38 @@ static Sys_var_ulong Sys_binlog_stmt_cache_size( NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(fix_binlog_stmt_cache_size)); +static Sys_var_bool Sys_binlog_large_transaction_optimization_enabled( + "binlog_large_transaction_optimization_enabled", + "Enables the large transaction optimization, which keeps " + "large-transaction commit latency low, avoids stalling concurrent " + "commits, and keeps binary log crash recovery fast regardless of " + "transaction size. When ON (the default), a transaction whose spilled " + "size exceeds binlog_large_transaction_optimization_threshold is " + "committed by promoting its temporary file into the binary log " + "sequence. When OFF, all transactions commit through the standard " + "code path.", + GLOBAL_VAR(opt_binlog_large_transaction_optimization_enabled), + CMD_LINE(OPT_ARG), DEFAULT(true), NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(nullptr), ON_UPDATE(fix_binlog_large_transaction_optimization_enabled)); + +static Sys_var_ulonglong Sys_binlog_large_transaction_optimization_threshold( + "binlog_large_transaction_optimization_threshold", + "The spilled size in bytes above which a transaction qualifies for the " + "large transaction optimization. Has no effect while " + "binlog_large_transaction_optimization_enabled is OFF.", + GLOBAL_VAR(opt_binlog_large_transaction_optimization_threshold), + CMD_LINE(REQUIRED_ARG), + VALID_RANGE(10 * 1024 * 1024, ULLONG_MAX), DEFAULT(128 * 1024 * 1024), + BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(check_binlog_large_transaction_optimization_threshold), + ON_UPDATE(nullptr)); + +void update_binlog_large_transaction_optimization_threshold() { + if (adjust_binlog_large_transaction_optimization_threshold(nullptr)) + Sys_binlog_large_transaction_optimization_threshold.update_default( + opt_binlog_large_transaction_optimization_threshold); +} + static Sys_var_int32 Sys_binlog_max_flush_queue_time( "binlog_max_flush_queue_time", "The maximum time that the binary log group commit will keep reading" diff --git a/sql/sys_vars.h b/sql/sys_vars.h index f129aee4cf94..68f7ab7e5b11 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -2841,6 +2841,7 @@ class Sys_var_binlog_encryption : public Sys_var_bool { void update_temptable_max_ram_default(); void update_parser_max_mem_size(); +void update_binlog_large_transaction_optimization_threshold(); void update_optimizer_switch(); #endif /* SYS_VARS_H_INCLUDED */ diff --git a/unittest/gunit/binlogevents/CMakeLists.txt b/unittest/gunit/binlogevents/CMakeLists.txt index 097ea5047a0d..12b472d6d2c3 100644 --- a/unittest/gunit/binlogevents/CMakeLists.txt +++ b/unittest/gunit/binlogevents/CMakeLists.txt @@ -36,6 +36,7 @@ SET(TESTS grow_calculator gtids heartbeat_codec + large_transaction_header payload_event_buffer_istream transaction_compression transaction_payload_codec diff --git a/unittest/gunit/binlogevents/large_transaction_header-t.cc b/unittest/gunit/binlogevents/large_transaction_header-t.cc new file mode 100644 index 000000000000..a919a0fb506d --- /dev/null +++ b/unittest/gunit/binlogevents/large_transaction_header-t.cc @@ -0,0 +1,115 @@ +/* Copyright (c) 2026, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is designed to work with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have either included with + the program or referenced in the documentation. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include +#include + +#include "my_byteorder.h" +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/binlog/event/control_events.h" +#include "mysql/binlog/event/large_transaction_header_event.h" + +namespace mysql::binlog::event::unittests { + +class LargeTrxHeaderTest : public ::testing::Test { + protected: + LargeTrxHeaderTest() : m_fde(BINLOG_VERSION, "9.7.0") {} + + /** + Build a serialized LTH event with a full fixed body (version, offset, + terminal event type). The version byte is caller-supplied so that + invalid-version cases can be exercised without truncating the body. + */ + std::vector build_event(uint8_t version, uint64_t offset, + uint8_t terminal_type, size_t padding_size) { + const size_t event_size = + LOG_EVENT_MINIMAL_HEADER_LEN + + Large_transaction_header_event::kFixedBodyLength + padding_size; + std::vector buf(event_size, '\0'); + uchar *p = reinterpret_cast(buf.data()); + int4store(p, 1755000000); // timestamp + p[EVENT_TYPE_OFFSET] = LARGE_TRANSACTION_HEADER_EVENT; + int4store(p + SERVER_ID_OFFSET, 1); + int4store(p + EVENT_LEN_OFFSET, static_cast(event_size)); + int4store(p + LOG_POS_OFFSET, 0); + int2store(p + FLAGS_OFFSET, LOG_EVENT_IGNORABLE_F); + uchar *body = p + LOG_EVENT_MINIMAL_HEADER_LEN; + body[0] = version; + int8store(body + 1, offset); + body[Large_transaction_header_event::kFixedBodyLength - 1] = terminal_type; + return buf; + } + + Format_description_event m_fde; +}; + +TEST_F(LargeTrxHeaderTest, DecodeRoundtrip) { + const uint64_t offsets[] = {0, 1, UINT32_MAX, 0x1122334455667788ULL, + UINT64_MAX}; + const size_t paddings[] = {0, 1, 4096, 128 * 1024}; + for (uint64_t offset : offsets) { + for (size_t padding : paddings) { + auto buf = build_event( + Large_transaction_header_event::kVersion, offset, + static_cast(XID_EVENT), padding); + Large_transaction_header_event ev(buf.data(), &m_fde); + ASSERT_TRUE(ev.header()->get_is_valid()); + EXPECT_EQ(ev.get_version(), Large_transaction_header_event::kVersion); + EXPECT_EQ(ev.get_terminating_event_offset(), offset); + EXPECT_EQ(ev.get_terminating_event_type(), + static_cast(XID_EVENT)); + EXPECT_EQ(ev.get_padding_size(), padding); + } + } +} + +TEST_F(LargeTrxHeaderTest, IgnorableFlagIsPreserved) { + auto buf = build_event(Large_transaction_header_event::kVersion, 42, + static_cast(XID_EVENT), 16); + Large_transaction_header_event ev(buf.data(), &m_fde); + ASSERT_TRUE(ev.header()->get_is_valid()); + EXPECT_NE(ev.header()->flags & LOG_EVENT_IGNORABLE_F, 0); +} + +TEST_F(LargeTrxHeaderTest, RejectsUnknownVersion) { + for (uint8_t bad_version : {uint8_t{0}, uint8_t{2}, uint8_t{255}}) { + auto buf = build_event(bad_version, 42, static_cast(XID_EVENT), + 16); + Large_transaction_header_event ev(buf.data(), &m_fde); + EXPECT_FALSE(ev.header()->get_is_valid()); + } +} + +TEST_F(LargeTrxHeaderTest, RejectsTruncatedBody) { + /* Body shorter than the fixed fields: version byte only. */ + auto buf = build_event(Large_transaction_header_event::kVersion, 42, + static_cast(XID_EVENT), 0); + buf.resize(LOG_EVENT_MINIMAL_HEADER_LEN + 1); + uchar *p = reinterpret_cast(buf.data()); + int4store(p + EVENT_LEN_OFFSET, static_cast(buf.size())); + Large_transaction_header_event ev(buf.data(), &m_fde); + EXPECT_FALSE(ev.header()->get_is_valid()); +} + +} // namespace mysql::binlog::event::unittests diff --git a/unittest/gunit/log_event_status_size-t.cc b/unittest/gunit/log_event_status_size-t.cc index d9023397cf90..7894c6bcfe26 100644 --- a/unittest/gunit/log_event_status_size-t.cc +++ b/unittest/gunit/log_event_status_size-t.cc @@ -75,8 +75,13 @@ class LogEventStatusSizeTest : public ::testing::Test { Query_log_event qe(srv.thd(), query.c_str(), query.length(), using_trans, immediate, suppress_use, error, ignore_command); + /* The cache creates its temporary file in #binlog_temp_files, + prepared at server startup; create it here. */ + ASSERT_FALSE(binlog_temp_files_dir.init("./gunit_binlog")); + Binlog_cache_storage os; - os.open(50000, 90000); // random values, bigger than maximal packet size + // random values, bigger than maximal packet size + os.open(50000, 90000, kBinlogTempFileReservedBytes); // set qe values to simulate maximal size of the status variables // artificial data