From f28766335b24e2c8cdb310247a85ecacf5f04fa6 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 28 Jun 2020 13:56:45 -0700 Subject: [PATCH 001/100] vttablet: stateManager tests WIP Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletserver/state_manager.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 832df33668d..e093c38df52 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -279,6 +279,8 @@ func (sm *stateManager) CheckMySQL() { // within timeBombDuration, it crashes the process. func (sm *stateManager) StopService() { defer close(sm.setTimeBomb()) + + log.Info("Stopping TabletServer") sm.SetServingType(sm.Target().TabletType, StateNotConnected, nil) } From 5b99487f8ed0e540b5eb69e343c731af580dee86 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Thu, 2 Jul 2020 22:51:35 -0700 Subject: [PATCH 002/100] tabletserver: create new ReplTracker skeleton Signed-off-by: Sugu Sougoumarane --- .../tabletserver/repltracker/reader.go | 220 ++++++++++++++++++ .../tabletserver/repltracker/reader_test.go | 114 +++++++++ .../tabletserver/repltracker/repltracker.go | 74 ++++++ .../tabletserver/repltracker/writer.go | 183 +++++++++++++++ .../tabletserver/repltracker/writer_test.go | 109 +++++++++ 5 files changed, 700 insertions(+) create mode 100644 go/vt/vttablet/tabletserver/repltracker/reader.go create mode 100644 go/vt/vttablet/tabletserver/repltracker/reader_test.go create mode 100644 go/vt/vttablet/tabletserver/repltracker/repltracker.go create mode 100644 go/vt/vttablet/tabletserver/repltracker/writer.go create mode 100644 go/vt/vttablet/tabletserver/repltracker/writer_test.go diff --git a/go/vt/vttablet/tabletserver/repltracker/reader.go b/go/vt/vttablet/tabletserver/repltracker/reader.go new file mode 100644 index 00000000000..5a2d950d329 --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/reader.go @@ -0,0 +1,220 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "fmt" + "sync" + "time" + + "vitess.io/vitess/go/vt/vtgate/evalengine" + + "vitess.io/vitess/go/vt/vterrors" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/timer" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + + querypb "vitess.io/vitess/go/vt/proto/query" +) + +const ( + sqlFetchMostRecentHeartbeat = "SELECT ts FROM %s.heartbeat WHERE keyspaceShard=%a" +) + +// heartbeatReader reads the heartbeat table at a configured interval in order +// to calculate replication lag. It is meant to be run on a replica, and paired +// with a heartbeatWriter on a master. +// Lag is calculated by comparing the most recent timestamp in the heartbeat +// table against the current time at read time. +type heartbeatReader struct { + env tabletenv.Env + + enabled bool + interval time.Duration + keyspaceShard string + now func() time.Time + errorLog *logutil.ThrottledLogger + + runMu sync.Mutex + isOpen bool + pool *connpool.Pool + ticks *timer.Timer + + lagMu sync.Mutex + lastKnownLag time.Duration + lastKnownError error +} + +// newHeartbeatReader returns a new heartbeatReader. +func newHeartbeatReader(env tabletenv.Env) *heartbeatReader { + config := env.Config() + if config.HeartbeatIntervalSeconds == 0 { + return &heartbeatReader{} + } + + heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9) + return &heartbeatReader{ + env: env, + enabled: true, + now: time.Now, + interval: heartbeatInterval, + ticks: timer.NewTimer(heartbeatInterval), + errorLog: logutil.NewThrottledLogger("HeartbeatReporter", 60*time.Second), + pool: connpool.NewPool(env, "HeartbeatReadPool", tabletenv.ConnPoolConfig{ + Size: 1, + IdleTimeoutSeconds: env.Config().OltpReadPool.IdleTimeoutSeconds, + }), + } +} + +// InitDBConfig initializes the target name for the heartbeatReader. +func (r *heartbeatReader) InitDBConfig(target querypb.Target) { + r.keyspaceShard = fmt.Sprintf("%s:%s", target.Keyspace, target.Shard) +} + +// Open starts the heartbeat ticker and opens the db pool. +func (r *heartbeatReader) Open() { + if !r.enabled { + return + } + r.runMu.Lock() + defer r.runMu.Unlock() + if r.isOpen { + return + } + + log.Info("Beginning heartbeat reads") + r.pool.Open(r.env.Config().DB.AppWithDB(), r.env.Config().DB.DbaWithDB(), r.env.Config().DB.AppDebugWithDB()) + r.ticks.Start(func() { r.readHeartbeat() }) + r.isOpen = true +} + +// Close cancels the watchHeartbeat periodic ticker and closes the db pool. +func (r *heartbeatReader) Close() { + if !r.enabled { + return + } + r.runMu.Lock() + defer r.runMu.Unlock() + if !r.isOpen { + return + } + r.ticks.Stop() + r.pool.Close() + log.Info("Stopped heartbeat reads") + r.isOpen = false +} + +// GetLatest returns the most recently recorded lag measurement or error encountered. +func (r *heartbeatReader) GetLatest() (time.Duration, error) { + r.lagMu.Lock() + defer r.lagMu.Unlock() + if r.lastKnownError != nil { + return 0, r.lastKnownError + } + return r.lastKnownLag, nil +} + +// readHeartbeat reads from the heartbeat table exactly once, updating +// the last known lag and/or error, and incrementing counters. +func (r *heartbeatReader) readHeartbeat() { + defer r.env.LogError() + + ctx, cancel := context.WithDeadline(context.Background(), r.now().Add(r.interval)) + defer cancel() + + res, err := r.fetchMostRecentHeartbeat(ctx) + if err != nil { + r.recordError(vterrors.Wrap(err, "Failed to read most recent heartbeat")) + return + } + ts, err := parseHeartbeatResult(res) + if err != nil { + r.recordError(vterrors.Wrap(err, "Failed to parse heartbeat result")) + return + } + + lag := r.now().Sub(time.Unix(0, ts)) + cumulativeLagNs.Add(lag.Nanoseconds()) + currentLagNs.Set(lag.Nanoseconds()) + reads.Add(1) + + r.lagMu.Lock() + r.lastKnownLag = lag + r.lastKnownError = nil + r.lagMu.Unlock() +} + +// fetchMostRecentHeartbeat fetches the most recently recorded heartbeat from the heartbeat table, +// returning a result with the timestamp of the heartbeat. +func (r *heartbeatReader) fetchMostRecentHeartbeat(ctx context.Context) (*sqltypes.Result, error) { + conn, err := r.pool.Get(ctx) + if err != nil { + return nil, err + } + defer conn.Recycle() + sel, err := r.bindHeartbeatFetch() + if err != nil { + return nil, err + } + return conn.Exec(ctx, sel, 1, false) +} + +// bindHeartbeatFetch takes a heartbeat read and adds the necessary +// fields to the query as bind vars. This is done to protect ourselves +// against a badly formed keyspace or shard name. +func (r *heartbeatReader) bindHeartbeatFetch() (string, error) { + bindVars := map[string]*querypb.BindVariable{ + "ks": sqltypes.StringBindVariable(r.keyspaceShard), + } + parsed := sqlparser.BuildParsedQuery(sqlFetchMostRecentHeartbeat, "_vt", ":ks") + bound, err := parsed.GenerateQuery(bindVars, nil) + if err != nil { + return "", err + } + return bound, nil +} + +// parseHeartbeatResult turns a raw result into the timestamp for processing. +func parseHeartbeatResult(res *sqltypes.Result) (int64, error) { + if len(res.Rows) != 1 { + return 0, fmt.Errorf("failed to read heartbeat: writer query did not result in 1 row. Got %v", len(res.Rows)) + } + ts, err := evalengine.ToInt64(res.Rows[0][0]) + if err != nil { + return 0, err + } + return ts, nil +} + +// recordError keeps track of the lastKnown error for reporting to the healthcheck. +// Errors tracked here are logged with throttling to cut down on log spam since +// operations can happen very frequently in this package. +func (r *heartbeatReader) recordError(err error) { + r.lagMu.Lock() + r.lastKnownError = err + r.lagMu.Unlock() + r.errorLog.Errorf("%v", err) + readErrors.Add(1) +} diff --git a/go/vt/vttablet/tabletserver/repltracker/reader_test.go b/go/vt/vttablet/tabletserver/repltracker/reader_test.go new file mode 100644 index 00000000000..43dd0695c43 --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/reader_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "fmt" + "testing" + "time" + + "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + + querypb "vitess.io/vitess/go/vt/proto/query" +) + +// TestReaderReadHeartbeat tests that reading a heartbeat sets the appropriate +// fields on the object. +func TestReaderReadHeartbeat(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + tr := newReader(db, mockNowFunc) + defer tr.Close() + + db.AddQuery(fmt.Sprintf("SELECT ts FROM %s.heartbeat WHERE keyspaceShard='%s'", "_vt", tr.keyspaceShard), &sqltypes.Result{ + Fields: []*querypb.Field{ + {Name: "ts", Type: sqltypes.Int64}, + }, + Rows: [][]sqltypes.Value{{ + sqltypes.NewInt64(now.Add(-10 * time.Second).UnixNano()), + }}, + }) + + cumulativeLagNs.Reset() + readErrors.Reset() + reads.Reset() + + tr.readHeartbeat() + lag, err := tr.GetLatest() + + if err != nil { + t.Fatalf("Should not be in error: %v", tr.lastKnownError) + } + if got, want := lag, 10*time.Second; got != want { + t.Fatalf("wrong latest lag: got = %v, want = %v", tr.lastKnownLag, want) + } + if got, want := cumulativeLagNs.Get(), 10*time.Second.Nanoseconds(); got != want { + t.Fatalf("wrong cumulative lag: got = %v, want = %v", got, want) + } + if got, want := reads.Get(), int64(1); got != want { + t.Fatalf("wrong read count: got = %v, want = %v", got, want) + } + if got, want := readErrors.Get(), int64(0); got != want { + t.Fatalf("wrong read error count: got = %v, want = %v", got, want) + } +} + +// TestReaderReadHeartbeatError tests that we properly account for errors +// encountered in the reading of heartbeat. +func TestReaderReadHeartbeatError(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + tr := newReader(db, mockNowFunc) + defer tr.Close() + + cumulativeLagNs.Reset() + readErrors.Reset() + + tr.readHeartbeat() + lag, err := tr.GetLatest() + + if err == nil { + t.Fatalf("Should be in error: %v", tr.lastKnownError) + } + if got, want := lag, 0*time.Second; got != want { + t.Fatalf("wrong lastKnownLag: got = %v, want = %v", got, want) + } + if got, want := cumulativeLagNs.Get(), int64(0); got != want { + t.Fatalf("wrong cumulative lag: got = %v, want = %v", got, want) + } + if got, want := readErrors.Get(), int64(1); got != want { + t.Fatalf("wrong read error count: got = %v, want = %v", got, want) + } +} + +func newReader(db *fakesqldb.DB, nowFunc func() time.Time) *heartbeatReader { + config := tabletenv.NewDefaultConfig() + config.HeartbeatIntervalSeconds = 1 + params, _ := db.ConnParams().MysqlParams() + cp := *params + dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") + + tr := newHeartbeatReader(tabletenv.NewEnv(config, "ReaderTest")) + tr.keyspaceShard = "test:0" + tr.now = nowFunc + tr.pool.Open(dbc.AppWithDB(), dbc.DbaWithDB(), dbc.AppDebugWithDB()) + + return tr +} diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go new file mode 100644 index 00000000000..a9b5548108b --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -0,0 +1,74 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "sync" + + "vitess.io/vitess/go/stats" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +) + +type method int + +const ( + none = method(iota) + polling + heartbeat +) + +var ( + // HeartbeatWrites keeps a count of the number of heartbeats written over time. + writes = stats.NewCounter("HeartbeatWrites", "Count of heartbeats written over time") + // HeartbeatWriteErrors keeps a count of errors encountered while writing heartbeats. + writeErrors = stats.NewCounter("HeartbeatWriteErrors", "Count of errors encountered while writing heartbeats") + // HeartbeatReads keeps a count of the number of heartbeats read over time. + reads = stats.NewCounter("HeartbeatReads", "Count of heartbeats read over time") + // HeartbeatReadErrors keeps a count of errors encountered while reading heartbeats. + readErrors = stats.NewCounter("HeartbeatReadErrors", "Count of errors encountered while reading heartbeats") + // HeartbeatCumulativeLagNs is incremented by the current lag at each heartbeat read interval. Plotting this + // over time allows calculating of a rolling average lag. + cumulativeLagNs = stats.NewCounter("HeartbeatCumulativeLagNs", "Incremented by the current lag at each heartbeat read interval") + // HeartbeatCurrentLagNs is a point-in-time calculation of the lag, updated at each heartbeat read interval. + currentLagNs = stats.NewGauge("HeartbeatCurrentLagNs", "Point in time calculation of the heartbeat lag") +) + +// ReplTracker tracks replication lag. +type ReplTracker struct { + mu sync.Mutex + isMaster bool + method method + + hw *heartbeatWriter + hr *heartbeatReader +} + +// NewReplTracker creates a new ReplTracker. +func NewReplTracker(env tabletenv.Env, alias topodatapb.TabletAlias) *ReplTracker { + return &ReplTracker{ + hw: newHeartbeatWriter(env, alias), + hr: newHeartbeatReader(env), + } +} + +// InitDBConfig initializes the target name. +func (rt *ReplTracker) InitDBConfig(target querypb.Target) { + rt.hw.InitDBConfig(target) + rt.hr.InitDBConfig(target) +} diff --git a/go/vt/vttablet/tabletserver/repltracker/writer.go b/go/vt/vttablet/tabletserver/repltracker/writer.go new file mode 100644 index 00000000000..db8433bbf0b --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/writer.go @@ -0,0 +1,183 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "fmt" + "sync" + "time" + + "vitess.io/vitess/go/vt/withddl" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/timer" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +const ( + sqlCreateSidecarDB = "create database if not exists %s" + sqlCreateHeartbeatTable = `CREATE TABLE IF NOT EXISTS %s.heartbeat ( + keyspaceShard VARBINARY(256) NOT NULL PRIMARY KEY, + tabletUid INT UNSIGNED NOT NULL, + ts BIGINT UNSIGNED NOT NULL + ) engine=InnoDB` + sqlUpsertHeartbeat = "INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%a, %a, %a) ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)" +) + +var withDDL = withddl.New([]string{ + fmt.Sprintf(sqlCreateSidecarDB, "_vt"), + fmt.Sprintf(sqlCreateHeartbeatTable, "_vt"), +}) + +// heartbeatWriter runs on master tablets and writes heartbeats to the _vt.heartbeat +// table at a regular interval, defined by heartbeat_interval. +type heartbeatWriter struct { + env tabletenv.Env + + enabled bool + interval time.Duration + tabletAlias topodatapb.TabletAlias + keyspaceShard string + now func() time.Time + errorLog *logutil.ThrottledLogger + + mu sync.Mutex + isOpen bool + pool *connpool.Pool + ticks *timer.Timer +} + +// newHeartbeatWriter creates a new heartbeatWriter. +func newHeartbeatWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *heartbeatWriter { + config := env.Config() + if config.HeartbeatIntervalSeconds == 0 { + return &heartbeatWriter{} + } + heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9) + return &heartbeatWriter{ + env: env, + enabled: true, + tabletAlias: alias, + now: time.Now, + interval: heartbeatInterval, + ticks: timer.NewTimer(heartbeatInterval), + errorLog: logutil.NewThrottledLogger("HeartbeatWriter", 60*time.Second), + pool: connpool.NewPool(env, "HeartbeatWritePool", tabletenv.ConnPoolConfig{ + Size: 1, + IdleTimeoutSeconds: env.Config().OltpReadPool.IdleTimeoutSeconds, + }), + } +} + +// InitDBConfig initializes the target name for the heartbeatWriter. +func (w *heartbeatWriter) InitDBConfig(target querypb.Target) { + w.keyspaceShard = fmt.Sprintf("%s:%s", target.Keyspace, target.Shard) +} + +// Open sets up the heartbeatWriter's db connection and launches the ticker +// responsible for periodically writing to the heartbeat table. +func (w *heartbeatWriter) Open() { + if !w.enabled { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if w.isOpen { + return + } + + log.Info("Beginning heartbeat writes") + w.pool.Open(w.env.Config().DB.AppWithDB(), w.env.Config().DB.DbaWithDB(), w.env.Config().DB.AppDebugWithDB()) + w.ticks.Start(w.writeHeartbeat) + w.isOpen = true +} + +// Close closes the heartbeatWriter's db connection and stops the periodic ticker. +func (w *heartbeatWriter) Close() { + if !w.enabled { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if !w.isOpen { + return + } + w.ticks.Stop() + w.pool.Close() + log.Info("Stopped heartbeat writes.") + w.isOpen = false +} + +// bindHeartbeatVars takes a heartbeat write (insert or update) and +// adds the necessary fields to the query as bind vars. This is done +// to protect ourselves against a badly formed keyspace or shard name. +func (w *heartbeatWriter) bindHeartbeatVars(query string) (string, error) { + bindVars := map[string]*querypb.BindVariable{ + "ks": sqltypes.StringBindVariable(w.keyspaceShard), + "ts": sqltypes.Int64BindVariable(w.now().UnixNano()), + "uid": sqltypes.Int64BindVariable(int64(w.tabletAlias.Uid)), + } + parsed := sqlparser.BuildParsedQuery(query, "_vt", ":ts", ":uid", ":ks") + bound, err := parsed.GenerateQuery(bindVars, nil) + if err != nil { + return "", err + } + return bound, nil +} + +// writeHeartbeat updates the heartbeat row for this tablet with the current time in nanoseconds. +func (w *heartbeatWriter) writeHeartbeat() { + if err := w.write(); err != nil { + w.recordError(err) + return + } + writes.Add(1) +} + +func (w *heartbeatWriter) write() error { + defer w.env.LogError() + ctx, cancel := context.WithDeadline(context.Background(), w.now().Add(w.interval)) + defer cancel() + upsert, err := w.bindHeartbeatVars(sqlUpsertHeartbeat) + if err != nil { + return err + } + conn, err := w.pool.Get(ctx) + if err != nil { + return err + } + defer conn.Recycle() + _, err = withDDL.Exec(ctx, upsert, conn.Exec) + if err != nil { + return err + } + return nil +} + +func (w *heartbeatWriter) recordError(err error) { + w.errorLog.Errorf("%v", err) + writeErrors.Add(1) +} diff --git a/go/vt/vttablet/tabletserver/repltracker/writer_test.go b/go/vt/vttablet/tabletserver/repltracker/writer_test.go new file mode 100644 index 00000000000..30f0cabf7da --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/writer_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gotest.tools/assert" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +) + +var ( + now = time.Now() + mockNowFunc = func() time.Time { + return now + } +) + +func TestCreateSchema(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + tw := newTestWriter(db, mockNowFunc) + defer tw.Close() + writes.Reset() + + db.OrderMatters() + upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)", + "_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard) + failInsert := fakesqldb.ExpectedExecuteFetch{ + Query: upsert, + Error: mysql.NewSQLError(mysql.ERBadDb, "", "bad db error"), + } + db.AddExpectedExecuteFetch(failInsert) + db.AddExpectedQuery(fmt.Sprintf(sqlCreateSidecarDB, "_vt"), nil) + db.AddExpectedQuery(fmt.Sprintf(sqlCreateHeartbeatTable, "_vt"), nil) + db.AddExpectedQuery(upsert, nil) + + err := tw.write() + require.NoError(t, err) +} + +func TestWriteHeartbeat(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + + tw := newTestWriter(db, mockNowFunc) + upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)", + "_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard) + db.AddQuery(upsert, &sqltypes.Result{}) + + writes.Reset() + writeErrors.Reset() + + tw.writeHeartbeat() + assert.Equal(t, int64(1), writes.Get()) + assert.Equal(t, int64(0), writeErrors.Get()) +} + +func TestWriteHeartbeatError(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + + tw := newTestWriter(db, mockNowFunc) + + writes.Reset() + writeErrors.Reset() + + tw.writeHeartbeat() + assert.Equal(t, int64(0), writes.Get()) + assert.Equal(t, int64(1), writeErrors.Get()) +} + +func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *heartbeatWriter { + config := tabletenv.NewDefaultConfig() + config.HeartbeatIntervalSeconds = 1 + + params, _ := db.ConnParams().MysqlParams() + cp := *params + dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") + + tw := newHeartbeatWriter(tabletenv.NewEnv(config, "WriterTest"), topodatapb.TabletAlias{Cell: "test", Uid: 1111}) + tw.keyspaceShard = "test:0" + tw.now = nowFunc + tw.pool.Open(dbc.AppWithDB(), dbc.DbaWithDB(), dbc.AppDebugWithDB()) + + return tw +} From 2a97a0e03fe3a743c0b7cfc5dd7394c2098fad37 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 12:47:27 -0700 Subject: [PATCH 003/100] vttablet: move hc flags to tabletenv.TabletConfig Signed-off-by: Sugu Sougoumarane --- config/tablet/default.yaml | 5 +++ go/cmd/vttablet/vttablet.go | 1 + go/vt/vttablet/tabletmanager/healthcheck.go | 26 ++++++--------- .../tabletmanager/healthcheck_test.go | 32 +++++++++---------- go/vt/vttablet/tabletmanager/state_change.go | 8 ++--- go/vt/vttablet/tabletmanager/tm_init.go | 12 +++++-- .../vttablet/tabletserver/tabletenv/config.go | 27 ++++++++++++++++ .../tabletserver/tabletenv/config_test.go | 26 +++++++++++++++ 8 files changed, 99 insertions(+), 38 deletions(-) diff --git a/config/tablet/default.yaml b/config/tablet/default.yaml index 68de610693e..e6a45003af6 100644 --- a/config/tablet/default.yaml +++ b/config/tablet/default.yaml @@ -78,6 +78,11 @@ oltp: maxRows: 10000 # queryserver-config-max-result-size warnRows: 0 # queryserver-config-warn-result-size +healthcheck: + intervalSeconds: 20 # health_check_interval + degradedThresholdSeconds: 30 # degraded_threshold + unhealthyThresholdSeconds: 7200 # unhealthy_threshold + hotRowProtection: mode: disable|dryRun|enable # enable_hot_row_protection, enable_hot_row_protection_dry_run # Default value is same as txPool.size. diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index e2af7e9cb2e..aed5ca61193 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -71,6 +71,7 @@ func main() { // config and mycnf intializations are intertwined. config, mycnf := initConfig(tabletAlias) + tabletmanager.InitConfig(config) ts := topo.Open() qsc := createTabletServer(config, ts, tabletAlias) diff --git a/go/vt/vttablet/tabletmanager/healthcheck.go b/go/vt/vttablet/tabletmanager/healthcheck.go index a1631661646..93e66fff76f 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck.go +++ b/go/vt/vttablet/tabletmanager/healthcheck.go @@ -28,7 +28,6 @@ package tabletmanager // we don't need to do that any more. import ( - "flag" "fmt" "html/template" "time" @@ -42,15 +41,10 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -const ( - defaultDegradedThreshold = time.Duration(30 * time.Second) - defaultUnhealthyThreshold = time.Duration(2 * time.Hour) -) - var ( - healthCheckInterval = flag.Duration("health_check_interval", 20*time.Second, "Interval between health checks") - degradedThreshold = flag.Duration("degraded_threshold", defaultDegradedThreshold, "replication lag after which a replica is considered degraded (only used in status UI)") - unhealthyThreshold = flag.Duration("unhealthy_threshold", defaultUnhealthyThreshold, "replication lag after which a replica is considered unhealthy") + healthCheckInterval = 20 * time.Second + degradedThreshold = 30 * time.Second + unhealthyThreshold = 2 * time.Hour ) // HealthRecord records one run of the health checker. @@ -67,7 +61,7 @@ func (r *HealthRecord) Class() string { switch { case r.Error != nil: return "unhealthy" - case r.ReplicationDelay > *degradedThreshold: + case r.ReplicationDelay > degradedThreshold: return "unhappy" default: return "healthy" @@ -79,7 +73,7 @@ func (r *HealthRecord) HTML() template.HTML { switch { case r.Error != nil: return template.HTML(fmt.Sprintf("unhealthy: %v", r.Error)) - case r.ReplicationDelay > *degradedThreshold: + case r.ReplicationDelay > degradedThreshold: return template.HTML(fmt.Sprintf("unhappy: %v behind on replication", r.ReplicationDelay)) default: html := "healthy" @@ -95,7 +89,7 @@ func (r *HealthRecord) HTML() template.HTML { // Degraded returns true if the replication delay is beyond degradedThreshold. func (r *HealthRecord) Degraded() bool { - return r.ReplicationDelay > *degradedThreshold + return r.ReplicationDelay > degradedThreshold } // ErrorString returns Error as a string. @@ -143,8 +137,8 @@ func (tm *TabletManager) initHealthCheck() { registerReplicationReporter(tm) registerHeartbeatReporter(tm.QueryServiceControl) - log.Infof("Starting periodic health check every %v", *healthCheckInterval) - t := timer.NewTimer(*healthCheckInterval) + log.Infof("Starting periodic health check every %v", healthCheckInterval) + t := timer.NewTimer(healthCheckInterval) servenv.OnTermSync(func() { // When we enter lameduck mode, we want to not call // the health check any more. After this returns, we @@ -215,11 +209,11 @@ func (tm *TabletManager) runHealthCheckLocked() { // delay. Use a maximum delay, so we can let vtgate // find the right replica, instead of erroring out. // (this works as the check below is a strict > operator). - replicationDelay = *unhealthyThreshold + replicationDelay = unhealthyThreshold healthErr = nil } if healthErr == nil { - if replicationDelay > *unhealthyThreshold { + if replicationDelay > unhealthyThreshold { healthErr = fmt.Errorf("reported replication lag: %v higher than unhealthy threshold: %v", replicationDelay.Seconds(), unhealthyThreshold.Seconds()) } } diff --git a/go/vt/vttablet/tabletmanager/healthcheck_test.go b/go/vt/vttablet/tabletmanager/healthcheck_test.go index 4f89f228c84..e941d808948 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck_test.go +++ b/go/vt/vttablet/tabletmanager/healthcheck_test.go @@ -65,18 +65,18 @@ func TestHealthRecordDeduplication(t *testing.T) { duplicate: true, }, { - left: &HealthRecord{Time: now, ReplicationDelay: defaultDegradedThreshold / 2}, - right: &HealthRecord{Time: later, ReplicationDelay: defaultDegradedThreshold / 3}, + left: &HealthRecord{Time: now, ReplicationDelay: degradedThreshold / 2}, + right: &HealthRecord{Time: later, ReplicationDelay: degradedThreshold / 3}, duplicate: true, }, { - left: &HealthRecord{Time: now, ReplicationDelay: defaultDegradedThreshold / 2}, - right: &HealthRecord{Time: later, ReplicationDelay: defaultDegradedThreshold * 2}, + left: &HealthRecord{Time: now, ReplicationDelay: degradedThreshold / 2}, + right: &HealthRecord{Time: later, ReplicationDelay: degradedThreshold * 2}, duplicate: false, }, { - left: &HealthRecord{Time: now, Error: errors.New("foo"), ReplicationDelay: defaultDegradedThreshold * 2}, - right: &HealthRecord{Time: later, ReplicationDelay: defaultDegradedThreshold * 2}, + left: &HealthRecord{Time: now, Error: errors.New("foo"), ReplicationDelay: degradedThreshold * 2}, + right: &HealthRecord{Time: later, ReplicationDelay: degradedThreshold * 2}, duplicate: false, }, } @@ -102,11 +102,11 @@ func TestHealthRecordClass(t *testing.T) { state: "unhealthy", }, { - r: &HealthRecord{ReplicationDelay: defaultDegradedThreshold * 2}, + r: &HealthRecord{ReplicationDelay: degradedThreshold * 2}, state: "unhappy", }, { - r: &HealthRecord{ReplicationDelay: defaultDegradedThreshold / 2}, + r: &HealthRecord{ReplicationDelay: degradedThreshold / 2}, state: "healthy", }, } @@ -272,7 +272,7 @@ func TestHealthCheckControlsQueryService(t *testing.T) { // healthcheck reports health.ErrReplicationNotRunning is still considered // healthy with high replication lag. func TestErrReplicationNotRunningIsHealthy(t *testing.T) { - *unhealthyThreshold = 10 * time.Minute + unhealthyThreshold = 10 * time.Minute ctx := context.Background() tm := createTestTM(ctx, t, nil) @@ -869,7 +869,7 @@ func TestStateChangeImmediateHealthBroadcast(t *testing.T) { func TestOldHealthCheck(t *testing.T) { ctx := context.Background() tm := createTestTM(ctx, t, nil) - *healthCheckInterval = 20 * time.Second + healthCheckInterval = 20 * time.Second tm._healthy = nil // last health check time is now, we're good @@ -879,13 +879,13 @@ func TestOldHealthCheck(t *testing.T) { } // last health check time is 2x interval ago, we're good - tm._healthyTime = time.Now().Add(-2 * *healthCheckInterval) + tm._healthyTime = time.Now().Add(-2 * healthCheckInterval) if _, healthy := tm.Healthy(); healthy != nil { t.Errorf("Healthy returned unexpected error: %v", healthy) } // last health check time is 4x interval ago, we're not good - tm._healthyTime = time.Now().Add(-4 * *healthCheckInterval) + tm._healthyTime = time.Now().Add(-4 * healthCheckInterval) if _, healthy := tm.Healthy(); healthy == nil || !strings.Contains(healthy.Error(), "last health check is too old") { t.Errorf("Healthy returned wrong error: %v", healthy) } @@ -897,8 +897,8 @@ func TestBackupStateChange(t *testing.T) { ctx := context.Background() tm := createTestTM(ctx, t, nil) - *degradedThreshold = 7 * time.Second - *unhealthyThreshold = 15 * time.Second + degradedThreshold = 7 * time.Second + unhealthyThreshold = 15 * time.Second if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { t.Fatal(err) @@ -945,8 +945,8 @@ func TestRestoreStateChange(t *testing.T) { ctx := context.Background() tm := createTestTM(ctx, t, nil) - *degradedThreshold = 7 * time.Second - *unhealthyThreshold = 15 * time.Second + degradedThreshold = 7 * time.Second + unhealthyThreshold = 15 * time.Second if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { t.Fatal(err) diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 95221562c1a..4b41d57238f 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -117,8 +117,8 @@ func (tm *TabletManager) broadcastHealth() { stats.HealthError = healthError.Error() } else { timeSinceLastCheck := time.Since(healthyTime) - if timeSinceLastCheck > *healthCheckInterval*3 { - stats.HealthError = fmt.Sprintf("last health check is too old: %s > %s", timeSinceLastCheck, *healthCheckInterval*3) + if timeSinceLastCheck > healthCheckInterval*3 { + stats.HealthError = fmt.Sprintf("last health check is too old: %s > %s", timeSinceLastCheck, healthCheckInterval*3) } } var ts int64 @@ -126,7 +126,7 @@ func (tm *TabletManager) broadcastHealth() { if !terTime.IsZero() { ts = terTime.Unix() } - go tm.QueryServiceControl.BroadcastHealth(ts, stats, *healthCheckInterval*3) + go tm.QueryServiceControl.BroadcastHealth(ts, stats, healthCheckInterval*3) } // refreshTablet needs to be run after an action may have changed the current @@ -218,7 +218,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable tm.mutex.Lock() tm._replicationDelay = replicationDelay tm.mutex.Unlock() - if tm._replicationDelay > *unhealthyThreshold { + if tm._replicationDelay > unhealthyThreshold { allowQuery = false disallowQueryReason = "replica tablet with unhealthy replication lag" } diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 98de6f68c9c..dc4106d522e 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -67,6 +67,7 @@ import ( "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tabletserver" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -229,6 +230,13 @@ type TabletManager struct { isPublishing bool } +// InitConfig is a temp function to keep things working during the refactor. +func InitConfig(config *tabletenv.TabletConfig) { + healthCheckInterval = time.Duration(config.Healthcheck.IntervalSeconds * 1e9) + degradedThreshold = time.Duration(config.Healthcheck.DegradedThresholdSeconds * 1e9) + unhealthyThreshold = time.Duration(config.Healthcheck.UnhealthyThresholdSeconds * 1e9) +} + // BuildTabletFromInput builds a tablet record from input parameters. func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32) (*topodatapb.Tablet, error) { hostname := *tabletHostname @@ -757,8 +765,8 @@ func (tm *TabletManager) Healthy() (time.Duration, error) { healthy := tm._healthy if healthy == nil { timeSinceLastCheck := time.Since(tm._healthyTime) - if timeSinceLastCheck > *healthCheckInterval*3 { - healthy = fmt.Errorf("last health check is too old: %s > %s", timeSinceLastCheck, *healthCheckInterval*3) + if timeSinceLastCheck > healthCheckInterval*3 { + healthy = fmt.Errorf("last health check is too old: %s > %s", timeSinceLastCheck, healthCheckInterval*3) } } diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index e454b49eac0..ce81f4292c0 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -72,6 +72,9 @@ var ( enableConsolidatorReplicas bool enableHeartbeat bool heartbeatInterval time.Duration + healthCheckInterval time.Duration + degradedThreshold time.Duration + unhealthyThreshold time.Duration ) func init() { @@ -140,6 +143,10 @@ func init() { flag.BoolVar(&enableConsolidator, "enable-consolidator", true, "This option enables the query consolidator.") flag.BoolVar(&enableConsolidatorReplicas, "enable-consolidator-replicas", false, "This option enables the query consolidator only on replicas.") flag.BoolVar(¤tConfig.CacheResultFields, "enable-query-plan-field-caching", defaultConfig.CacheResultFields, "This option fetches & caches fields (columns) when storing query plans") + + flag.DurationVar(&healthCheckInterval, "health_check_interval", 20*time.Second, "Interval between health checks") + flag.DurationVar(°radedThreshold, "degraded_threshold", 30*time.Second, "replication lag after which a replica is considered degraded (only used in status UI)") + flag.DurationVar(&unhealthyThreshold, "unhealthy_threshold", 2*time.Hour, "replication lag after which a replica is considered unhealthy") } // Init must be called after flag.Parse, and before doing any other operations. @@ -172,6 +179,10 @@ func Init() { currentConfig.HeartbeatIntervalSeconds = float64(heartbeatInterval) / float64(time.Second) } + currentConfig.Healthcheck.IntervalSeconds = float64(healthCheckInterval) / float64(time.Second) + currentConfig.Healthcheck.DegradedThresholdSeconds = float64(degradedThreshold) / float64(time.Second) + currentConfig.Healthcheck.UnhealthyThresholdSeconds = float64(unhealthyThreshold) / float64(time.Second) + switch *streamlog.QueryLogFormat { case streamlog.QueryLogFormatText: case streamlog.QueryLogFormatJSON: @@ -199,6 +210,8 @@ type TabletConfig struct { Oltp OltpConfig `json:"oltp,omitempty"` HotRowProtection HotRowProtectionConfig `json:"hotRowProtection,omitempty"` + Healthcheck HealthcheckConfig `json:"healthcheck,omitempty"` + Consolidator string `json:"consolidator,omitempty"` HeartbeatIntervalSeconds float64 `json:"heartbeatIntervalSeconds,omitempty"` ShutdownGracePeriodSeconds float64 `json:"shutdownGracePeriodSeconds,omitempty"` @@ -255,6 +268,13 @@ type HotRowProtectionConfig struct { MaxConcurrency int `json:"maxConcurrency,omitempty"` } +// HealthcheckConfig contains the config for healthcheck. +type HealthcheckConfig struct { + IntervalSeconds float64 `json:"intervalSeconds,omitempty"` + DegradedThresholdSeconds float64 `json:"degradedThresholdSeconds,omitempty"` + UnhealthyThresholdSeconds float64 `json:"unhealthyThresholdSeconds,omitempty"` +} + // TransactionLimitConfig captures configuration of transaction pool slots // limiter configuration. type TransactionLimitConfig struct { @@ -336,6 +356,8 @@ func (c *TabletConfig) verifyTransactionLimitConfig() error { return nil } +// Some of these values are for documentation purposes. +// They actually get overwritten during Init. var defaultConfig = TabletConfig{ OltpReadPool: ConnPoolConfig{ Size: 16, @@ -357,6 +379,11 @@ var defaultConfig = TabletConfig{ TxTimeoutSeconds: 30, MaxRows: 10000, }, + Healthcheck: HealthcheckConfig{ + IntervalSeconds: 20, + DegradedThresholdSeconds: 30, + UnhealthyThresholdSeconds: 7200, + }, HotRowProtection: HotRowProtectionConfig{ Mode: Disable, // Default value is the same as TxPool.Size. diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index 9a4dfb2eee9..0a871fb7b3a 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -63,6 +63,7 @@ func TestConfigParse(t *testing.T) { repl: password: '****' socket: a +healthcheck: {} hotRowProtection: {} olapReadPool: {} oltp: {} @@ -104,6 +105,10 @@ func TestDefaultConfig(t *testing.T) { require.NoError(t, err) want := `cacheResultFields: true consolidator: enable +healthcheck: + degradedThresholdSeconds: 30 + intervalSeconds: 20 + unhealthyThresholdSeconds: 7200 hotRowProtection: maxConcurrency: 5 maxGlobalQueueSize: 1000 @@ -202,6 +207,9 @@ func TestFlags(t *testing.T) { want.TxPool.IdleTimeoutSeconds = 1800 want.HotRowProtection.Mode = Disable want.Consolidator = Enable + want.Healthcheck.IntervalSeconds = 20 + want.Healthcheck.DegradedThresholdSeconds = 30 + want.Healthcheck.UnhealthyThresholdSeconds = 7200 assert.Equal(t, want.DB, currentConfig.DB) assert.Equal(t, want, currentConfig) @@ -265,4 +273,22 @@ func TestFlags(t *testing.T) { Init() want.HeartbeatIntervalSeconds = 0 assert.Equal(t, want, currentConfig) + + healthCheckInterval = 1 * time.Second + currentConfig.Healthcheck.IntervalSeconds = 0 + Init() + want.Healthcheck.IntervalSeconds = 1 + assert.Equal(t, want, currentConfig) + + degradedThreshold = 2 * time.Second + currentConfig.Healthcheck.DegradedThresholdSeconds = 0 + Init() + want.Healthcheck.DegradedThresholdSeconds = 2 + assert.Equal(t, want, currentConfig) + + unhealthyThreshold = 3 * time.Second + currentConfig.Healthcheck.UnhealthyThresholdSeconds = 0 + Init() + want.Healthcheck.UnhealthyThresholdSeconds = 3 + assert.Equal(t, want, currentConfig) } From 4b08c81a01f9e09544acc97ff080629d3fbe09b7 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 13:53:26 -0700 Subject: [PATCH 004/100] vttablet: type safe Seconds instead of float64 Introduce a new Seconds type with explicit conversions to and from time.Duration. Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/heartbeat/reader.go | 2 +- go/vt/vttablet/heartbeat/writer.go | 2 +- go/vt/vttablet/tabletmanager/tm_init.go | 6 +-- go/vt/vttablet/tabletserver/connpool/pool.go | 4 +- .../tabletserver/query_engine_test.go | 6 +-- .../tabletserver/repltracker/reader.go | 2 +- .../tabletserver/repltracker/writer.go | 2 +- go/vt/vttablet/tabletserver/schema/engine.go | 2 +- .../tabletserver/schema/engine_test.go | 8 +-- .../vttablet/tabletserver/tabletenv/config.go | 48 ++++++++--------- .../tabletserver/tabletenv/seconds.go | 41 +++++++++++++++ .../tabletserver/tabletenv/seconds_test.go | 52 +++++++++++++++++++ go/vt/vttablet/tabletserver/tabletserver.go | 2 +- go/vt/vttablet/tabletserver/tx_engine.go | 4 +- go/vt/vttablet/tabletserver/tx_pool.go | 2 +- 15 files changed, 138 insertions(+), 45 deletions(-) create mode 100644 go/vt/vttablet/tabletserver/tabletenv/seconds.go create mode 100644 go/vt/vttablet/tabletserver/tabletenv/seconds_test.go diff --git a/go/vt/vttablet/heartbeat/reader.go b/go/vt/vttablet/heartbeat/reader.go index e6b4afcb096..d2598295fa5 100644 --- a/go/vt/vttablet/heartbeat/reader.go +++ b/go/vt/vttablet/heartbeat/reader.go @@ -74,7 +74,7 @@ func NewReader(env tabletenv.Env) *Reader { return &Reader{} } - heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9) + heartbeatInterval := config.HeartbeatIntervalSeconds.Get() return &Reader{ env: env, enabled: true, diff --git a/go/vt/vttablet/heartbeat/writer.go b/go/vt/vttablet/heartbeat/writer.go index 0c212133ff9..2cead8afd0b 100644 --- a/go/vt/vttablet/heartbeat/writer.go +++ b/go/vt/vttablet/heartbeat/writer.go @@ -76,7 +76,7 @@ func NewWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *Writer { if config.HeartbeatIntervalSeconds == 0 { return &Writer{} } - heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9) + heartbeatInterval := config.HeartbeatIntervalSeconds.Get() return &Writer{ env: env, enabled: true, diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index dc4106d522e..a4d5244fa5f 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -232,9 +232,9 @@ type TabletManager struct { // InitConfig is a temp function to keep things working during the refactor. func InitConfig(config *tabletenv.TabletConfig) { - healthCheckInterval = time.Duration(config.Healthcheck.IntervalSeconds * 1e9) - degradedThreshold = time.Duration(config.Healthcheck.DegradedThresholdSeconds * 1e9) - unhealthyThreshold = time.Duration(config.Healthcheck.UnhealthyThresholdSeconds * 1e9) + healthCheckInterval = config.Healthcheck.IntervalSeconds.Get() + degradedThreshold = config.Healthcheck.DegradedThresholdSeconds.Get() + unhealthyThreshold = config.Healthcheck.UnhealthyThresholdSeconds.Get() } // BuildTabletFromInput builds a tablet record from input parameters. diff --git a/go/vt/vttablet/tabletserver/connpool/pool.go b/go/vt/vttablet/tabletserver/connpool/pool.go index d5f6a347bf0..26e464d2ccf 100644 --- a/go/vt/vttablet/tabletserver/connpool/pool.go +++ b/go/vt/vttablet/tabletserver/connpool/pool.go @@ -62,13 +62,13 @@ type Pool struct { // NewPool creates a new Pool. The name is used // to publish stats only. func NewPool(env tabletenv.Env, name string, cfg tabletenv.ConnPoolConfig) *Pool { - idleTimeout := time.Duration(cfg.IdleTimeoutSeconds * 1e9) + idleTimeout := cfg.IdleTimeoutSeconds.Get() cp := &Pool{ env: env, name: name, capacity: cfg.Size, prefillParallelism: cfg.PrefillParallelism, - timeout: time.Duration(cfg.TimeoutSeconds * 1e9), + timeout: cfg.TimeoutSeconds.Get(), idleTimeout: idleTimeout, waiterCap: int64(cfg.MaxWaiters), dbaPool: dbconnpool.NewConnectionPool("", 1, idleTimeout, 0), diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index 902ce858202..0a24b1d96d3 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -277,9 +277,9 @@ func newTestQueryEngine(queryCacheSize int, idleTimeout time.Duration, strict bo config := tabletenv.NewDefaultConfig() config.DB = dbcfgs config.QueryCacheSize = queryCacheSize - config.OltpReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9) - config.OlapReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9) - config.TxPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9) + config.OltpReadPool.IdleTimeoutSeconds.Set(idleTimeout) + config.OlapReadPool.IdleTimeoutSeconds.Set(idleTimeout) + config.TxPool.IdleTimeoutSeconds.Set(idleTimeout) env := tabletenv.NewEnv(config, "TabletServerTest") se := schema.NewEngine(env) qe := NewQueryEngine(env, se) diff --git a/go/vt/vttablet/tabletserver/repltracker/reader.go b/go/vt/vttablet/tabletserver/repltracker/reader.go index 5a2d950d329..fdb051bc5e6 100644 --- a/go/vt/vttablet/tabletserver/repltracker/reader.go +++ b/go/vt/vttablet/tabletserver/repltracker/reader.go @@ -73,7 +73,7 @@ func newHeartbeatReader(env tabletenv.Env) *heartbeatReader { return &heartbeatReader{} } - heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9) + heartbeatInterval := config.HeartbeatIntervalSeconds.Get() return &heartbeatReader{ env: env, enabled: true, diff --git a/go/vt/vttablet/tabletserver/repltracker/writer.go b/go/vt/vttablet/tabletserver/repltracker/writer.go index db8433bbf0b..61d1c4d5bf9 100644 --- a/go/vt/vttablet/tabletserver/repltracker/writer.go +++ b/go/vt/vttablet/tabletserver/repltracker/writer.go @@ -76,7 +76,7 @@ func newHeartbeatWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *heartb if config.HeartbeatIntervalSeconds == 0 { return &heartbeatWriter{} } - heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9) + heartbeatInterval := config.HeartbeatIntervalSeconds.Get() return &heartbeatWriter{ env: env, enabled: true, diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 5a36ac1eb46..63f68dd695f 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -75,7 +75,7 @@ type Engine struct { // NewEngine creates a new Engine. func NewEngine(env tabletenv.Env) *Engine { - reloadTime := time.Duration(env.Config().SchemaReloadIntervalSeconds * 1e9) + reloadTime := env.Config().SchemaReloadIntervalSeconds.Get() se := &Engine{ env: env, // We need three connections: one for the reloader, one for diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index bfff13f3730..1a06de562b6 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -344,10 +344,10 @@ func TestStatsURL(t *testing.T) { func newEngine(queryCacheSize int, reloadTime time.Duration, idleTimeout time.Duration, strict bool, db *fakesqldb.DB) *Engine { config := tabletenv.NewDefaultConfig() config.QueryCacheSize = queryCacheSize - config.SchemaReloadIntervalSeconds = float64(reloadTime) / 1e9 - config.OltpReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9) - config.OlapReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9) - config.TxPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9) + config.SchemaReloadIntervalSeconds.Set(reloadTime) + config.OltpReadPool.IdleTimeoutSeconds.Set(idleTimeout) + config.OlapReadPool.IdleTimeoutSeconds.Set(idleTimeout) + config.TxPool.IdleTimeoutSeconds.Set(idleTimeout) se := NewEngine(tabletenv.NewEnv(config, "SchemaTest")) se.InitDBConfig(newDBConfigs(db).DbaWithDB()) return se diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index ce81f4292c0..01af523aff7 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -88,8 +88,8 @@ func init() { flag.IntVar(¤tConfig.TxPool.PrefillParallelism, "queryserver-config-transaction-prefill-parallelism", defaultConfig.TxPool.PrefillParallelism, "query server transaction prefill parallelism, a non-zero value will prefill the pool using the specified parallism.") flag.IntVar(¤tConfig.MessagePostponeParallelism, "queryserver-config-message-postpone-cap", defaultConfig.MessagePostponeParallelism, "query server message postpone cap is the maximum number of messages that can be postponed at any given time. Set this number to substantially lower than transaction cap, so that the transaction pool isn't exhausted by the message subsystem.") flag.IntVar(&deprecatedFoundRowsPoolSize, "client-found-rows-pool-size", 0, "DEPRECATED: queryserver-config-transaction-cap will be used instead.") - flag.Float64Var(¤tConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") - flag.Float64Var(¤tConfig.ShutdownGracePeriodSeconds, "transaction_shutdown_grace_period", defaultConfig.ShutdownGracePeriodSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.") + SecondsVar(¤tConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") + SecondsVar(¤tConfig.ShutdownGracePeriodSeconds, "transaction_shutdown_grace_period", defaultConfig.ShutdownGracePeriodSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.") flag.IntVar(¤tConfig.Oltp.MaxRows, "queryserver-config-max-result-size", defaultConfig.Oltp.MaxRows, "query server max result size, maximum number of rows allowed to return from vttablet for non-streaming queries.") flag.IntVar(¤tConfig.Oltp.WarnRows, "queryserver-config-warn-result-size", defaultConfig.Oltp.WarnRows, "query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this") flag.IntVar(&deprecatedMaxDMLRows, "queryserver-config-max-dml-rows", 0, "query server max dml rows per statement, maximum number of rows allowed to return at a time for an update or delete with either 1) an equality where clauses on primary keys, or 2) a subselect statement. For update and delete statements in above two categories, vttablet will split the original query into multiple small queries based on this configuration value. ") @@ -98,12 +98,12 @@ func init() { flag.IntVar(¤tConfig.StreamBufferSize, "queryserver-config-stream-buffer-size", defaultConfig.StreamBufferSize, "query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size.") flag.IntVar(¤tConfig.QueryCacheSize, "queryserver-config-query-cache-size", defaultConfig.QueryCacheSize, "query server query cache size, maximum number of queries to be cached. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.") - flag.Float64Var(¤tConfig.SchemaReloadIntervalSeconds, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadIntervalSeconds, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.") - flag.Float64Var(¤tConfig.Oltp.QueryTimeoutSeconds, "queryserver-config-query-timeout", defaultConfig.Oltp.QueryTimeoutSeconds, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.") - flag.Float64Var(¤tConfig.OltpReadPool.TimeoutSeconds, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.TimeoutSeconds, "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.") - flag.Float64Var(¤tConfig.OlapReadPool.TimeoutSeconds, "queryserver-config-stream-pool-timeout", defaultConfig.OlapReadPool.TimeoutSeconds, "query server stream pool timeout (in seconds), it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout.") - flag.Float64Var(¤tConfig.TxPool.TimeoutSeconds, "queryserver-config-txpool-timeout", defaultConfig.TxPool.TimeoutSeconds, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full") - flag.Float64Var(¤tConfig.OltpReadPool.IdleTimeoutSeconds, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeoutSeconds, "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.") + SecondsVar(¤tConfig.SchemaReloadIntervalSeconds, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadIntervalSeconds, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.") + SecondsVar(¤tConfig.Oltp.QueryTimeoutSeconds, "queryserver-config-query-timeout", defaultConfig.Oltp.QueryTimeoutSeconds, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.") + SecondsVar(¤tConfig.OltpReadPool.TimeoutSeconds, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.TimeoutSeconds, "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.") + SecondsVar(¤tConfig.OlapReadPool.TimeoutSeconds, "queryserver-config-stream-pool-timeout", defaultConfig.OlapReadPool.TimeoutSeconds, "query server stream pool timeout (in seconds), it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout.") + SecondsVar(¤tConfig.TxPool.TimeoutSeconds, "queryserver-config-txpool-timeout", defaultConfig.TxPool.TimeoutSeconds, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full") + SecondsVar(¤tConfig.OltpReadPool.IdleTimeoutSeconds, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeoutSeconds, "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.") flag.IntVar(¤tConfig.OltpReadPool.MaxWaiters, "queryserver-config-query-pool-waiter-cap", defaultConfig.OltpReadPool.MaxWaiters, "query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection") flag.IntVar(¤tConfig.TxPool.MaxWaiters, "queryserver-config-txpool-waiter-cap", defaultConfig.TxPool.MaxWaiters, "query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection") // tableacl related configurations. @@ -117,7 +117,7 @@ func init() { flag.BoolVar(&deprecatedAutocommit, "enable-autocommit", true, "This flag is deprecated. Autocommit is always allowed.") flag.BoolVar(¤tConfig.TwoPCEnable, "twopc_enable", defaultConfig.TwoPCEnable, "if the flag is on, 2pc is enabled. Other 2pc flags must be supplied.") flag.StringVar(¤tConfig.TwoPCCoordinatorAddress, "twopc_coordinator_address", defaultConfig.TwoPCCoordinatorAddress, "address of the (VTGate) process(es) that will be used to notify of abandoned transactions.") - flag.Float64Var(¤tConfig.TwoPCAbandonAge, "twopc_abandon_age", defaultConfig.TwoPCAbandonAge, "time in seconds. Any unresolved transaction older than this time will be sent to the coordinator to be resolved.") + SecondsVar(¤tConfig.TwoPCAbandonAge, "twopc_abandon_age", defaultConfig.TwoPCAbandonAge, "time in seconds. Any unresolved transaction older than this time will be sent to the coordinator to be resolved.") flag.BoolVar(¤tConfig.EnableTxThrottler, "enable-tx-throttler", defaultConfig.EnableTxThrottler, "If true replication-lag-based throttling on transactions will be enabled.") flag.StringVar(¤tConfig.TxThrottlerConfig, "tx-throttler-config", defaultConfig.TxThrottlerConfig, "The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message") flagutil.StringListVar(¤tConfig.TxThrottlerHealthCheckCells, "tx-throttler-healthcheck-cells", defaultConfig.TxThrottlerHealthCheckCells, "A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler.") @@ -176,12 +176,12 @@ func Init() { } if enableHeartbeat { - currentConfig.HeartbeatIntervalSeconds = float64(heartbeatInterval) / float64(time.Second) + currentConfig.HeartbeatIntervalSeconds.Set(heartbeatInterval) } - currentConfig.Healthcheck.IntervalSeconds = float64(healthCheckInterval) / float64(time.Second) - currentConfig.Healthcheck.DegradedThresholdSeconds = float64(degradedThreshold) / float64(time.Second) - currentConfig.Healthcheck.UnhealthyThresholdSeconds = float64(unhealthyThreshold) / float64(time.Second) + currentConfig.Healthcheck.IntervalSeconds.Set(healthCheckInterval) + currentConfig.Healthcheck.DegradedThresholdSeconds.Set(degradedThreshold) + currentConfig.Healthcheck.UnhealthyThresholdSeconds.Set(unhealthyThreshold) switch *streamlog.QueryLogFormat { case streamlog.QueryLogFormatText: @@ -213,12 +213,12 @@ type TabletConfig struct { Healthcheck HealthcheckConfig `json:"healthcheck,omitempty"` Consolidator string `json:"consolidator,omitempty"` - HeartbeatIntervalSeconds float64 `json:"heartbeatIntervalSeconds,omitempty"` - ShutdownGracePeriodSeconds float64 `json:"shutdownGracePeriodSeconds,omitempty"` + HeartbeatIntervalSeconds Seconds `json:"heartbeatIntervalSeconds,omitempty"` + ShutdownGracePeriodSeconds Seconds `json:"shutdownGracePeriodSeconds,omitempty"` PassthroughDML bool `json:"passthroughDML,omitempty"` StreamBufferSize int `json:"streamBufferSize,omitempty"` QueryCacheSize int `json:"queryCacheSize,omitempty"` - SchemaReloadIntervalSeconds float64 `json:"schemaReloadIntervalSeconds,omitempty"` + SchemaReloadIntervalSeconds Seconds `json:"schemaReloadIntervalSeconds,omitempty"` WatchReplication bool `json:"watchReplication,omitempty"` TrackSchemaVersions bool `json:"trackSchemaVersions,omitempty"` TerseErrors bool `json:"terseErrors,omitempty"` @@ -232,7 +232,7 @@ type TabletConfig struct { TableACLExemptACL string `json:"-"` TwoPCEnable bool `json:"-"` TwoPCCoordinatorAddress string `json:"-"` - TwoPCAbandonAge float64 `json:"-"` + TwoPCAbandonAge Seconds `json:"-"` EnableTxThrottler bool `json:"-"` TxThrottlerConfig string `json:"-"` @@ -246,16 +246,16 @@ type TabletConfig struct { // ConnPoolConfig contains the config for a conn pool. type ConnPoolConfig struct { Size int `json:"size,omitempty"` - TimeoutSeconds float64 `json:"timeoutSeconds,omitempty"` - IdleTimeoutSeconds float64 `json:"idleTimeoutSeconds,omitempty"` + TimeoutSeconds Seconds `json:"timeoutSeconds,omitempty"` + IdleTimeoutSeconds Seconds `json:"idleTimeoutSeconds,omitempty"` PrefillParallelism int `json:"prefillParallelism,omitempty"` MaxWaiters int `json:"maxWaiters,omitempty"` } // OltpConfig contains the config for oltp settings. type OltpConfig struct { - QueryTimeoutSeconds float64 `json:"queryTimeoutSeconds,omitempty"` - TxTimeoutSeconds float64 `json:"txTimeoutSeconds,omitempty"` + QueryTimeoutSeconds Seconds `json:"queryTimeoutSeconds,omitempty"` + TxTimeoutSeconds Seconds `json:"txTimeoutSeconds,omitempty"` MaxRows int `json:"maxRpws,omitempty"` WarnRows int `json:"warnRows,omitempty"` } @@ -270,9 +270,9 @@ type HotRowProtectionConfig struct { // HealthcheckConfig contains the config for healthcheck. type HealthcheckConfig struct { - IntervalSeconds float64 `json:"intervalSeconds,omitempty"` - DegradedThresholdSeconds float64 `json:"degradedThresholdSeconds,omitempty"` - UnhealthyThresholdSeconds float64 `json:"unhealthyThresholdSeconds,omitempty"` + IntervalSeconds Seconds `json:"intervalSeconds,omitempty"` + DegradedThresholdSeconds Seconds `json:"degradedThresholdSeconds,omitempty"` + UnhealthyThresholdSeconds Seconds `json:"unhealthyThresholdSeconds,omitempty"` } // TransactionLimitConfig captures configuration of transaction pool slots diff --git a/go/vt/vttablet/tabletserver/tabletenv/seconds.go b/go/vt/vttablet/tabletserver/tabletenv/seconds.go new file mode 100644 index 00000000000..9e01622f2a5 --- /dev/null +++ b/go/vt/vttablet/tabletserver/tabletenv/seconds.go @@ -0,0 +1,41 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletenv + +import ( + "flag" + "time" +) + +// Seconds provides convenience functions for extracting +// duration from flaot64 seconds values. +type Seconds float64 + +// SecondsVar is like a flag.Float64Var, but it works for Seconds. +func SecondsVar(p *Seconds, name string, value Seconds, usage string) { + flag.Float64Var((*float64)(p), name, float64(value), usage) +} + +// Get converts Seconds to time.Duration +func (s Seconds) Get() time.Duration { + return time.Duration(s * Seconds(1*time.Second)) +} + +// Set sets the value from time.Duration +func (s *Seconds) Set(d time.Duration) { + *s = Seconds(d) / Seconds(1*time.Second) +} diff --git a/go/vt/vttablet/tabletserver/tabletenv/seconds_test.go b/go/vt/vttablet/tabletserver/tabletenv/seconds_test.go new file mode 100644 index 00000000000..98b9a59bbdc --- /dev/null +++ b/go/vt/vttablet/tabletserver/tabletenv/seconds_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletenv + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/yaml2" +) + +func TestSecondsYaml(t *testing.T) { + type testSecond struct { + Value Seconds `json:"value"` + } + + ts := testSecond{ + Value: 1, + } + gotBytes, err := yaml2.Marshal(&ts) + require.NoError(t, err) + wantBytes := "value: 1\n" + assert.Equal(t, wantBytes, string(gotBytes)) + + var gotts testSecond + err = yaml2.Unmarshal([]byte(wantBytes), &gotts) + require.NoError(t, err) + assert.Equal(t, ts, gotts) +} + +func TestSecondsGetSet(t *testing.T) { + var val Seconds + val.Set(2 * time.Second) + assert.Equal(t, Seconds(2), val) + assert.Equal(t, 2*time.Second, val.Get()) +} diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index ae1dfa4c176..e7811e5e70c 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -143,7 +143,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to exporter: exporter, stats: tabletenv.NewStats(exporter), config: config, - QueryTimeout: sync2.NewAtomicDuration(time.Duration(config.Oltp.QueryTimeoutSeconds * 1e9)), + QueryTimeout: sync2.NewAtomicDuration(config.Oltp.QueryTimeoutSeconds.Get()), TerseErrors: config.TerseErrors, enableHotRowProtection: config.HotRowProtection.Mode != tabletenv.Disable, topoServer: topoServer, diff --git a/go/vt/vttablet/tabletserver/tx_engine.go b/go/vt/vttablet/tabletserver/tx_engine.go index ab778b438b5..62c0545ad3b 100644 --- a/go/vt/vttablet/tabletserver/tx_engine.go +++ b/go/vt/vttablet/tabletserver/tx_engine.go @@ -107,7 +107,7 @@ func NewTxEngine(env tabletenv.Env) *TxEngine { config := env.Config() te := &TxEngine{ env: env, - shutdownGracePeriod: time.Duration(config.ShutdownGracePeriodSeconds * 1e9), + shutdownGracePeriod: config.ShutdownGracePeriodSeconds.Get(), reservedConnStats: env.Exporter().NewTimings("ReservedConnections", "Reserved connections stats", "operation"), } limiter := txlimiter.New(env) @@ -124,7 +124,7 @@ func NewTxEngine(env tabletenv.Env) *TxEngine { } } te.coordinatorAddress = config.TwoPCCoordinatorAddress - te.abandonAge = time.Duration(config.TwoPCAbandonAge * 1e9) + te.abandonAge = config.TwoPCAbandonAge.Get() te.ticks = timer.NewTimer(te.abandonAge / 2) // Set the prepared pool capacity to something lower than diff --git a/go/vt/vttablet/tabletserver/tx_pool.go b/go/vt/vttablet/tabletserver/tx_pool.go index ce5ae0f023b..f9598297528 100644 --- a/go/vt/vttablet/tabletserver/tx_pool.go +++ b/go/vt/vttablet/tabletserver/tx_pool.go @@ -77,7 +77,7 @@ type ( // NewTxPool creates a new TxPool. It's not operational until it's Open'd. func NewTxPool(env tabletenv.Env, limiter txlimiter.TxLimiter) *TxPool { config := env.Config() - transactionTimeout := time.Duration(config.Oltp.TxTimeoutSeconds * 1e9) + transactionTimeout := config.Oltp.TxTimeoutSeconds.Get() axp := &TxPool{ env: env, scp: NewStatefulConnPool(env), From 3af6df57390bdb8373d520e3ba026f5a5e1d6df1 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 14:45:46 -0700 Subject: [PATCH 005/100] vttablet: enable_replication_reporter -> tabletenv Signed-off-by: Sugu Sougoumarane --- config/tablet/default.yaml | 7 ++-- go/vt/vttablet/heartbeat/reader.go | 4 +-- go/vt/vttablet/heartbeat/reader_test.go | 3 +- go/vt/vttablet/heartbeat/writer.go | 4 +-- go/vt/vttablet/heartbeat/writer_test.go | 3 +- .../tabletmanager/heartbeat_reporter.go | 2 +- .../tabletmanager/replication_reporter.go | 7 ++-- go/vt/vttablet/tabletmanager/tm_init.go | 2 ++ .../tabletserver/repltracker/reader.go | 4 +-- .../tabletserver/repltracker/reader_test.go | 3 +- .../tabletserver/repltracker/writer.go | 4 +-- .../tabletserver/repltracker/writer_test.go | 3 +- .../vttablet/tabletserver/tabletenv/config.go | 32 +++++++++++++++++-- .../tabletserver/tabletenv/config_test.go | 24 ++++++++++++-- 14 files changed, 76 insertions(+), 26 deletions(-) diff --git a/config/tablet/default.yaml b/config/tablet/default.yaml index e6a45003af6..5f4583aca36 100644 --- a/config/tablet/default.yaml +++ b/config/tablet/default.yaml @@ -83,15 +83,18 @@ healthcheck: degradedThresholdSeconds: 30 # degraded_threshold unhealthyThresholdSeconds: 7200 # unhealthy_threshold +replicationTracker: + mode: disable # enable_replication_reporter + heartbeatIntervalMilliseconds: 0 # heartbeat_enable, heartbeat_interval + hotRowProtection: mode: disable|dryRun|enable # enable_hot_row_protection, enable_hot_row_protection_dry_run - # Default value is same as txPool.size. + # Recommended value: same as txPool.size. maxQueueSize: 20 # hot_row_protection_max_queue_size maxGlobalQueueSize: 1000 # hot_row_protection_max_global_queue_size maxConcurrency: 5 # hot_row_protection_concurrent_transactions consolidator: enable|disable|notOnMaster # enable-consolidator, enable-consolidator-replicas -heartbeatIntervalMilliseconds: 0 # heartbeat_enable, heartbeat_interval shutdownGracePeriodSeconds: 0 # transaction_shutdown_grace_period passthroughDML: false # queryserver-config-passthrough-dmls streamBufferSize: 32768 # queryserver-config-stream-buffer-size diff --git a/go/vt/vttablet/heartbeat/reader.go b/go/vt/vttablet/heartbeat/reader.go index d2598295fa5..bbdcfd7f881 100644 --- a/go/vt/vttablet/heartbeat/reader.go +++ b/go/vt/vttablet/heartbeat/reader.go @@ -70,11 +70,11 @@ type Reader struct { // NewReader returns a new heartbeat reader. func NewReader(env tabletenv.Env) *Reader { config := env.Config() - if config.HeartbeatIntervalSeconds == 0 { + if config.ReplicationTracker.Mode != tabletenv.Heartbeat { return &Reader{} } - heartbeatInterval := config.HeartbeatIntervalSeconds.Get() + heartbeatInterval := config.ReplicationTracker.HeartbeatIntervalSeconds.Get() return &Reader{ env: env, enabled: true, diff --git a/go/vt/vttablet/heartbeat/reader_test.go b/go/vt/vttablet/heartbeat/reader_test.go index e22d28d8094..1fcf23b61df 100644 --- a/go/vt/vttablet/heartbeat/reader_test.go +++ b/go/vt/vttablet/heartbeat/reader_test.go @@ -100,7 +100,8 @@ func TestReaderReadHeartbeatError(t *testing.T) { func newReader(db *fakesqldb.DB, nowFunc func() time.Time) *Reader { config := tabletenv.NewDefaultConfig() - config.HeartbeatIntervalSeconds = 1 + config.ReplicationTracker.Mode = tabletenv.Heartbeat + config.ReplicationTracker.HeartbeatIntervalSeconds = 1 params, _ := db.ConnParams().MysqlParams() cp := *params dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") diff --git a/go/vt/vttablet/heartbeat/writer.go b/go/vt/vttablet/heartbeat/writer.go index 2cead8afd0b..9853e803db7 100644 --- a/go/vt/vttablet/heartbeat/writer.go +++ b/go/vt/vttablet/heartbeat/writer.go @@ -73,10 +73,10 @@ type Writer struct { // NewWriter creates a new Writer. func NewWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *Writer { config := env.Config() - if config.HeartbeatIntervalSeconds == 0 { + if config.ReplicationTracker.Mode != tabletenv.Heartbeat { return &Writer{} } - heartbeatInterval := config.HeartbeatIntervalSeconds.Get() + heartbeatInterval := config.ReplicationTracker.HeartbeatIntervalSeconds.Get() return &Writer{ env: env, enabled: true, diff --git a/go/vt/vttablet/heartbeat/writer_test.go b/go/vt/vttablet/heartbeat/writer_test.go index 0d80c676236..e424644898a 100644 --- a/go/vt/vttablet/heartbeat/writer_test.go +++ b/go/vt/vttablet/heartbeat/writer_test.go @@ -94,7 +94,8 @@ func TestWriteHeartbeatError(t *testing.T) { func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *Writer { config := tabletenv.NewDefaultConfig() - config.HeartbeatIntervalSeconds = 1 + config.ReplicationTracker.Mode = tabletenv.Heartbeat + config.ReplicationTracker.HeartbeatIntervalSeconds = 1 params, _ := db.ConnParams().MysqlParams() cp := *params diff --git a/go/vt/vttablet/tabletmanager/heartbeat_reporter.go b/go/vt/vttablet/tabletmanager/heartbeat_reporter.go index 11b36752ca2..7697e376397 100644 --- a/go/vt/vttablet/tabletmanager/heartbeat_reporter.go +++ b/go/vt/vttablet/tabletmanager/heartbeat_reporter.go @@ -34,7 +34,7 @@ type Reporter struct { // RegisterReporter registers the heartbeat reader as a healthcheck reporter so that its // measurements will be picked up in healthchecks. func registerHeartbeatReporter(controller tabletserver.Controller) { - if tabletenv.NewCurrentConfig().HeartbeatIntervalSeconds == 0 { + if tabletenv.NewCurrentConfig().ReplicationTracker.Mode != tabletenv.Heartbeat { return } diff --git a/go/vt/vttablet/tabletmanager/replication_reporter.go b/go/vt/vttablet/tabletmanager/replication_reporter.go index 4ed7e3ed631..975a2254a30 100644 --- a/go/vt/vttablet/tabletmanager/replication_reporter.go +++ b/go/vt/vttablet/tabletmanager/replication_reporter.go @@ -17,7 +17,6 @@ limitations under the License. package tabletmanager import ( - "flag" "fmt" "html/template" "time" @@ -33,9 +32,7 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" ) -var ( - enableReplicationReporter = flag.Bool("enable_replication_reporter", false, "Register the health check module that monitors MySQL replication") -) +var enableReplicationReporter bool // replicationReporter implements health.Reporter type replicationReporter struct { @@ -157,7 +154,7 @@ func repairReplication(ctx context.Context, tm *TabletManager) error { } func registerReplicationReporter(tm *TabletManager) { - if *enableReplicationReporter { + if enableReplicationReporter { health.DefaultAggregator.Register("replication_reporter", &replicationReporter{ tm: tm, diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index a4d5244fa5f..6e464297762 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -235,6 +235,8 @@ func InitConfig(config *tabletenv.TabletConfig) { healthCheckInterval = config.Healthcheck.IntervalSeconds.Get() degradedThreshold = config.Healthcheck.DegradedThresholdSeconds.Get() unhealthyThreshold = config.Healthcheck.UnhealthyThresholdSeconds.Get() + + enableReplicationReporter = config.ReplicationTracker.Mode == tabletenv.Polling } // BuildTabletFromInput builds a tablet record from input parameters. diff --git a/go/vt/vttablet/tabletserver/repltracker/reader.go b/go/vt/vttablet/tabletserver/repltracker/reader.go index fdb051bc5e6..933d3f0a4a3 100644 --- a/go/vt/vttablet/tabletserver/repltracker/reader.go +++ b/go/vt/vttablet/tabletserver/repltracker/reader.go @@ -69,11 +69,11 @@ type heartbeatReader struct { // newHeartbeatReader returns a new heartbeatReader. func newHeartbeatReader(env tabletenv.Env) *heartbeatReader { config := env.Config() - if config.HeartbeatIntervalSeconds == 0 { + if config.ReplicationTracker.Mode != tabletenv.Heartbeat { return &heartbeatReader{} } - heartbeatInterval := config.HeartbeatIntervalSeconds.Get() + heartbeatInterval := config.ReplicationTracker.HeartbeatIntervalSeconds.Get() return &heartbeatReader{ env: env, enabled: true, diff --git a/go/vt/vttablet/tabletserver/repltracker/reader_test.go b/go/vt/vttablet/tabletserver/repltracker/reader_test.go index 43dd0695c43..4462fb6aee3 100644 --- a/go/vt/vttablet/tabletserver/repltracker/reader_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/reader_test.go @@ -100,7 +100,8 @@ func TestReaderReadHeartbeatError(t *testing.T) { func newReader(db *fakesqldb.DB, nowFunc func() time.Time) *heartbeatReader { config := tabletenv.NewDefaultConfig() - config.HeartbeatIntervalSeconds = 1 + config.ReplicationTracker.Mode = tabletenv.Heartbeat + config.ReplicationTracker.HeartbeatIntervalSeconds = 1 params, _ := db.ConnParams().MysqlParams() cp := *params dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") diff --git a/go/vt/vttablet/tabletserver/repltracker/writer.go b/go/vt/vttablet/tabletserver/repltracker/writer.go index 61d1c4d5bf9..5f4d31910a7 100644 --- a/go/vt/vttablet/tabletserver/repltracker/writer.go +++ b/go/vt/vttablet/tabletserver/repltracker/writer.go @@ -73,10 +73,10 @@ type heartbeatWriter struct { // newHeartbeatWriter creates a new heartbeatWriter. func newHeartbeatWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *heartbeatWriter { config := env.Config() - if config.HeartbeatIntervalSeconds == 0 { + if config.ReplicationTracker.Mode != tabletenv.Heartbeat { return &heartbeatWriter{} } - heartbeatInterval := config.HeartbeatIntervalSeconds.Get() + heartbeatInterval := config.ReplicationTracker.HeartbeatIntervalSeconds.Get() return &heartbeatWriter{ env: env, enabled: true, diff --git a/go/vt/vttablet/tabletserver/repltracker/writer_test.go b/go/vt/vttablet/tabletserver/repltracker/writer_test.go index 30f0cabf7da..7cdcc94a556 100644 --- a/go/vt/vttablet/tabletserver/repltracker/writer_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/writer_test.go @@ -94,7 +94,8 @@ func TestWriteHeartbeatError(t *testing.T) { func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *heartbeatWriter { config := tabletenv.NewDefaultConfig() - config.HeartbeatIntervalSeconds = 1 + config.ReplicationTracker.Mode = tabletenv.Heartbeat + config.ReplicationTracker.HeartbeatIntervalSeconds = 1 params, _ := db.ConnParams().MysqlParams() cp := *params diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 01af523aff7..bae33bfa4ed 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -37,6 +37,8 @@ const ( Disable = "disable" Dryrun = "dryRun" NotOnMaster = "notOnMaster" + Polling = "polling" + Heartbeat = "heartbeat" ) var ( @@ -75,6 +77,7 @@ var ( healthCheckInterval time.Duration degradedThreshold time.Duration unhealthyThreshold time.Duration + enableReplicationReporter bool ) func init() { @@ -147,6 +150,8 @@ func init() { flag.DurationVar(&healthCheckInterval, "health_check_interval", 20*time.Second, "Interval between health checks") flag.DurationVar(°radedThreshold, "degraded_threshold", 30*time.Second, "replication lag after which a replica is considered degraded (only used in status UI)") flag.DurationVar(&unhealthyThreshold, "unhealthy_threshold", 2*time.Hour, "replication lag after which a replica is considered unhealthy") + + flag.BoolVar(&enableReplicationReporter, "enable_replication_reporter", false, "Use polling to track replication lag.") } // Init must be called after flag.Parse, and before doing any other operations. @@ -175,8 +180,16 @@ func Init() { currentConfig.Consolidator = Disable } - if enableHeartbeat { - currentConfig.HeartbeatIntervalSeconds.Set(heartbeatInterval) + switch { + case enableHeartbeat: + currentConfig.ReplicationTracker.Mode = Heartbeat + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds.Set(heartbeatInterval) + case enableReplicationReporter: + currentConfig.ReplicationTracker.Mode = Polling + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 + default: + currentConfig.ReplicationTracker.Mode = Disable + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 } currentConfig.Healthcheck.IntervalSeconds.Set(healthCheckInterval) @@ -212,8 +225,10 @@ type TabletConfig struct { Healthcheck HealthcheckConfig `json:"healthcheck,omitempty"` + ReplicationTracker ReplicationTrackerConfig `json:"replicationTracker,omitempty"` + + // Consolidator can be enable, disable, or notOnMaster. Default is enable. Consolidator string `json:"consolidator,omitempty"` - HeartbeatIntervalSeconds Seconds `json:"heartbeatIntervalSeconds,omitempty"` ShutdownGracePeriodSeconds Seconds `json:"shutdownGracePeriodSeconds,omitempty"` PassthroughDML bool `json:"passthroughDML,omitempty"` StreamBufferSize int `json:"streamBufferSize,omitempty"` @@ -262,6 +277,7 @@ type OltpConfig struct { // HotRowProtectionConfig contains the config for hot row protection. type HotRowProtectionConfig struct { + // Mode can be disable, dryRun or enable. Default is disable. Mode string `json:"mode,omitempty"` MaxQueueSize int `json:"maxQueueSize,omitempty"` MaxGlobalQueueSize int `json:"maxGlobalQueueSize,omitempty"` @@ -275,6 +291,13 @@ type HealthcheckConfig struct { UnhealthyThresholdSeconds Seconds `json:"unhealthyThresholdSeconds,omitempty"` } +// ReplicationTrackerConfig contains the config for the replication tracker. +type ReplicationTrackerConfig struct { + // Mode can be disable, polling or heartbeat. Default is disable. + Mode string `json:"mode,omitempty"` + HeartbeatIntervalSeconds Seconds `json:"heartbeatIntervalSeconds,omitempty"` +} + // TransactionLimitConfig captures configuration of transaction pool slots // limiter configuration. type TransactionLimitConfig struct { @@ -384,6 +407,9 @@ var defaultConfig = TabletConfig{ DegradedThresholdSeconds: 30, UnhealthyThresholdSeconds: 7200, }, + ReplicationTracker: ReplicationTrackerConfig{ + Mode: Disable, + }, HotRowProtection: HotRowProtectionConfig{ Mode: Disable, // Default value is the same as TxPool.Size. diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index 0a871fb7b3a..81bee701980 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -73,6 +73,7 @@ oltpReadPool: prefillParallelism: 30 size: 16 timeoutSeconds: 10 +replicationTracker: {} txPool: {} ` assert.Equal(t, wantBytes, string(gotBytes)) @@ -127,6 +128,8 @@ oltpReadPool: maxWaiters: 5000 size: 16 queryCacheSize: 5000 +replicationTracker: + mode: disable schemaReloadIntervalSeconds: 1800 streamBufferSize: 32768 txPool: @@ -210,6 +213,7 @@ func TestFlags(t *testing.T) { want.Healthcheck.IntervalSeconds = 20 want.Healthcheck.DegradedThresholdSeconds = 30 want.Healthcheck.UnhealthyThresholdSeconds = 7200 + want.ReplicationTracker.Mode = Disable assert.Equal(t, want.DB, currentConfig.DB) assert.Equal(t, want, currentConfig) @@ -263,15 +267,29 @@ func TestFlags(t *testing.T) { enableHeartbeat = true heartbeatInterval = 1 * time.Second + currentConfig.ReplicationTracker.Mode = "" + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 Init() - want.HeartbeatIntervalSeconds = 1 + want.ReplicationTracker.Mode = Heartbeat + want.ReplicationTracker.HeartbeatIntervalSeconds = 1 assert.Equal(t, want, currentConfig) enableHeartbeat = false heartbeatInterval = 1 * time.Second - currentConfig.HeartbeatIntervalSeconds = 0 + currentConfig.ReplicationTracker.Mode = "" + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 + Init() + want.ReplicationTracker.Mode = Disable + want.ReplicationTracker.HeartbeatIntervalSeconds = 0 + assert.Equal(t, want, currentConfig) + + enableReplicationReporter = true + heartbeatInterval = 1 * time.Second + currentConfig.ReplicationTracker.Mode = "" + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 Init() - want.HeartbeatIntervalSeconds = 0 + want.ReplicationTracker.Mode = Polling + want.ReplicationTracker.HeartbeatIntervalSeconds = 0 assert.Equal(t, want, currentConfig) healthCheckInterval = 1 * time.Second From 6d65f9fd72596b72ab9ce875c918d3cdc9115b43 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 16:33:47 -0700 Subject: [PATCH 006/100] vttablet: saving work. need mysqld in tabletserver Signed-off-by: Sugu Sougoumarane --- .../tabletserver/repltracker/reader.go | 4 +- .../tabletserver/repltracker/reader_test.go | 4 +- .../tabletserver/repltracker/repltracker.go | 59 +++++++++++++++---- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/go/vt/vttablet/tabletserver/repltracker/reader.go b/go/vt/vttablet/tabletserver/repltracker/reader.go index 933d3f0a4a3..590f507f67c 100644 --- a/go/vt/vttablet/tabletserver/repltracker/reader.go +++ b/go/vt/vttablet/tabletserver/repltracker/reader.go @@ -126,8 +126,8 @@ func (r *heartbeatReader) Close() { r.isOpen = false } -// GetLatest returns the most recently recorded lag measurement or error encountered. -func (r *heartbeatReader) GetLatest() (time.Duration, error) { +// Status returns the most recently recorded lag measurement or error encountered. +func (r *heartbeatReader) Status() (time.Duration, error) { r.lagMu.Lock() defer r.lagMu.Unlock() if r.lastKnownError != nil { diff --git a/go/vt/vttablet/tabletserver/repltracker/reader_test.go b/go/vt/vttablet/tabletserver/repltracker/reader_test.go index 4462fb6aee3..c98a75878d8 100644 --- a/go/vt/vttablet/tabletserver/repltracker/reader_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/reader_test.go @@ -51,7 +51,7 @@ func TestReaderReadHeartbeat(t *testing.T) { reads.Reset() tr.readHeartbeat() - lag, err := tr.GetLatest() + lag, err := tr.Status() if err != nil { t.Fatalf("Should not be in error: %v", tr.lastKnownError) @@ -82,7 +82,7 @@ func TestReaderReadHeartbeatError(t *testing.T) { readErrors.Reset() tr.readHeartbeat() - lag, err := tr.GetLatest() + lag, err := tr.Status() if err == nil { t.Fatalf("Should be in error: %v", tr.lastKnownError) diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index a9b5548108b..be581b066e8 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -18,6 +18,7 @@ package repltracker import ( "sync" + "time" "vitess.io/vitess/go/stats" querypb "vitess.io/vitess/go/vt/proto/query" @@ -25,14 +26,6 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) -type method int - -const ( - none = method(iota) - polling - heartbeat -) - var ( // HeartbeatWrites keeps a count of the number of heartbeats written over time. writes = stats.NewCounter("HeartbeatWrites", "Count of heartbeats written over time") @@ -53,7 +46,7 @@ var ( type ReplTracker struct { mu sync.Mutex isMaster bool - method method + mode string hw *heartbeatWriter hr *heartbeatReader @@ -62,8 +55,9 @@ type ReplTracker struct { // NewReplTracker creates a new ReplTracker. func NewReplTracker(env tabletenv.Env, alias topodatapb.TabletAlias) *ReplTracker { return &ReplTracker{ - hw: newHeartbeatWriter(env, alias), - hr: newHeartbeatReader(env), + mode: env.Config().ReplicationTracker.Mode, + hw: newHeartbeatWriter(env, alias), + hr: newHeartbeatReader(env), } } @@ -72,3 +66,46 @@ func (rt *ReplTracker) InitDBConfig(target querypb.Target) { rt.hw.InitDBConfig(target) rt.hr.InitDBConfig(target) } + +// MakeMaster must be called if the tablet type becomes MASTER. +func (rt *ReplTracker) MakeMaster() { + rt.mu.Lock() + defer rt.mu.Unlock() + + switch rt.mode { + case tabletenv.Heartbeat: + rt.hr.Close() + rt.hw.Open() + default: + rt.isMaster = true + } +} + +// MakeNonMaster must be called if the tablet type becomes non-MASTER. +func (rt *ReplTracker) MakeNonMaster() { + rt.mu.Lock() + defer rt.mu.Unlock() + + switch rt.mode { + case tabletenv.Heartbeat: + rt.hw.Close() + rt.hr.Open() + default: + rt.isMaster = false + } +} + +// Status reports the replication status. +func (rt *ReplTracker) Status() (time.Duration, error) { + rt.mu.Lock() + defer rt.mu.Unlock() + + if rt.isMaster || rt.mode == tabletenv.Disable { + return 0, nil + } + switch rt.mode { + case tabletenv.Heartbeat: + return rt.hr.Status() + } + return 0, nil +} From 4776c746c249410ef96f1705369866165c7690af Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 17:59:20 -0700 Subject: [PATCH 007/100] vttablet: add mysqld to tabletserver Signed-off-by: Sugu Sougoumarane --- go/vt/vtexplain/vtexplain_vttablet.go | 2 +- go/vt/vttablet/endtoend/framework/server.go | 2 +- go/vt/vttablet/tabletmanager/tm_init.go | 2 +- go/vt/vttablet/tabletserver/controller.go | 3 ++- .../tabletserver/query_executor_test.go | 2 +- .../tabletserver/repltracker/repltracker.go | 22 +++++++++++-------- go/vt/vttablet/tabletserver/tabletserver.go | 11 +++++++--- .../tabletserver/tabletserver_test.go | 2 +- go/vt/vttablet/tabletservermock/controller.go | 3 ++- 9 files changed, 30 insertions(+), 19 deletions(-) diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index f05e39f0ad0..bde82e1f8bb 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -110,7 +110,7 @@ func newTablet(opts *Options, t *topodatapb.Tablet) *explainTablet { Shard: t.Shard, TabletType: topodatapb.TabletType_MASTER, } - tsv.StartService(target, dbcfgs) + tsv.StartService(target, dbcfgs, nil /* mysqld */) // clear all the schema initialization queries out of the tablet // to avoid clutttering the output diff --git a/go/vt/vttablet/endtoend/framework/server.go b/go/vt/vttablet/endtoend/framework/server.go index a14ded01fc3..32be799bb1c 100644 --- a/go/vt/vttablet/endtoend/framework/server.go +++ b/go/vt/vttablet/endtoend/framework/server.go @@ -85,7 +85,7 @@ func StartServer(connParams, connAppDebugParams mysql.ConnParams, dbName string) Server = tabletserver.NewTabletServer("", config, TopoServer, topodatapb.TabletAlias{}) Server.Register() - err := Server.StartService(Target, dbcfgs) + err := Server.StartService(Target, dbcfgs, nil /* mysqld */) if err != nil { return vterrors.Wrap(err, "could not start service") } diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 6e464297762..790c1a1bbc2 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -324,7 +324,7 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { Keyspace: tablet.Keyspace, Shard: tablet.Shard, TabletType: tablet.Type, - }, tm.DBConfigs) + }, tm.DBConfigs, tm.MysqlDaemon) if err != nil { return vterrors.Wrap(err, "failed to InitDBConfig") } diff --git a/go/vt/vttablet/tabletserver/controller.go b/go/vt/vttablet/tabletserver/controller.go index 70472c64f75..ba4456895b9 100644 --- a/go/vt/vttablet/tabletserver/controller.go +++ b/go/vt/vttablet/tabletserver/controller.go @@ -20,6 +20,7 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletserver/rules" @@ -47,7 +48,7 @@ type Controller interface { Stats() *tabletenv.Stats // InitDBConfig sets up the db config vars. - InitDBConfig(querypb.Target, *dbconfigs.DBConfigs) error + InitDBConfig(querypb.Target, *dbconfigs.DBConfigs, mysqlctl.MysqlDaemon) error // SetServingType transitions the query service to the required serving type. // Returns true if the state of QueryService or the tablet type changed. diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index 922cf7d2762..5fc1c1c1256 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -1109,7 +1109,7 @@ func newTestTabletServer(ctx context.Context, flags executorFlags, db *fakesqldb tsv := NewTabletServer("TabletServerTest", config, memorytopo.NewServer(""), topodatapb.TabletAlias{}) dbconfigs := newDBConfigs(db) target := querypb.Target{TabletType: topodatapb.TabletType_MASTER} - err := tsv.StartService(target, dbconfigs) + err := tsv.StartService(target, dbconfigs, nil /* mysqld */) if err != nil { panic(err) } diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index be581b066e8..8be95f16ecf 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -21,25 +21,27 @@ import ( "time" "vitess.io/vitess/go/stats" + "vitess.io/vitess/go/vt/mysqlctl" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) +// TODO(sougou): rename these back after deprecating hr & hw var ( // HeartbeatWrites keeps a count of the number of heartbeats written over time. - writes = stats.NewCounter("HeartbeatWrites", "Count of heartbeats written over time") + writes = stats.NewCounter("XHeartbeatWrites", "Count of heartbeats written over time") // HeartbeatWriteErrors keeps a count of errors encountered while writing heartbeats. - writeErrors = stats.NewCounter("HeartbeatWriteErrors", "Count of errors encountered while writing heartbeats") + writeErrors = stats.NewCounter("XHeartbeatWriteErrors", "Count of errors encountered while writing heartbeats") // HeartbeatReads keeps a count of the number of heartbeats read over time. - reads = stats.NewCounter("HeartbeatReads", "Count of heartbeats read over time") + reads = stats.NewCounter("XHeartbeatReads", "Count of heartbeats read over time") // HeartbeatReadErrors keeps a count of errors encountered while reading heartbeats. - readErrors = stats.NewCounter("HeartbeatReadErrors", "Count of errors encountered while reading heartbeats") + readErrors = stats.NewCounter("XHeartbeatReadErrors", "Count of errors encountered while reading heartbeats") // HeartbeatCumulativeLagNs is incremented by the current lag at each heartbeat read interval. Plotting this // over time allows calculating of a rolling average lag. - cumulativeLagNs = stats.NewCounter("HeartbeatCumulativeLagNs", "Incremented by the current lag at each heartbeat read interval") + cumulativeLagNs = stats.NewCounter("XHeartbeatCumulativeLagNs", "Incremented by the current lag at each heartbeat read interval") // HeartbeatCurrentLagNs is a point-in-time calculation of the lag, updated at each heartbeat read interval. - currentLagNs = stats.NewGauge("HeartbeatCurrentLagNs", "Point in time calculation of the heartbeat lag") + currentLagNs = stats.NewGauge("XHeartbeatCurrentLagNs", "Point in time calculation of the heartbeat lag") ) // ReplTracker tracks replication lag. @@ -48,8 +50,9 @@ type ReplTracker struct { isMaster bool mode string - hw *heartbeatWriter - hr *heartbeatReader + mysqld mysqlctl.MysqlDaemon + hw *heartbeatWriter + hr *heartbeatReader } // NewReplTracker creates a new ReplTracker. @@ -62,7 +65,8 @@ func NewReplTracker(env tabletenv.Env, alias topodatapb.TabletAlias) *ReplTracke } // InitDBConfig initializes the target name. -func (rt *ReplTracker) InitDBConfig(target querypb.Target) { +func (rt *ReplTracker) InitDBConfig(target querypb.Target, mysqld mysqlctl.MysqlDaemon) { + rt.mysqld = mysqld rt.hw.InitDBConfig(target) rt.hr.InitDBConfig(target) } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index e7811e5e70c..dcb7f591d6b 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -42,6 +42,7 @@ import ( "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -56,6 +57,7 @@ import ( "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletserver/messager" "vitess.io/vitess/go/vt/vttablet/tabletserver/planbuilder" + "vitess.io/vitess/go/vt/vttablet/tabletserver/repltracker" "vitess.io/vitess/go/vt/vttablet/tabletserver/rules" "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -96,6 +98,7 @@ type TabletServer struct { se *schema.Engine hw *heartbeat.Writer hr *heartbeat.Reader + rr *repltracker.ReplTracker vstreamer *vstreamer.Engine tracker *schema.Tracker watcher *ReplicationWatcher @@ -156,6 +159,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsv.se = schema.NewEngine(tsv) tsv.hw = heartbeat.NewWriter(tsv, alias) tsv.hr = heartbeat.NewReader(tsv) + tsv.rr = repltracker.NewReplTracker(tsv, alias) tsv.vstreamer = vstreamer.NewEngine(tsv, srvTopoServer, tsv.se, alias.Cell) tsv.tracker = schema.NewTracker(tsv, tsv.vstreamer, tsv.se) tsv.watcher = NewReplicationWatcher(tsv, tsv.vstreamer, tsv.config) @@ -201,7 +205,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to // InitDBConfig initializes the db config variables for TabletServer. You must call this function // to complete the creation of TabletServer. -func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DBConfigs) error { +func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DBConfigs, mysqld mysqlctl.MysqlDaemon) error { if tsv.sm.State() != StateNotConnected { return vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "InitDBConfig failed, current state: %s", tsv.sm.StateByName()) } @@ -211,6 +215,7 @@ func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.D tsv.se.InitDBConfig(tsv.config.DB.DbaWithDB()) tsv.hw.InitDBConfig(target) tsv.hr.InitDBConfig(target) + tsv.rr.InitDBConfig(target, mysqld) tsv.txThrottler.InitDBConfig(target) tsv.vstreamer.InitDBConfig(target.Keyspace) return nil @@ -320,8 +325,8 @@ func (tsv *TabletServer) SetServingType(tabletType topodatapb.TabletType, servin // StartService is a convenience function for InitDBConfig->SetServingType // with serving=true. -func (tsv *TabletServer) StartService(target querypb.Target, dbcfgs *dbconfigs.DBConfigs) error { - if err := tsv.InitDBConfig(target, dbcfgs); err != nil { +func (tsv *TabletServer) StartService(target querypb.Target, dbcfgs *dbconfigs.DBConfigs, mysqld mysqlctl.MysqlDaemon) error { + if err := tsv.InitDBConfig(target, dbcfgs, mysqld); err != nil { return err } _, err := tsv.sm.SetServingType(target.TabletType, StateServing, nil) diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index e72c376327c..c0a5a27f445 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -2165,7 +2165,7 @@ func setupTabletServerTestCustom(t *testing.T, config *tabletenv.TabletConfig) ( require.Equal(t, StateNotConnected, tsv.sm.State()) dbcfgs := newDBConfigs(db) target := querypb.Target{TabletType: topodatapb.TabletType_MASTER} - err := tsv.StartService(target, dbcfgs) + err := tsv.StartService(target, dbcfgs, nil /* mysqld */) require.NoError(t, err) return db, tsv } diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go index a98a6523fe3..f3aa4f26697 100644 --- a/go/vt/vttablet/tabletservermock/controller.go +++ b/go/vt/vttablet/tabletservermock/controller.go @@ -25,6 +25,7 @@ import ( "time" "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vttablet/queryservice" @@ -123,7 +124,7 @@ func (tqsc *Controller) AddStatusPart() { } // InitDBConfig is part of the tabletserver.Controller interface -func (tqsc *Controller) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DBConfigs) error { +func (tqsc *Controller) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DBConfigs, _ mysqlctl.MysqlDaemon) error { tqsc.mu.Lock() defer tqsc.mu.Unlock() From 7098531b463e8b3c29c4fc13549d449e23b6244b Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 18:25:36 -0700 Subject: [PATCH 008/100] vttablet: ReplTracker functional Signed-off-by: Sugu Sougoumarane --- .../tabletserver/repltracker/poller.go | 59 +++++++++++++++++++ .../tabletserver/repltracker/repltracker.go | 21 +++---- 2 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 go/vt/vttablet/tabletserver/repltracker/poller.go diff --git a/go/vt/vttablet/tabletserver/repltracker/poller.go b/go/vt/vttablet/tabletserver/repltracker/poller.go new file mode 100644 index 00000000000..dd77cc22188 --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/poller.go @@ -0,0 +1,59 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "sync" + "time" + + "vitess.io/vitess/go/vt/mysqlctl" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" +) + +type poller struct { + mysqld mysqlctl.MysqlDaemon + + mu sync.Mutex + lag time.Duration + timeRecorded time.Time +} + +func (p *poller) InitDBConfig(mysqld mysqlctl.MysqlDaemon) { + p.mysqld = mysqld +} + +func (p *poller) Status() (time.Duration, error) { + p.mu.Lock() + defer p.mu.Unlock() + + status, err := p.mysqld.ReplicationStatus() + if err != nil { + return 0, err + } + + if !status.ReplicationRunning() { + if p.timeRecorded.IsZero() { + return 0, vterrors.Errorf(vtrpcpb.Code_UNAVAILABLE, "replication is not running") + } + return time.Since(p.timeRecorded) + p.lag, nil + } + + p.lag = time.Duration(status.SecondsBehindMaster) * time.Second + p.timeRecorded = time.Now() + return p.lag, nil +} diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 8be95f16ecf..457288859ca 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -50,25 +50,26 @@ type ReplTracker struct { isMaster bool mode string - mysqld mysqlctl.MysqlDaemon hw *heartbeatWriter hr *heartbeatReader + poller *poller } // NewReplTracker creates a new ReplTracker. func NewReplTracker(env tabletenv.Env, alias topodatapb.TabletAlias) *ReplTracker { return &ReplTracker{ - mode: env.Config().ReplicationTracker.Mode, - hw: newHeartbeatWriter(env, alias), - hr: newHeartbeatReader(env), + mode: env.Config().ReplicationTracker.Mode, + hw: newHeartbeatWriter(env, alias), + hr: newHeartbeatReader(env), + poller: &poller{}, } } // InitDBConfig initializes the target name. func (rt *ReplTracker) InitDBConfig(target querypb.Target, mysqld mysqlctl.MysqlDaemon) { - rt.mysqld = mysqld rt.hw.InitDBConfig(target) rt.hr.InitDBConfig(target) + rt.poller.InitDBConfig(mysqld) } // MakeMaster must be called if the tablet type becomes MASTER. @@ -104,12 +105,12 @@ func (rt *ReplTracker) Status() (time.Duration, error) { rt.mu.Lock() defer rt.mu.Unlock() - if rt.isMaster || rt.mode == tabletenv.Disable { + switch { + case rt.isMaster || rt.mode == tabletenv.Disable: return 0, nil - } - switch rt.mode { - case tabletenv.Heartbeat: + case rt.mode == tabletenv.Heartbeat: return rt.hr.Status() } - return 0, nil + // rt.mode == tabletenv.Poller + return rt.poller.Status() } From cddd0de07f7aa62c16bec5ea8de741385b7d4960 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 20:11:35 -0700 Subject: [PATCH 009/100] vttablet: switch to use replTracker Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/heartbeat/heartbeat.go | 49 ---- go/vt/vttablet/heartbeat/reader.go | 228 ------------------ go/vt/vttablet/heartbeat/reader_test.go | 115 --------- go/vt/vttablet/heartbeat/writer.go | 186 -------------- go/vt/vttablet/heartbeat/writer_test.go | 110 --------- .../tabletserver/repltracker/repltracker.go | 22 +- go/vt/vttablet/tabletserver/state_manager.go | 75 +++--- .../tabletserver/state_manager_test.go | 112 +++++---- go/vt/vttablet/tabletserver/tabletserver.go | 26 +- 9 files changed, 122 insertions(+), 801 deletions(-) delete mode 100644 go/vt/vttablet/heartbeat/heartbeat.go delete mode 100644 go/vt/vttablet/heartbeat/reader.go delete mode 100644 go/vt/vttablet/heartbeat/reader_test.go delete mode 100644 go/vt/vttablet/heartbeat/writer.go delete mode 100644 go/vt/vttablet/heartbeat/writer_test.go diff --git a/go/vt/vttablet/heartbeat/heartbeat.go b/go/vt/vttablet/heartbeat/heartbeat.go deleted file mode 100644 index fbba04c71db..00000000000 --- a/go/vt/vttablet/heartbeat/heartbeat.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package heartbeat contains a writer and reader of heartbeats for a master-replica cluster. -// This is similar to Percona's pt-heartbeat, and is meant to supplement the information -// returned from SHOW SLAVE STATUS. In some circumstances, lag returned from SHOW SLAVE STATUS -// is incorrect and is at best only at 1 second resolution. The heartbeat package directly -// tests replication by writing a record with a timestamp on the master, and comparing that -// timestamp after reading it on the replica. This happens at the interval defined by heartbeat_interval. -// Note: the lag reported will be affected by clock drift, so it is recommended to run ntpd or similar. -// -// The data collected by the heartbeat package is made available in /debug/vars in counters prefixed by Heartbeat*. -// It's additionally used as a source for healthchecks and will impact the serving state of a tablet, if enabled. -// The heartbeat interval is purposefully kept distinct from the health check interval because lag measurement -// requires more frequent polling that the healthcheck typically is configured for. -package heartbeat - -import ( - "vitess.io/vitess/go/stats" -) - -var ( - // HeartbeatWrites keeps a count of the number of heartbeats written over time. - writes = stats.NewCounter("HeartbeatWrites", "Count of heartbeats written over time") - // HeartbeatWriteErrors keeps a count of errors encountered while writing heartbeats. - writeErrors = stats.NewCounter("HeartbeatWriteErrors", "Count of errors encountered while writing heartbeats") - // HeartbeatReads keeps a count of the number of heartbeats read over time. - reads = stats.NewCounter("HeartbeatReads", "Count of heartbeats read over time") - // HeartbeatReadErrors keeps a count of errors encountered while reading heartbeats. - readErrors = stats.NewCounter("HeartbeatReadErrors", "Count of errors encountered while reading heartbeats") - // HeartbeatCumulativeLagNs is incremented by the current lag at each heartbeat read interval. Plotting this - // over time allows calculating of a rolling average lag. - cumulativeLagNs = stats.NewCounter("HeartbeatCumulativeLagNs", "Incremented by the current lag at each heartbeat read interval") - // HeartbeatCurrentLagNs is a point-in-time calculation of the lag, updated at each heartbeat read interval. - currentLagNs = stats.NewGauge("HeartbeatCurrentLagNs", "Point in time calculation of the heartbeat lag") -) diff --git a/go/vt/vttablet/heartbeat/reader.go b/go/vt/vttablet/heartbeat/reader.go deleted file mode 100644 index bbdcfd7f881..00000000000 --- a/go/vt/vttablet/heartbeat/reader.go +++ /dev/null @@ -1,228 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package heartbeat - -import ( - "fmt" - "sync" - "time" - - "vitess.io/vitess/go/vt/vtgate/evalengine" - - "vitess.io/vitess/go/vt/vterrors" - - "golang.org/x/net/context" - - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/timer" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" - - querypb "vitess.io/vitess/go/vt/proto/query" -) - -const ( - sqlFetchMostRecentHeartbeat = "SELECT ts FROM %s.heartbeat WHERE keyspaceShard=%a" -) - -// Reader reads the heartbeat table at a configured interval in order -// to calculate replication lag. It is meant to be run on a replica, and paired -// with a Writer on a master. It's primarily created and launched from Reporter. -// Lag is calculated by comparing the most recent timestamp in the heartbeat -// table against the current time at read time. This value is reported in metrics and -// also to the healthchecks. -type Reader struct { - env tabletenv.Env - - enabled bool - interval time.Duration - keyspaceShard string - now func() time.Time - errorLog *logutil.ThrottledLogger - - runMu sync.Mutex - isOpen bool - pool *connpool.Pool - ticks *timer.Timer - - lagMu sync.Mutex - lastKnownLag time.Duration - lastKnownError error -} - -// NewReader returns a new heartbeat reader. -func NewReader(env tabletenv.Env) *Reader { - config := env.Config() - if config.ReplicationTracker.Mode != tabletenv.Heartbeat { - return &Reader{} - } - - heartbeatInterval := config.ReplicationTracker.HeartbeatIntervalSeconds.Get() - return &Reader{ - env: env, - enabled: true, - now: time.Now, - interval: heartbeatInterval, - ticks: timer.NewTimer(heartbeatInterval), - errorLog: logutil.NewThrottledLogger("HeartbeatReporter", 60*time.Second), - pool: connpool.NewPool(env, "HeartbeatReadPool", tabletenv.ConnPoolConfig{ - Size: 1, - IdleTimeoutSeconds: env.Config().OltpReadPool.IdleTimeoutSeconds, - }), - } -} - -// InitDBConfig initializes the target name for the Reader. -func (r *Reader) InitDBConfig(target querypb.Target) { - r.keyspaceShard = fmt.Sprintf("%s:%s", target.Keyspace, target.Shard) -} - -// Open starts the heartbeat ticker and opens the db pool. It may be called multiple -// times, as long as it was closed since last invocation. -func (r *Reader) Open() { - if !r.enabled { - return - } - r.runMu.Lock() - defer r.runMu.Unlock() - if r.isOpen { - return - } - - log.Info("Beginning heartbeat reads") - r.pool.Open(r.env.Config().DB.AppWithDB(), r.env.Config().DB.DbaWithDB(), r.env.Config().DB.AppDebugWithDB()) - r.ticks.Start(func() { r.readHeartbeat() }) - r.isOpen = true -} - -// Close cancels the watchHeartbeat periodic ticker and closes the db pool. -// A reader object can be re-opened after closing. -func (r *Reader) Close() { - if !r.enabled { - return - } - r.runMu.Lock() - defer r.runMu.Unlock() - if !r.isOpen { - return - } - r.ticks.Stop() - r.pool.Close() - log.Info("Stopped heartbeat reads") - r.isOpen = false -} - -// GetLatest returns the most recently recorded lag measurement or error encountered. -func (r *Reader) GetLatest() (time.Duration, error) { - r.lagMu.Lock() - defer r.lagMu.Unlock() - if r.lastKnownError != nil { - return 0, r.lastKnownError - } - return r.lastKnownLag, nil -} - -// readHeartbeat reads from the heartbeat table exactly once, updating -// the last known lag and/or error, and incrementing counters. -func (r *Reader) readHeartbeat() { - defer r.env.LogError() - - ctx, cancel := context.WithDeadline(context.Background(), r.now().Add(r.interval)) - defer cancel() - - res, err := r.fetchMostRecentHeartbeat(ctx) - if err != nil { - r.recordError(vterrors.Wrap(err, "Failed to read most recent heartbeat")) - return - } - ts, err := parseHeartbeatResult(res) - if err != nil { - r.recordError(vterrors.Wrap(err, "Failed to parse heartbeat result")) - return - } - - lag := r.now().Sub(time.Unix(0, ts)) - cumulativeLagNs.Add(lag.Nanoseconds()) - currentLagNs.Set(lag.Nanoseconds()) - reads.Add(1) - - r.lagMu.Lock() - r.lastKnownLag = lag - r.lastKnownError = nil - r.lagMu.Unlock() -} - -// fetchMostRecentHeartbeat fetches the most recently recorded heartbeat from the heartbeat table, -// returning a result with the timestamp of the heartbeat. -func (r *Reader) fetchMostRecentHeartbeat(ctx context.Context) (*sqltypes.Result, error) { - conn, err := r.pool.Get(ctx) - if err != nil { - return nil, err - } - defer conn.Recycle() - sel, err := r.bindHeartbeatFetch() - if err != nil { - return nil, err - } - return conn.Exec(ctx, sel, 1, false) -} - -// bindHeartbeatFetch takes a heartbeat read and adds the necessary -// fields to the query as bind vars. This is done to protect ourselves -// against a badly formed keyspace or shard name. -func (r *Reader) bindHeartbeatFetch() (string, error) { - bindVars := map[string]*querypb.BindVariable{ - "ks": sqltypes.StringBindVariable(r.keyspaceShard), - } - parsed := sqlparser.BuildParsedQuery(sqlFetchMostRecentHeartbeat, "_vt", ":ks") - bound, err := parsed.GenerateQuery(bindVars, nil) - if err != nil { - return "", err - } - return bound, nil -} - -// parseHeartbeatResult turns a raw result into the timestamp for processing. -func parseHeartbeatResult(res *sqltypes.Result) (int64, error) { - if len(res.Rows) != 1 { - return 0, fmt.Errorf("failed to read heartbeat: writer query did not result in 1 row. Got %v", len(res.Rows)) - } - ts, err := evalengine.ToInt64(res.Rows[0][0]) - if err != nil { - return 0, err - } - return ts, nil -} - -// recordError keeps track of the lastKnown error for reporting to the healthcheck. -// Errors tracked here are logged with throttling to cut down on log spam since -// operations can happen very frequently in this package. -func (r *Reader) recordError(err error) { - r.lagMu.Lock() - r.lastKnownError = err - r.lagMu.Unlock() - r.errorLog.Errorf("%v", err) - readErrors.Add(1) -} - -// IsOpen returns true if Reader is open. -func (r *Reader) IsOpen() bool { - return r.isOpen -} diff --git a/go/vt/vttablet/heartbeat/reader_test.go b/go/vt/vttablet/heartbeat/reader_test.go deleted file mode 100644 index 1fcf23b61df..00000000000 --- a/go/vt/vttablet/heartbeat/reader_test.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package heartbeat - -import ( - "fmt" - "testing" - "time" - - "vitess.io/vitess/go/mysql/fakesqldb" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" - - querypb "vitess.io/vitess/go/vt/proto/query" -) - -// TestReaderReadHeartbeat tests that reading a heartbeat sets the appropriate -// fields on the object. -func TestReaderReadHeartbeat(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - tr := newReader(db, mockNowFunc) - defer tr.Close() - - db.AddQuery(fmt.Sprintf("SELECT ts FROM %s.heartbeat WHERE keyspaceShard='%s'", "_vt", tr.keyspaceShard), &sqltypes.Result{ - Fields: []*querypb.Field{ - {Name: "ts", Type: sqltypes.Int64}, - }, - Rows: [][]sqltypes.Value{{ - sqltypes.NewInt64(now.Add(-10 * time.Second).UnixNano()), - }}, - }) - - cumulativeLagNs.Reset() - readErrors.Reset() - reads.Reset() - - tr.readHeartbeat() - lag, err := tr.GetLatest() - - if err != nil { - t.Fatalf("Should not be in error: %v", tr.lastKnownError) - } - if got, want := lag, 10*time.Second; got != want { - t.Fatalf("wrong latest lag: got = %v, want = %v", tr.lastKnownLag, want) - } - if got, want := cumulativeLagNs.Get(), 10*time.Second.Nanoseconds(); got != want { - t.Fatalf("wrong cumulative lag: got = %v, want = %v", got, want) - } - if got, want := reads.Get(), int64(1); got != want { - t.Fatalf("wrong read count: got = %v, want = %v", got, want) - } - if got, want := readErrors.Get(), int64(0); got != want { - t.Fatalf("wrong read error count: got = %v, want = %v", got, want) - } -} - -// TestReaderReadHeartbeatError tests that we properly account for errors -// encountered in the reading of heartbeat. -func TestReaderReadHeartbeatError(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - tr := newReader(db, mockNowFunc) - defer tr.Close() - - cumulativeLagNs.Reset() - readErrors.Reset() - - tr.readHeartbeat() - lag, err := tr.GetLatest() - - if err == nil { - t.Fatalf("Should be in error: %v", tr.lastKnownError) - } - if got, want := lag, 0*time.Second; got != want { - t.Fatalf("wrong lastKnownLag: got = %v, want = %v", got, want) - } - if got, want := cumulativeLagNs.Get(), int64(0); got != want { - t.Fatalf("wrong cumulative lag: got = %v, want = %v", got, want) - } - if got, want := readErrors.Get(), int64(1); got != want { - t.Fatalf("wrong read error count: got = %v, want = %v", got, want) - } -} - -func newReader(db *fakesqldb.DB, nowFunc func() time.Time) *Reader { - config := tabletenv.NewDefaultConfig() - config.ReplicationTracker.Mode = tabletenv.Heartbeat - config.ReplicationTracker.HeartbeatIntervalSeconds = 1 - params, _ := db.ConnParams().MysqlParams() - cp := *params - dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") - - tr := NewReader(tabletenv.NewEnv(config, "ReaderTest")) - tr.keyspaceShard = "test:0" - tr.now = nowFunc - tr.pool.Open(dbc.AppWithDB(), dbc.DbaWithDB(), dbc.AppDebugWithDB()) - - return tr -} diff --git a/go/vt/vttablet/heartbeat/writer.go b/go/vt/vttablet/heartbeat/writer.go deleted file mode 100644 index 9853e803db7..00000000000 --- a/go/vt/vttablet/heartbeat/writer.go +++ /dev/null @@ -1,186 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package heartbeat - -import ( - "fmt" - "sync" - "time" - - "vitess.io/vitess/go/vt/withddl" - - "golang.org/x/net/context" - - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/timer" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" - - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -const ( - sqlCreateSidecarDB = "create database if not exists %s" - sqlCreateHeartbeatTable = `CREATE TABLE IF NOT EXISTS %s.heartbeat ( - keyspaceShard VARBINARY(256) NOT NULL PRIMARY KEY, - tabletUid INT UNSIGNED NOT NULL, - ts BIGINT UNSIGNED NOT NULL - ) engine=InnoDB` - sqlUpsertHeartbeat = "INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%a, %a, %a) ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)" -) - -var withDDL = withddl.New([]string{ - fmt.Sprintf(sqlCreateSidecarDB, "_vt"), - fmt.Sprintf(sqlCreateHeartbeatTable, "_vt"), -}) - -// Writer runs on master tablets and writes heartbeats to the _vt.heartbeat -// table at a regular interval, defined by heartbeat_interval. -type Writer struct { - env tabletenv.Env - - enabled bool - interval time.Duration - tabletAlias topodatapb.TabletAlias - keyspaceShard string - now func() time.Time - errorLog *logutil.ThrottledLogger - - mu sync.Mutex - isOpen bool - pool *connpool.Pool - ticks *timer.Timer -} - -// NewWriter creates a new Writer. -func NewWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *Writer { - config := env.Config() - if config.ReplicationTracker.Mode != tabletenv.Heartbeat { - return &Writer{} - } - heartbeatInterval := config.ReplicationTracker.HeartbeatIntervalSeconds.Get() - return &Writer{ - env: env, - enabled: true, - tabletAlias: alias, - now: time.Now, - interval: heartbeatInterval, - ticks: timer.NewTimer(heartbeatInterval), - errorLog: logutil.NewThrottledLogger("HeartbeatWriter", 60*time.Second), - pool: connpool.NewPool(env, "HeartbeatWritePool", tabletenv.ConnPoolConfig{ - Size: 1, - IdleTimeoutSeconds: env.Config().OltpReadPool.IdleTimeoutSeconds, - }), - } -} - -// InitDBConfig initializes the target name for the Writer. -func (w *Writer) InitDBConfig(target querypb.Target) { - w.keyspaceShard = fmt.Sprintf("%s:%s", target.Keyspace, target.Shard) -} - -// Open sets up the Writer's db connection and launches the ticker -// responsible for periodically writing to the heartbeat table. -// Open may be called multiple times, as long as it was closed since -// last invocation. -func (w *Writer) Open() { - if !w.enabled { - return - } - w.mu.Lock() - defer w.mu.Unlock() - if w.isOpen { - return - } - - log.Info("Beginning heartbeat writes") - w.pool.Open(w.env.Config().DB.AppWithDB(), w.env.Config().DB.DbaWithDB(), w.env.Config().DB.AppDebugWithDB()) - w.ticks.Start(w.writeHeartbeat) - w.isOpen = true -} - -// Close closes the Writer's db connection and stops the periodic ticker. A writer -// object can be re-opened after closing. -func (w *Writer) Close() { - if !w.enabled { - return - } - w.mu.Lock() - defer w.mu.Unlock() - if !w.isOpen { - return - } - w.ticks.Stop() - w.pool.Close() - log.Info("Stopped heartbeat writes.") - w.isOpen = false -} - -// bindHeartbeatVars takes a heartbeat write (insert or update) and -// adds the necessary fields to the query as bind vars. This is done -// to protect ourselves against a badly formed keyspace or shard name. -func (w *Writer) bindHeartbeatVars(query string) (string, error) { - bindVars := map[string]*querypb.BindVariable{ - "ks": sqltypes.StringBindVariable(w.keyspaceShard), - "ts": sqltypes.Int64BindVariable(w.now().UnixNano()), - "uid": sqltypes.Int64BindVariable(int64(w.tabletAlias.Uid)), - } - parsed := sqlparser.BuildParsedQuery(query, "_vt", ":ts", ":uid", ":ks") - bound, err := parsed.GenerateQuery(bindVars, nil) - if err != nil { - return "", err - } - return bound, nil -} - -// writeHeartbeat updates the heartbeat row for this tablet with the current time in nanoseconds. -func (w *Writer) writeHeartbeat() { - if err := w.write(); err != nil { - w.recordError(err) - return - } - writes.Add(1) -} - -func (w *Writer) write() error { - defer w.env.LogError() - ctx, cancel := context.WithDeadline(context.Background(), w.now().Add(w.interval)) - defer cancel() - upsert, err := w.bindHeartbeatVars(sqlUpsertHeartbeat) - if err != nil { - return err - } - conn, err := w.pool.Get(ctx) - if err != nil { - return err - } - defer conn.Recycle() - _, err = withDDL.Exec(ctx, upsert, conn.Exec) - if err != nil { - return err - } - return nil -} - -func (w *Writer) recordError(err error) { - w.errorLog.Errorf("%v", err) - writeErrors.Add(1) -} diff --git a/go/vt/vttablet/heartbeat/writer_test.go b/go/vt/vttablet/heartbeat/writer_test.go deleted file mode 100644 index e424644898a..00000000000 --- a/go/vt/vttablet/heartbeat/writer_test.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package heartbeat - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "gotest.tools/assert" - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/mysql/fakesqldb" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/dbconfigs" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" -) - -var ( - now = time.Now() - mockNowFunc = func() time.Time { - return now - } -) - -func TestCreateSchema(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - tw := newTestWriter(db, mockNowFunc) - defer tw.Close() - writes.Reset() - - db.OrderMatters() - upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)", - "_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard) - failInsert := fakesqldb.ExpectedExecuteFetch{ - Query: upsert, - Error: mysql.NewSQLError(mysql.ERBadDb, "", "bad db error"), - } - db.AddExpectedExecuteFetch(failInsert) - db.AddExpectedQuery(fmt.Sprintf(sqlCreateSidecarDB, "_vt"), nil) - db.AddExpectedQuery(fmt.Sprintf(sqlCreateHeartbeatTable, "_vt"), nil) - db.AddExpectedQuery(upsert, nil) - - err := tw.write() - require.NoError(t, err) -} - -func TestWriteHeartbeat(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - - tw := newTestWriter(db, mockNowFunc) - upsert := fmt.Sprintf("INSERT INTO %s.heartbeat (ts, tabletUid, keyspaceShard) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE ts=VALUES(ts), tabletUid=VALUES(tabletUid)", - "_vt", now.UnixNano(), tw.tabletAlias.Uid, tw.keyspaceShard) - db.AddQuery(upsert, &sqltypes.Result{}) - - writes.Reset() - writeErrors.Reset() - - tw.writeHeartbeat() - assert.Equal(t, int64(1), writes.Get()) - assert.Equal(t, int64(0), writeErrors.Get()) -} - -func TestWriteHeartbeatError(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - - tw := newTestWriter(db, mockNowFunc) - - writes.Reset() - writeErrors.Reset() - - tw.writeHeartbeat() - assert.Equal(t, int64(0), writes.Get()) - assert.Equal(t, int64(1), writeErrors.Get()) -} - -func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *Writer { - config := tabletenv.NewDefaultConfig() - config.ReplicationTracker.Mode = tabletenv.Heartbeat - config.ReplicationTracker.HeartbeatIntervalSeconds = 1 - - params, _ := db.ConnParams().MysqlParams() - cp := *params - dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") - - tw := NewWriter(tabletenv.NewEnv(config, "WriterTest"), topodatapb.TabletAlias{Cell: "test", Uid: 1111}) - tw.keyspaceShard = "test:0" - tw.now = nowFunc - tw.pool.Open(dbc.AppWithDB(), dbc.DbaWithDB(), dbc.AppDebugWithDB()) - - return tw -} diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 457288859ca..2f7157ad3a5 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -27,28 +27,28 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) -// TODO(sougou): rename these back after deprecating hr & hw var ( // HeartbeatWrites keeps a count of the number of heartbeats written over time. - writes = stats.NewCounter("XHeartbeatWrites", "Count of heartbeats written over time") + writes = stats.NewCounter("HeartbeatWrites", "Count of heartbeats written over time") // HeartbeatWriteErrors keeps a count of errors encountered while writing heartbeats. - writeErrors = stats.NewCounter("XHeartbeatWriteErrors", "Count of errors encountered while writing heartbeats") + writeErrors = stats.NewCounter("HeartbeatWriteErrors", "Count of errors encountered while writing heartbeats") // HeartbeatReads keeps a count of the number of heartbeats read over time. - reads = stats.NewCounter("XHeartbeatReads", "Count of heartbeats read over time") + reads = stats.NewCounter("HeartbeatReads", "Count of heartbeats read over time") // HeartbeatReadErrors keeps a count of errors encountered while reading heartbeats. - readErrors = stats.NewCounter("XHeartbeatReadErrors", "Count of errors encountered while reading heartbeats") + readErrors = stats.NewCounter("HeartbeatReadErrors", "Count of errors encountered while reading heartbeats") // HeartbeatCumulativeLagNs is incremented by the current lag at each heartbeat read interval. Plotting this // over time allows calculating of a rolling average lag. - cumulativeLagNs = stats.NewCounter("XHeartbeatCumulativeLagNs", "Incremented by the current lag at each heartbeat read interval") + cumulativeLagNs = stats.NewCounter("HeartbeatCumulativeLagNs", "Incremented by the current lag at each heartbeat read interval") // HeartbeatCurrentLagNs is a point-in-time calculation of the lag, updated at each heartbeat read interval. - currentLagNs = stats.NewGauge("XHeartbeatCurrentLagNs", "Point in time calculation of the heartbeat lag") + currentLagNs = stats.NewGauge("HeartbeatCurrentLagNs", "Point in time calculation of the heartbeat lag") ) // ReplTracker tracks replication lag. type ReplTracker struct { + mode string + mu sync.Mutex isMaster bool - mode string hw *heartbeatWriter hr *heartbeatReader @@ -100,6 +100,12 @@ func (rt *ReplTracker) MakeNonMaster() { } } +// Close closes ReplTracker. +func (rt *ReplTracker) Close() { + rt.hw.Close() + rt.hr.Close() +} + // Status reports the replication status. func (rt *ReplTracker) Status() (time.Duration, error) { rt.mu.Lock() diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index e093c38df52..3190be4c6e9 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -98,8 +98,7 @@ type stateManager struct { // Close must be done in reverse order. // All Close functions must be called before Open. se schemaEngine - hw subComponent - hr subComponent + rt replTracker vstreamer subComponent tracker subComponent watcher subComponent @@ -115,34 +114,43 @@ type stateManager struct { timebombDuration time.Duration } -type schemaEngine interface { - Open() error - MakeNonMaster() - Close() -} +type ( + schemaEngine interface { + Open() error + MakeNonMaster() + Close() + } -type queryEngine interface { - Open() error - IsMySQLReachable() error - StopServing() - Close() -} + replTracker interface { + MakeMaster() + MakeNonMaster() + Close() + Status() (time.Duration, error) + } -type txEngine interface { - AcceptReadWrite() error - AcceptReadOnly() error - Close() -} + queryEngine interface { + Open() error + IsMySQLReachable() error + StopServing() + Close() + } -type subComponent interface { - Open() - Close() -} + txEngine interface { + AcceptReadWrite() error + AcceptReadOnly() error + Close() + } -type txThrottler interface { - Open() error - Close() -} + subComponent interface { + Open() + Close() + } + + txThrottler interface { + Open() error + Close() + } +) // SetServingType changes the state to the specified settings. // If a transition is in progress, it waits and then executes the @@ -360,13 +368,12 @@ func (sm *stateManager) VerifyTarget(ctx context.Context, target *querypb.Target func (sm *stateManager) serveMaster() error { sm.watcher.Close() - sm.hr.Close() if err := sm.connect(); err != nil { return err } - sm.hw.Open() + sm.rt.MakeMaster() sm.tracker.Open() if err := sm.te.AcceptReadWrite(); err != nil { return err @@ -380,13 +387,12 @@ func (sm *stateManager) unserveMaster() error { sm.unserveCommon() sm.watcher.Close() - sm.hr.Close() if err := sm.connect(); err != nil { return err } - sm.hw.Open() + sm.rt.MakeMaster() sm.tracker.Open() sm.setState(topodatapb.TabletType_MASTER, StateNotServing) return nil @@ -395,7 +401,6 @@ func (sm *stateManager) unserveMaster() error { func (sm *stateManager) serveNonMaster(wantTabletType topodatapb.TabletType) error { sm.messager.Close() sm.tracker.Close() - sm.hw.Close() sm.se.MakeNonMaster() if err := sm.connect(); err != nil { @@ -405,7 +410,7 @@ func (sm *stateManager) serveNonMaster(wantTabletType topodatapb.TabletType) err if err := sm.te.AcceptReadOnly(); err != nil { return err } - sm.hr.Open() + sm.rt.MakeNonMaster() sm.watcher.Open() sm.setState(wantTabletType, StateServing) return nil @@ -415,14 +420,13 @@ func (sm *stateManager) unserveNonMaster(wantTabletType topodatapb.TabletType) e sm.unserveCommon() sm.tracker.Close() - sm.hw.Close() sm.se.MakeNonMaster() if err := sm.connect(); err != nil { return err } - sm.hr.Open() + sm.rt.MakeNonMaster() sm.watcher.Open() sm.setState(wantTabletType, StateNotServing) return nil @@ -456,8 +460,7 @@ func (sm *stateManager) closeAll() { sm.watcher.Close() sm.tracker.Close() sm.vstreamer.Close() - sm.hr.Close() - sm.hw.Close() + sm.rt.Close() sm.se.Close() sm.setState(topodatapb.TabletType_UNKNOWN, StateNotConnected) } diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index 89ee61cbe71..6b46f4f0da1 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -62,16 +62,15 @@ func TestStateManagerServeMaster(t *testing.T) { assert.Equal(t, int32(0), sm.lameduck.Get()) verifySubcomponent(t, 1, sm.watcher, testStateClosed) - verifySubcomponent(t, 2, sm.hr, testStateClosed) - verifySubcomponent(t, 3, sm.se, testStateOpen) - verifySubcomponent(t, 4, sm.vstreamer, testStateOpen) - verifySubcomponent(t, 5, sm.qe, testStateOpen) - verifySubcomponent(t, 6, sm.txThrottler, testStateOpen) - verifySubcomponent(t, 7, sm.hw, testStateOpen) - verifySubcomponent(t, 8, sm.tracker, testStateOpen) - verifySubcomponent(t, 9, sm.te, testStateAcceptReadWrite) - verifySubcomponent(t, 10, sm.messager, testStateOpen) + verifySubcomponent(t, 2, sm.se, testStateOpen) + verifySubcomponent(t, 3, sm.vstreamer, testStateOpen) + verifySubcomponent(t, 4, sm.qe, testStateOpen) + verifySubcomponent(t, 5, sm.txThrottler, testStateOpen) + verifySubcomponent(t, 6, sm.rt, testStateMaster) + verifySubcomponent(t, 7, sm.tracker, testStateOpen) + verifySubcomponent(t, 8, sm.te, testStateMaster) + verifySubcomponent(t, 9, sm.messager, testStateOpen) assert.False(t, sm.se.(*testSchemaEngine).nonMaster) assert.True(t, sm.qe.(*testQueryEngine).isReachable) @@ -89,16 +88,15 @@ func TestStateManagerServeNonMaster(t *testing.T) { verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.tracker, testStateClosed) - verifySubcomponent(t, 3, sm.hw, testStateClosed) assert.True(t, sm.se.(*testSchemaEngine).nonMaster) - verifySubcomponent(t, 4, sm.se, testStateOpen) - verifySubcomponent(t, 5, sm.vstreamer, testStateOpen) - verifySubcomponent(t, 6, sm.qe, testStateOpen) - verifySubcomponent(t, 7, sm.txThrottler, testStateOpen) - verifySubcomponent(t, 8, sm.te, testStateAcceptReadOnly) - verifySubcomponent(t, 9, sm.hr, testStateOpen) - verifySubcomponent(t, 10, sm.watcher, testStateOpen) + verifySubcomponent(t, 3, sm.se, testStateOpen) + verifySubcomponent(t, 4, sm.vstreamer, testStateOpen) + verifySubcomponent(t, 5, sm.qe, testStateOpen) + verifySubcomponent(t, 6, sm.txThrottler, testStateOpen) + verifySubcomponent(t, 7, sm.te, testStateNonMaster) + verifySubcomponent(t, 8, sm.rt, testStateNonMaster) + verifySubcomponent(t, 9, sm.watcher, testStateOpen) assert.Equal(t, topodatapb.TabletType_REPLICA, sm.target.TabletType) assert.Equal(t, StateServing, sm.state) @@ -115,15 +113,14 @@ func TestStateManagerUnserveMaster(t *testing.T) { assert.True(t, sm.qe.(*testQueryEngine).stopServing) verifySubcomponent(t, 3, sm.watcher, testStateClosed) - verifySubcomponent(t, 4, sm.hr, testStateClosed) - verifySubcomponent(t, 5, sm.se, testStateOpen) - verifySubcomponent(t, 6, sm.vstreamer, testStateOpen) - verifySubcomponent(t, 7, sm.qe, testStateOpen) - verifySubcomponent(t, 8, sm.txThrottler, testStateOpen) + verifySubcomponent(t, 4, sm.se, testStateOpen) + verifySubcomponent(t, 5, sm.vstreamer, testStateOpen) + verifySubcomponent(t, 6, sm.qe, testStateOpen) + verifySubcomponent(t, 7, sm.txThrottler, testStateOpen) - verifySubcomponent(t, 9, sm.hw, testStateOpen) - verifySubcomponent(t, 10, sm.tracker, testStateOpen) + verifySubcomponent(t, 8, sm.rt, testStateMaster) + verifySubcomponent(t, 9, sm.tracker, testStateOpen) assert.Equal(t, topodatapb.TabletType_MASTER, sm.target.TabletType) assert.Equal(t, StateNotServing, sm.state) @@ -140,16 +137,15 @@ func TestStateManagerUnserveNonmaster(t *testing.T) { assert.True(t, sm.qe.(*testQueryEngine).stopServing) verifySubcomponent(t, 3, sm.tracker, testStateClosed) - verifySubcomponent(t, 4, sm.hw, testStateClosed) assert.True(t, sm.se.(*testSchemaEngine).nonMaster) - verifySubcomponent(t, 5, sm.se, testStateOpen) - verifySubcomponent(t, 6, sm.vstreamer, testStateOpen) - verifySubcomponent(t, 7, sm.qe, testStateOpen) - verifySubcomponent(t, 8, sm.txThrottler, testStateOpen) + verifySubcomponent(t, 4, sm.se, testStateOpen) + verifySubcomponent(t, 5, sm.vstreamer, testStateOpen) + verifySubcomponent(t, 6, sm.qe, testStateOpen) + verifySubcomponent(t, 7, sm.txThrottler, testStateOpen) - verifySubcomponent(t, 9, sm.hr, testStateOpen) - verifySubcomponent(t, 10, sm.watcher, testStateOpen) + verifySubcomponent(t, 8, sm.rt, testStateNonMaster) + verifySubcomponent(t, 9, sm.watcher, testStateOpen) assert.Equal(t, topodatapb.TabletType_RDONLY, sm.target.TabletType) assert.Equal(t, StateNotServing, sm.state) @@ -170,9 +166,8 @@ func TestStateManagerClose(t *testing.T) { verifySubcomponent(t, 5, sm.watcher, testStateClosed) verifySubcomponent(t, 6, sm.tracker, testStateClosed) verifySubcomponent(t, 7, sm.vstreamer, testStateClosed) - verifySubcomponent(t, 8, sm.hr, testStateClosed) - verifySubcomponent(t, 9, sm.hw, testStateClosed) - verifySubcomponent(t, 10, sm.se, testStateClosed) + verifySubcomponent(t, 8, sm.rt, testStateClosed) + verifySubcomponent(t, 9, sm.se, testStateClosed) assert.Equal(t, topodatapb.TabletType_RDONLY, sm.target.TabletType) assert.Equal(t, StateNotConnected, sm.state) @@ -244,16 +239,15 @@ func TestStateManagerSetServingTypeNoChange(t *testing.T) { verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.tracker, testStateClosed) - verifySubcomponent(t, 3, sm.hw, testStateClosed) assert.True(t, sm.se.(*testSchemaEngine).nonMaster) - verifySubcomponent(t, 4, sm.se, testStateOpen) - verifySubcomponent(t, 5, sm.vstreamer, testStateOpen) - verifySubcomponent(t, 6, sm.qe, testStateOpen) - verifySubcomponent(t, 7, sm.txThrottler, testStateOpen) - verifySubcomponent(t, 8, sm.te, testStateAcceptReadOnly) - verifySubcomponent(t, 9, sm.hr, testStateOpen) - verifySubcomponent(t, 10, sm.watcher, testStateOpen) + verifySubcomponent(t, 3, sm.se, testStateOpen) + verifySubcomponent(t, 4, sm.vstreamer, testStateOpen) + verifySubcomponent(t, 5, sm.qe, testStateOpen) + verifySubcomponent(t, 6, sm.txThrottler, testStateOpen) + verifySubcomponent(t, 7, sm.te, testStateNonMaster) + verifySubcomponent(t, 8, sm.rt, testStateNonMaster) + verifySubcomponent(t, 9, sm.watcher, testStateOpen) assert.Equal(t, topodatapb.TabletType_REPLICA, sm.target.TabletType) assert.Equal(t, StateServing, sm.state) @@ -457,8 +451,7 @@ func newTestStateManager(t *testing.T) *stateManager { order.Set(0) return &stateManager{ se: &testSchemaEngine{}, - hw: &testSubcomponent{}, - hr: &testSubcomponent{}, + rt: &testReplTracker{}, vstreamer: &testSubcomponent{}, tracker: &testSubcomponent{}, watcher: &testSubcomponent{}, @@ -490,8 +483,8 @@ const ( _ = testState(iota) testStateOpen testStateClosed - testStateAcceptReadOnly - testStateAcceptReadWrite + testStateMaster + testStateNonMaster ) type orderState interface { @@ -532,6 +525,29 @@ func (te *testSchemaEngine) Close() { te.state = testStateClosed } +type testReplTracker struct { + testOrderState +} + +func (te *testReplTracker) MakeMaster() { + te.order = order.Add(1) + te.state = testStateMaster +} + +func (te *testReplTracker) MakeNonMaster() { + te.order = order.Add(1) + te.state = testStateNonMaster +} + +func (te *testReplTracker) Close() { + te.order = order.Add(1) + te.state = testStateClosed +} + +func (te *testReplTracker) Status() (time.Duration, error) { + return 0, nil +} + type testQueryEngine struct { testOrderState isReachable bool @@ -570,13 +586,13 @@ type testTxEngine struct { func (te *testTxEngine) AcceptReadWrite() error { te.order = order.Add(1) - te.state = testStateAcceptReadWrite + te.state = testStateMaster return nil } func (te *testTxEngine) AcceptReadOnly() error { te.order = order.Add(1) - te.state = testStateAcceptReadOnly + te.state = testStateNonMaster return nil } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index dcb7f591d6b..91c37959803 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -53,7 +53,6 @@ import ( "vitess.io/vitess/go/vt/tableacl" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vttablet/heartbeat" "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletserver/messager" "vitess.io/vitess/go/vt/vttablet/tabletserver/planbuilder" @@ -96,9 +95,7 @@ type TabletServer struct { // These are sub-components of TabletServer. se *schema.Engine - hw *heartbeat.Writer - hr *heartbeat.Reader - rr *repltracker.ReplTracker + rt *repltracker.ReplTracker vstreamer *vstreamer.Engine tracker *schema.Tracker watcher *ReplicationWatcher @@ -157,9 +154,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsOnce.Do(func() { srvTopoServer = srvtopo.NewResilientServer(topoServer, "TabletSrvTopo") }) tsv.se = schema.NewEngine(tsv) - tsv.hw = heartbeat.NewWriter(tsv, alias) - tsv.hr = heartbeat.NewReader(tsv) - tsv.rr = repltracker.NewReplTracker(tsv, alias) + tsv.rt = repltracker.NewReplTracker(tsv, alias) tsv.vstreamer = vstreamer.NewEngine(tsv, srvTopoServer, tsv.se, alias.Cell) tsv.tracker = schema.NewTracker(tsv, tsv.vstreamer, tsv.se) tsv.watcher = NewReplicationWatcher(tsv, tsv.vstreamer, tsv.config) @@ -170,8 +165,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsv.sm = &stateManager{ se: tsv.se, - hw: tsv.hw, - hr: tsv.hr, + rt: tsv.rt, vstreamer: tsv.vstreamer, tracker: tsv.tracker, watcher: tsv.watcher, @@ -213,9 +207,7 @@ func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.D tsv.config.DB = dbcfgs tsv.se.InitDBConfig(tsv.config.DB.DbaWithDB()) - tsv.hw.InitDBConfig(target) - tsv.hr.InitDBConfig(target) - tsv.rr.InitDBConfig(target, mysqld) + tsv.rt.InitDBConfig(target, mysqld) tsv.txThrottler.InitDBConfig(target) tsv.vstreamer.InitDBConfig(target.Keyspace) return nil @@ -1442,15 +1434,7 @@ func (tsv *TabletServer) BroadcastHealth(terTimestamp int64, stats *querypb.Real // HeartbeatLag returns the current lag as calculated by the heartbeat // package, if heartbeat is enabled. Otherwise returns 0. func (tsv *TabletServer) HeartbeatLag() (time.Duration, error) { - // If the reader is closed and we are not serving, then the - // query service is shutdown and this value is not being updated. - // We return healthy from this as a signal to the healtcheck to attempt - // to start the query service again. If the query service fails to start - // with an error, then that error is be reported by the healthcheck. - if !tsv.hr.IsOpen() && !tsv.IsServing() { - return 0, nil - } - return tsv.hr.GetLatest() + return tsv.rt.Status() } // EnterLameduck causes tabletserver to enter the lameduck state. This From 52d6b06281b8043b7ed0f39cc837ceb0cc89111b Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 3 Jul 2020 21:52:28 -0700 Subject: [PATCH 010/100] vttablet: tests for ReplTracker Signed-off-by: Sugu Sougoumarane --- .../tabletserver/repltracker/poller_test.go | 53 ++++++++++ .../tabletserver/repltracker/repltracker.go | 12 +-- .../repltracker/repltracker_test.go | 96 +++++++++++++++++++ .../tabletserver/repltracker/writer_test.go | 2 +- 4 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 go/vt/vttablet/tabletserver/repltracker/poller_test.go create mode 100644 go/vt/vttablet/tabletserver/repltracker/repltracker_test.go diff --git a/go/vt/vttablet/tabletserver/repltracker/poller_test.go b/go/vt/vttablet/tabletserver/repltracker/poller_test.go new file mode 100644 index 00000000000..718b7054b3e --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/poller_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" +) + +func TestPoller(t *testing.T) { + poller := &poller{} + mysqld := fakemysqldaemon.NewFakeMysqlDaemon(nil) + poller.InitDBConfig(mysqld) + + mysqld.ReplicationStatusError = errors.New("err") + _, err := poller.Status() + assert.Equal(t, "err", err.Error()) + + mysqld.ReplicationStatusError = nil + mysqld.Replicating = false + _, err = poller.Status() + assert.Equal(t, "replication is not running", err.Error()) + + mysqld.Replicating = true + mysqld.SecondsBehindMaster = 1 + lag, err := poller.Status() + assert.NoError(t, err) + assert.Equal(t, 1*time.Second, lag) + + time.Sleep(10 * time.Millisecond) + mysqld.Replicating = false + lag, err = poller.Status() + assert.NoError(t, err) + assert.Less(t, int64(1*time.Second), int64(lag)) +} diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 2f7157ad3a5..720e0380598 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -77,12 +77,10 @@ func (rt *ReplTracker) MakeMaster() { rt.mu.Lock() defer rt.mu.Unlock() - switch rt.mode { - case tabletenv.Heartbeat: + rt.isMaster = true + if rt.mode == tabletenv.Heartbeat { rt.hr.Close() rt.hw.Open() - default: - rt.isMaster = true } } @@ -91,12 +89,10 @@ func (rt *ReplTracker) MakeNonMaster() { rt.mu.Lock() defer rt.mu.Unlock() - switch rt.mode { - case tabletenv.Heartbeat: + rt.isMaster = false + if rt.mode == tabletenv.Heartbeat { rt.hw.Close() rt.hr.Open() - default: - rt.isMaster = false } } diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go b/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go new file mode 100644 index 00000000000..632859ab33e --- /dev/null +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repltracker + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +) + +func TestReplTracker(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + + config := tabletenv.NewDefaultConfig() + config.ReplicationTracker.Mode = tabletenv.Heartbeat + config.ReplicationTracker.HeartbeatIntervalSeconds = 1 + params, _ := db.ConnParams().MysqlParams() + cp := *params + config.DB = dbconfigs.NewTestDBConfigs(cp, cp, "") + env := tabletenv.NewEnv(config, "ReplTrackerTest") + alias := topodatapb.TabletAlias{ + Cell: "cell", + Uid: 1, + } + target := querypb.Target{} + mysqld := fakemysqldaemon.NewFakeMysqlDaemon(nil) + + rt := NewReplTracker(env, alias) + rt.InitDBConfig(target, mysqld) + assert.Equal(t, tabletenv.Heartbeat, rt.mode) + assert.True(t, rt.hw.enabled) + assert.True(t, rt.hr.enabled) + + rt.MakeMaster() + assert.True(t, rt.hw.isOpen) + assert.False(t, rt.hr.isOpen) + assert.True(t, rt.isMaster) + + lag, err := rt.Status() + assert.NoError(t, err) + assert.Equal(t, time.Duration(0), lag) + + rt.MakeNonMaster() + assert.False(t, rt.hw.isOpen) + assert.True(t, rt.hr.isOpen) + assert.False(t, rt.isMaster) + + rt.hr.lastKnownLag = 1 * time.Second + lag, err = rt.Status() + assert.NoError(t, err) + assert.Equal(t, 1*time.Second, lag) + + rt.Close() + assert.False(t, rt.hw.isOpen) + assert.False(t, rt.hr.isOpen) + + config.ReplicationTracker.Mode = tabletenv.Polling + rt = NewReplTracker(env, alias) + rt.InitDBConfig(target, mysqld) + assert.Equal(t, tabletenv.Polling, rt.mode) + assert.Equal(t, mysqld, rt.poller.mysqld) + assert.False(t, rt.hw.enabled) + assert.False(t, rt.hr.enabled) + + rt.MakeNonMaster() + assert.False(t, rt.hw.isOpen) + assert.False(t, rt.hr.isOpen) + assert.False(t, rt.isMaster) + + mysqld.ReplicationStatusError = errors.New("err") + _, err = rt.Status() + assert.Equal(t, "err", err.Error()) +} diff --git a/go/vt/vttablet/tabletserver/repltracker/writer_test.go b/go/vt/vttablet/tabletserver/repltracker/writer_test.go index 7cdcc94a556..b5c55295d46 100644 --- a/go/vt/vttablet/tabletserver/repltracker/writer_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/writer_test.go @@ -21,8 +21,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gotest.tools/assert" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" From fff8340d404bfa01186014e0fb020f4dfebd1640 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 4 Jul 2020 15:03:11 -0700 Subject: [PATCH 011/100] vttablet: new healthStreamer Signed-off-by: Sugu Sougoumarane --- .../vttablet/tabletserver/health_streamer.go | 165 ++++++++++++++++ .../tabletserver/health_streamer_test.go | 187 ++++++++++++++++++ go/vt/vttablet/tabletserver/state_manager.go | 1 - .../tabletserver/state_manager_test.go | 4 - 4 files changed, 352 insertions(+), 5 deletions(-) create mode 100644 go/vt/vttablet/tabletserver/health_streamer.go create mode 100644 go/vt/vttablet/tabletserver/health_streamer_test.go diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go new file mode 100644 index 00000000000..6a6b6ab6862 --- /dev/null +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -0,0 +1,165 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver + +import ( + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "golang.org/x/net/context" + "vitess.io/vitess/go/timer" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +) + +// healthStreamer streams health information to callers. +type healthStreamer struct { + interval time.Duration + degradedThreshold time.Duration + unhealthyThreshold time.Duration + + replStatusFunc func() (time.Duration, error) + blpStatusFunc func() (int64, int32) + stats *tabletenv.Stats + ticks *timer.Timer + + mu sync.Mutex + clients map[chan *querypb.StreamHealthResponse]struct{} + state *querypb.StreamHealthResponse + // serving reflects the state of stateManager. + // We may still broadcast as not serving if there are + // replication errors, or if lag is above threshold. + serving bool +} + +func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStatusFunc func() (time.Duration, error), blpStatusFunc func() (int64, int32)) *healthStreamer { + hc := env.Config().Healthcheck + return &healthStreamer{ + interval: hc.IntervalSeconds.Get(), + degradedThreshold: hc.DegradedThresholdSeconds.Get(), + unhealthyThreshold: hc.UnhealthyThresholdSeconds.Get(), + + replStatusFunc: replStatusFunc, + blpStatusFunc: blpStatusFunc, + stats: env.Stats(), + clients: make(map[chan *querypb.StreamHealthResponse]struct{}), + ticks: timer.NewTimer(hc.IntervalSeconds.Get()), + + state: &querypb.StreamHealthResponse{ + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + HealthError: "tabletserver uninitialized", + }, + }, + } +} + +func (hs *healthStreamer) InitDBConfig(target querypb.Target) { + hs.state.Target = &target +} + +func (hs *healthStreamer) Stream(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) error { + ch := hs.register() + defer hs.unregister(ch) + + for { + select { + case <-ctx.Done(): + return nil + case shr := <-ch: + if err := callback(shr); err != nil { + return err + } + } + } +} + +func (hs *healthStreamer) register() chan *querypb.StreamHealthResponse { + hs.mu.Lock() + defer hs.mu.Unlock() + + ch := make(chan *querypb.StreamHealthResponse, 1) + hs.clients[ch] = struct{}{} + + // Start is idempotent. + hs.ticks.Start(hs.Broadcast) + + // Send the current state immediately. + ch <- proto.Clone(hs.state).(*querypb.StreamHealthResponse) + return ch +} + +func (hs *healthStreamer) unregister(ch chan *querypb.StreamHealthResponse) { + hs.mu.Lock() + defer hs.mu.Unlock() + + delete(hs.clients, ch) + + if len(hs.clients) == 0 { + hs.ticks.Stop() + } +} + +func (hs *healthStreamer) Broadcast() { + hs.mu.Lock() + defer hs.mu.Unlock() + + if len(hs.clients) == 0 { + return + } + + var healthy bool + lag, err := hs.replStatusFunc() + if err != nil { + hs.state.RealtimeStats.HealthError = err.Error() + hs.state.RealtimeStats.SecondsBehindMaster = 0 + healthy = false + } else { + hs.state.RealtimeStats.HealthError = "" + hs.state.RealtimeStats.SecondsBehindMaster = uint32(lag.Seconds()) + healthy = lag <= hs.unhealthyThreshold + } + hs.state.Serving = hs.serving && healthy + + hs.state.RealtimeStats.SecondsBehindMasterFilteredReplication, hs.state.RealtimeStats.BinlogPlayersCount = hs.blpStatusFunc() + hs.state.RealtimeStats.Qps = hs.stats.QPSRates.TotalRate() + + shr := proto.Clone(hs.state).(*querypb.StreamHealthResponse) + + for ch := range hs.clients { + select { + case ch <- shr: + default: + } + } +} + +func (hs *healthStreamer) ChangeState(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) { + hs.mu.Lock() + defer hs.mu.Unlock() + + hs.state.Target.TabletType = tabletType + if tabletType == topodatapb.TabletType_MASTER { + hs.state.TabletExternallyReparentedTimestamp = terTimestamp.Unix() + } else { + hs.state.TabletExternallyReparentedTimestamp = 0 + } + hs.serving = serving + hs.ticks.Trigger() +} diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go new file mode 100644 index 00000000000..fd4e3f9eafe --- /dev/null +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -0,0 +1,187 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +) + +func TestHealthStreamerBroadcast(t *testing.T) { + config := tabletenv.NewDefaultConfig() + config.Healthcheck.IntervalSeconds = tabletenv.Seconds(0.01) + env := tabletenv.NewEnv(config, "ReplTrackerTest") + alias := topodatapb.TabletAlias{ + Cell: "cell", + Uid: 1, + } + hs := newHealthStreamer(env, alias, replFunc, blpFunc) + target := querypb.Target{} + hs.InitDBConfig(target) + + ch, cancel := testStream(t, hs) + defer cancel() + + shr := <-ch + want := &querypb.StreamHealthResponse{ + Target: &querypb.Target{}, + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + HealthError: "tabletserver uninitialized", + }, + } + assert.Equal(t, want, shr) + + // The next fetch will broadcast newly obtained info. + shr = <-ch + want = &querypb.StreamHealthResponse{ + Target: &querypb.Target{}, + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + SecondsBehindMaster: 1, + SecondsBehindMasterFilteredReplication: 1, + BinlogPlayersCount: 2, + }, + } + assert.Equal(t, want, shr) + + // Test master and timestamp. + now := time.Now() + hs.ChangeState(topodatapb.TabletType_MASTER, now, true) + shr = <-ch + want = &querypb.StreamHealthResponse{ + Target: &querypb.Target{ + TabletType: topodatapb.TabletType_MASTER, + }, + TabletAlias: &alias, + Serving: true, + TabletExternallyReparentedTimestamp: now.Unix(), + RealtimeStats: &querypb.RealtimeStats{ + SecondsBehindMaster: 1, + SecondsBehindMasterFilteredReplication: 1, + BinlogPlayersCount: 2, + }, + } + assert.Equal(t, want, shr) + + // Test non-serving, and 0 timestamp for non-master. + hs.ChangeState(topodatapb.TabletType_REPLICA, now, false) + shr = <-ch + want = &querypb.StreamHealthResponse{ + Target: &querypb.Target{ + TabletType: topodatapb.TabletType_REPLICA, + }, + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + SecondsBehindMaster: 1, + SecondsBehindMasterFilteredReplication: 1, + BinlogPlayersCount: 2, + }, + } + assert.Equal(t, want, shr) + + // Test Health error. + hs.mu.Lock() + hs.replStatusFunc = replFuncErr + hs.mu.Unlock() + hs.ChangeState(topodatapb.TabletType_REPLICA, now, true) + shr = <-ch + want = &querypb.StreamHealthResponse{ + Target: &querypb.Target{ + TabletType: topodatapb.TabletType_REPLICA, + }, + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + HealthError: "repl err", + SecondsBehindMasterFilteredReplication: 1, + BinlogPlayersCount: 2, + }, + } + assert.Equal(t, want, shr) + + // Test Unhealthy threshold + hs.mu.Lock() + hs.replStatusFunc = replFuncUnhealthy + hs.mu.Unlock() + shr = <-ch + want = &querypb.StreamHealthResponse{ + Target: &querypb.Target{ + TabletType: topodatapb.TabletType_REPLICA, + }, + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + SecondsBehindMaster: 10800, + SecondsBehindMasterFilteredReplication: 1, + BinlogPlayersCount: 2, + }, + } + assert.Equal(t, want, shr) + + // Test everything back to normal. + hs.mu.Lock() + hs.replStatusFunc = replFunc + hs.mu.Unlock() + shr = <-ch + want = &querypb.StreamHealthResponse{ + Target: &querypb.Target{ + TabletType: topodatapb.TabletType_REPLICA, + }, + Serving: true, + TabletAlias: &alias, + RealtimeStats: &querypb.RealtimeStats{ + SecondsBehindMaster: 1, + SecondsBehindMasterFilteredReplication: 1, + BinlogPlayersCount: 2, + }, + } + assert.Equal(t, want, shr) +} + +func testStream(t *testing.T, hs *healthStreamer) (<-chan *querypb.StreamHealthResponse, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan *querypb.StreamHealthResponse) + go func() { + _ = hs.Stream(ctx, func(shr *querypb.StreamHealthResponse) error { + ch <- shr + return nil + }) + }() + return ch, cancel +} + +func replFunc() (time.Duration, error) { + return 1 * time.Second, nil +} + +func replFuncUnhealthy() (time.Duration, error) { + return 3 * time.Hour, nil +} + +func replFuncErr() (time.Duration, error) { + return 0, errors.New("repl err") +} + +func blpFunc() (int64, int32) { + return 1, 2 +} diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 3190be4c6e9..824ed7add8f 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -125,7 +125,6 @@ type ( MakeMaster() MakeNonMaster() Close() - Status() (time.Duration, error) } queryEngine interface { diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index 6b46f4f0da1..189f164efa9 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -544,10 +544,6 @@ func (te *testReplTracker) Close() { te.state = testStateClosed } -func (te *testReplTracker) Status() (time.Duration, error) { - return 0, nil -} - type testQueryEngine struct { testOrderState isReachable bool From e1a1170664beef3d364364239e0b3e051ced8d67 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 4 Jul 2020 16:59:17 -0700 Subject: [PATCH 012/100] vttablet: wire-up most of healthStreamer Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/endtoend/framework/client.go | 3 +- go/vt/vttablet/tabletmanager/healthcheck.go | 7 ++- .../tabletmanager/healthcheck_test.go | 2 +- .../vttablet/tabletmanager/rpc_replication.go | 6 +- go/vt/vttablet/tabletmanager/state_change.go | 8 +-- go/vt/vttablet/tabletserver/controller.go | 2 +- .../vttablet/tabletserver/health_streamer.go | 12 ++-- .../tabletserver/health_streamer_test.go | 5 +- go/vt/vttablet/tabletserver/state_manager.go | 17 ++++-- .../tabletserver/state_manager_test.go | 56 ++++++++++++++----- go/vt/vttablet/tabletserver/tabletserver.go | 9 ++- .../tabletserver/tabletserver_test.go | 10 ++-- go/vt/vttablet/tabletservermock/controller.go | 2 +- 13 files changed, 92 insertions(+), 47 deletions(-) diff --git a/go/vt/vttablet/endtoend/framework/client.go b/go/vt/vttablet/endtoend/framework/client.go index b38553a0e93..7f8e1a0a205 100644 --- a/go/vt/vttablet/endtoend/framework/client.go +++ b/go/vt/vttablet/endtoend/framework/client.go @@ -18,6 +18,7 @@ package framework import ( "errors" + "time" "golang.org/x/net/context" "vitess.io/vitess/go/sqltypes" @@ -161,7 +162,7 @@ func (client *QueryClient) ReadTransaction(dtid string) (*querypb.TransactionMet // SetServingType is for testing transitions. // It currently supports only master->replica and back. func (client *QueryClient) SetServingType(tabletType topodatapb.TabletType) error { - _, err := client.server.SetServingType(tabletType, true, nil) + _, err := client.server.SetServingType(tabletType, time.Time{}, true, nil) return err } diff --git a/go/vt/vttablet/tabletmanager/healthcheck.go b/go/vt/vttablet/tabletmanager/healthcheck.go index 93e66fff76f..67874d1bcc5 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck.go +++ b/go/vt/vttablet/tabletmanager/healthcheck.go @@ -182,6 +182,7 @@ func (tm *TabletManager) runHealthCheckLocked() { tm.checkLock() // read the current tablet record and tablet control tablet := tm.Tablet() + terTime := tm.masterTermStartTime() tm.mutex.Lock() shouldBeServing := tm._disallowQueryService == "" ignoreErrorExpr := tm._ignoreHealthErrorExpr @@ -237,7 +238,7 @@ func (tm *TabletManager) runHealthCheckLocked() { // We don't care if the QueryService state actually // changed because we'll broadcast the latest health // status after this immediately anyway. - _ /* state changed */, healthErr = tm.QueryServiceControl.SetServingType(tablet.Type, true, nil) + _ /* state changed */, healthErr = tm.QueryServiceControl.SetServingType(tablet.Type, terTime, true, nil) } } else { if isServing { @@ -254,7 +255,7 @@ func (tm *TabletManager) runHealthCheckLocked() { // changed because we'll broadcast the latest health // status after this immediately anyway. log.Infof("Disabling query service because of health-check failure: %v", healthErr) - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, false, nil); err != nil { + if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, terTime, false, nil); err != nil { log.Errorf("SetServingType(serving=false) failed: %v", err) } } @@ -332,5 +333,5 @@ func (tm *TabletManager) terminateHealthChecks() { // go?). After servenv lameduck, the queryservice is stopped // from a servenv.OnClose() hook anyway. log.Infof("Disabling query service after lameduck in terminating healthchecks") - tm.QueryServiceControl.SetServingType(tablet.Type, false, nil) + tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), false, nil) } diff --git a/go/vt/vttablet/tabletmanager/healthcheck_test.go b/go/vt/vttablet/tabletmanager/healthcheck_test.go index e941d808948..954f001f5d7 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck_test.go +++ b/go/vt/vttablet/tabletmanager/healthcheck_test.go @@ -430,7 +430,7 @@ func TestQueryServiceStopped(t *testing.T) { // shut down query service and prevent it from starting again // (this is to simulate mysql going away, tablet server detecting it // and shutting itself down). Intercept the message - tm.QueryServiceControl.SetServingType(topodatapb.TabletType_REPLICA, false, nil) + tm.QueryServiceControl.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, false, nil) tm.QueryServiceControl.(*tabletservermock.Controller).SetServingTypeError = fmt.Errorf("test cannot start query service") if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_REPLICA); err != nil { t.Fatal(err) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index eddeebbbbd1..cb2781865f5 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -381,12 +381,12 @@ func (tm *TabletManager) demoteMaster(ctx context.Context, revertPartialFailure // considered successful. If we are already not serving, this will be // idempotent. log.Infof("DemoteMaster disabling query service") - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, false, nil); err != nil { + if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), false, nil); err != nil { return nil, vterrors.Wrap(err, "SetServingType(serving=false) failed") } defer func() { if finalErr != nil && revertPartialFailure && wasServing { - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, true, nil); err != nil { + if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true, nil); err != nil { log.Warningf("SetServingType(serving=true) failed during revert: %v", err) } } @@ -460,7 +460,7 @@ func (tm *TabletManager) UndoDemoteMaster(ctx context.Context) error { // Update serving graph tablet := tm.Tablet() log.Infof("UndoDemoteMaster re-enabling query service") - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, true, nil); err != nil { + if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true, nil); err != nil { return vterrors.Wrap(err, "SetServingType(serving=true) failed") } diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 4b41d57238f..a02b7440866 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -173,6 +173,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable defer span.Finish() allowQuery := topo.IsRunningQueryService(newTablet.Type) + terTime := tm.masterTermStartTime() // Read the shard to get SourceShards / TabletControlMap if // we're going to use it. @@ -275,8 +276,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable newTablet.Type == topodatapb.TabletType_MASTER { // When promoting from replica to master, allow both master and replica // queries to be served during gracePeriod. - if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, - true, []topodatapb.TabletType{oldTablet.Type}); err == nil { + if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, []topodatapb.TabletType{oldTablet.Type}); err == nil { // If successful, broadcast to vtgate and then wait. tm.broadcastHealth() time.Sleep(*gracePeriod) @@ -285,7 +285,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable } } - if stateChanged, err := tm.QueryServiceControl.SetServingType(newTablet.Type, true, nil); err == nil { + if stateChanged, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, nil); err == nil { // If the state changed, broadcast to vtgate. // (e.g. this happens when the tablet was already master, but it just // changed from NOT_SERVING to SERVING due to @@ -307,7 +307,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable } log.Infof("Disabling query service on type change, reason: %v", disallowQueryReason) - if stateChanged, err := tm.QueryServiceControl.SetServingType(newTablet.Type, false, nil); err == nil { + if stateChanged, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, false, nil); err == nil { // If the state changed, broadcast to vtgate. // (e.g. this happens when the tablet was already master, but it just // changed from SERVING to NOT_SERVING because filtered replication was diff --git a/go/vt/vttablet/tabletserver/controller.go b/go/vt/vttablet/tabletserver/controller.go index ba4456895b9..cc91b14bd5e 100644 --- a/go/vt/vttablet/tabletserver/controller.go +++ b/go/vt/vttablet/tabletserver/controller.go @@ -52,7 +52,7 @@ type Controller interface { // SetServingType transitions the query service to the required serving type. // Returns true if the state of QueryService or the tablet type changed. - SetServingType(tabletType topodatapb.TabletType, serving bool, alsoAllow []topodatapb.TabletType) (bool, error) + SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool, alsoAllow []topodatapb.TabletType) (bool, error) // EnterLameduck causes tabletserver to enter the lameduck state. EnterLameduck() diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index 6a6b6ab6862..d0eef1d27e6 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -25,9 +25,14 @@ import ( "vitess.io/vitess/go/timer" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) +// blpFunc is a legaacy feature. +// TODO(sougou): remove after legacy resharding worflows are removed. +var blpFunc = vreplication.StatusSummary + // healthStreamer streams health information to callers. type healthStreamer struct { interval time.Duration @@ -35,7 +40,6 @@ type healthStreamer struct { unhealthyThreshold time.Duration replStatusFunc func() (time.Duration, error) - blpStatusFunc func() (int64, int32) stats *tabletenv.Stats ticks *timer.Timer @@ -48,7 +52,7 @@ type healthStreamer struct { serving bool } -func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStatusFunc func() (time.Duration, error), blpStatusFunc func() (int64, int32)) *healthStreamer { +func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStatusFunc func() (time.Duration, error)) *healthStreamer { hc := env.Config().Healthcheck return &healthStreamer{ interval: hc.IntervalSeconds.Get(), @@ -56,12 +60,12 @@ func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStat unhealthyThreshold: hc.UnhealthyThresholdSeconds.Get(), replStatusFunc: replStatusFunc, - blpStatusFunc: blpStatusFunc, stats: env.Stats(), clients: make(map[chan *querypb.StreamHealthResponse]struct{}), ticks: timer.NewTimer(hc.IntervalSeconds.Get()), state: &querypb.StreamHealthResponse{ + Target: &querypb.Target{}, TabletAlias: &alias, RealtimeStats: &querypb.RealtimeStats{ HealthError: "tabletserver uninitialized", @@ -137,7 +141,7 @@ func (hs *healthStreamer) Broadcast() { } hs.state.Serving = hs.serving && healthy - hs.state.RealtimeStats.SecondsBehindMasterFilteredReplication, hs.state.RealtimeStats.BinlogPlayersCount = hs.blpStatusFunc() + hs.state.RealtimeStats.SecondsBehindMasterFilteredReplication, hs.state.RealtimeStats.BinlogPlayersCount = blpFunc() hs.state.RealtimeStats.Qps = hs.stats.QPSRates.TotalRate() shr := proto.Clone(hs.state).(*querypb.StreamHealthResponse) diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index fd4e3f9eafe..d17a4b704b0 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -36,7 +36,8 @@ func TestHealthStreamerBroadcast(t *testing.T) { Cell: "cell", Uid: 1, } - hs := newHealthStreamer(env, alias, replFunc, blpFunc) + blpFunc = testBlpFunc + hs := newHealthStreamer(env, alias, replFunc) target := querypb.Target{} hs.InitDBConfig(target) @@ -182,6 +183,6 @@ func replFuncErr() (time.Duration, error) { return 0, errors.New("repl err") } -func blpFunc() (int64, int32) { +func testBlpFunc() (int64, int32) { return 1, 2 } diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 824ed7add8f..bca94eb61b9 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -89,7 +89,8 @@ type stateManager struct { target querypb.Target retrying bool // TODO(sougou): deprecate alsoAllow - alsoAllow []topodatapb.TabletType + alsoAllow []topodatapb.TabletType + terTimestamp time.Time requests sync.WaitGroup lameduck sync2.AtomicInt32 @@ -107,6 +108,10 @@ type stateManager struct { te txEngine messager subComponent + // notify will be invoked by stateManager on every state change. + // The implementation is provided by healthStreamer.Status. + notify func(topodatapb.TabletType, time.Time, bool) + // checkMySQLThrottler ensures that CheckMysql // doesn't get spammed. checkMySQLThrottler *sync2.Semaphore @@ -158,7 +163,7 @@ type ( // be honored. // If sm is already in the requested state, it returns stateChanged as // false. -func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, state servingState, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { +func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { defer sm.ExitLameduck() if tabletType == topodatapb.TabletType_RESTORE { @@ -167,7 +172,7 @@ func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, state s } log.Infof("Starting transition to %v %v", tabletType, stateName[state]) - if sm.mustTransition(tabletType, state, alsoAllow) { + if sm.mustTransition(tabletType, terTimestamp, state, alsoAllow) { return true, sm.execTransition(tabletType, state) } return false, nil @@ -177,7 +182,7 @@ func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, state s // state. If so, it acquires the semaphore and returns true. If a transition is // already in progress, it waits. If the desired state is already reached, it // returns false without acquiring the semaphore. -func (sm *stateManager) mustTransition(tabletType topodatapb.TabletType, state servingState, alsoAllow []topodatapb.TabletType) bool { +func (sm *stateManager) mustTransition(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState, alsoAllow []topodatapb.TabletType) bool { sm.transitioning.Acquire() sm.mu.Lock() defer sm.mu.Unlock() @@ -185,6 +190,7 @@ func (sm *stateManager) mustTransition(tabletType topodatapb.TabletType, state s sm.wantTabletType = tabletType sm.wantState = state sm.alsoAllow = alsoAllow + sm.terTimestamp = terTimestamp if sm.target.TabletType == tabletType && sm.state == state { sm.transitioning.Release() return false @@ -288,7 +294,7 @@ func (sm *stateManager) StopService() { defer close(sm.setTimeBomb()) log.Info("Stopping TabletServer") - sm.SetServingType(sm.Target().TabletType, StateNotConnected, nil) + sm.SetServingType(sm.Target().TabletType, time.Time{}, StateNotConnected, nil) } // StartRequest validates the current state and target and registers @@ -497,6 +503,7 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving ServingState: stateInfo(state), TabletType: sm.target.TabletType.String(), }) + sm.notify(tabletType, sm.terTimestamp, sm.state == StateServing && sm.wantState == StateServing) } // EnterLameduck causes tabletserver to enter the lameduck state. This diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index 189f164efa9..bc6845c57fa 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -31,6 +31,8 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) +var testNow = time.Now() + func TestStateManagerStateByName(t *testing.T) { states := []servingState{ StateNotConnected, @@ -55,11 +57,12 @@ func TestStateManagerStateByName(t *testing.T) { func TestStateManagerServeMaster(t *testing.T) { sm := newTestStateManager(t) sm.EnterLameduck() - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.NoError(t, err) assert.True(t, stateChanged) assert.Equal(t, int32(0), sm.lameduck.Get()) + assert.Equal(t, testNow, sm.terTimestamp) verifySubcomponent(t, 1, sm.watcher, testStateClosed) @@ -82,7 +85,7 @@ func TestStateManagerServeMaster(t *testing.T) { func TestStateManagerServeNonMaster(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -104,7 +107,7 @@ func TestStateManagerServeNonMaster(t *testing.T) { func TestStateManagerUnserveMaster(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, StateNotServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateNotServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -128,7 +131,7 @@ func TestStateManagerUnserveMaster(t *testing.T) { func TestStateManagerUnserveNonmaster(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_RDONLY, StateNotServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -153,7 +156,7 @@ func TestStateManagerUnserveNonmaster(t *testing.T) { func TestStateManagerClose(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_RDONLY, StateNotConnected, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotConnected, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -175,7 +178,7 @@ func TestStateManagerClose(t *testing.T) { func TestStateManagerStopService(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -202,7 +205,7 @@ func (te *testWatcher) Close() { go func() { defer te.wg.Done() - stateChanged, err := te.sm.SetServingType(topodatapb.TabletType_RDONLY, StateNotServing, nil) + stateChanged, err := te.sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotServing, nil) assert.NoError(te.t, err) assert.True(te.t, stateChanged) }() @@ -215,7 +218,7 @@ func TestStateManagerSetServingTypeRace(t *testing.T) { sm: sm, } sm.watcher = te - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -229,11 +232,11 @@ func TestStateManagerSetServingTypeRace(t *testing.T) { func TestStateManagerSetServingTypeNoChange(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) require.NoError(t, err) assert.True(t, stateChanged) - stateChanged, err = sm.SetServingType(topodatapb.TabletType_REPLICA, StateServing, nil) + stateChanged, err = sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) require.NoError(t, err) assert.False(t, stateChanged) @@ -260,7 +263,7 @@ func TestStateManagerTransitionFailRetry(t *testing.T) { sm := newTestStateManager(t) sm.qe.(*testQueryEngine).failMySQL = true - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.Error(t, err) assert.True(t, stateChanged) @@ -291,7 +294,7 @@ func TestStateManagerTransitionFailRetry(t *testing.T) { func TestStateManagerRestoreType(t *testing.T) { sm := newTestStateManager(t) sm.EnterLameduck() - stateChanged, err := sm.SetServingType(topodatapb.TabletType_RESTORE, StateNotServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_RESTORE, testNow, StateNotServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -306,7 +309,7 @@ func TestStateManagerCheckMySQL(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, StateServing, nil) + stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.NoError(t, err) assert.True(t, stateChanged) @@ -409,7 +412,7 @@ func TestStateManagerWaitForRequests(t *testing.T) { sm.target = *target sm.timebombDuration = 10 * time.Second - _, err := sm.SetServingType(topodatapb.TabletType_MASTER, StateServing, nil) + _, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.NoError(t, err) err = sm.StartRequest(ctx, target, false) @@ -441,6 +444,30 @@ func TestStateManagerWaitForRequests(t *testing.T) { assert.Equal(t, StateNotConnected, sm.State()) } +func TestStateManagerNotify(t *testing.T) { + sm := newTestStateManager(t) + var ( + gotType topodatapb.TabletType + gotts time.Time + gotServing bool + ) + + sm.notify = func(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) { + gotType = tabletType + gotts = terTimestamp + gotServing = serving + } + stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + require.NoError(t, err) + assert.True(t, stateChanged) + + assert.Equal(t, topodatapb.TabletType_MASTER, sm.target.TabletType) + assert.Equal(t, StateServing, sm.state) + assert.Equal(t, topodatapb.TabletType_MASTER, gotType) + assert.Equal(t, testNow, gotts) + assert.True(t, gotServing) +} + func verifySubcomponent(t *testing.T, order int64, component interface{}, state testState) { tos := component.(orderState) assert.Equal(t, order, tos.Order()) @@ -459,6 +486,7 @@ func newTestStateManager(t *testing.T) *stateManager { txThrottler: &testTxThrottler{}, te: &testTxEngine{}, messager: &testSubcomponent{}, + notify: func(topodatapb.TabletType, time.Time, bool) {}, transitioning: sync2.NewSemaphore(1, 0), checkMySQLThrottler: sync2.NewSemaphore(1, 0), diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 91c37959803..874aef24ecf 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -103,6 +103,7 @@ type TabletServer struct { txThrottler *txthrottler.TxThrottler te *TxEngine messager *messager.Engine + hs *healthStreamer // sm manages state transitions. sm *stateManager @@ -162,6 +163,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsv.txThrottler = txthrottler.NewTxThrottler(tsv.config, topoServer) tsv.te = NewTxEngine(tsv) tsv.messager = messager.NewEngine(tsv, tsv.se, tsv.vstreamer) + tsv.hs = newHealthStreamer(tsv, alias, tsv.rt.Status) tsv.sm = &stateManager{ se: tsv.se, @@ -173,6 +175,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to txThrottler: tsv.txThrottler, te: tsv.te, messager: tsv.messager, + notify: tsv.hs.ChangeState, transitioning: sync2.NewSemaphore(1, 0), checkMySQLThrottler: sync2.NewSemaphore(1, 0), @@ -307,12 +310,12 @@ func (tsv *TabletServer) InitACL(tableACLConfigFile string, enforceTableACLConfi // primary serving type, while alsoAllow specifies other tablet types that // should also be honored for serving. // Returns true if the state of QueryService or the tablet type changed. -func (tsv *TabletServer) SetServingType(tabletType topodatapb.TabletType, serving bool, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { +func (tsv *TabletServer) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { state := StateNotServing if serving { state = StateServing } - return tsv.sm.SetServingType(tabletType, state, alsoAllow) + return tsv.sm.SetServingType(tabletType, terTimestamp, state, alsoAllow) } // StartService is a convenience function for InitDBConfig->SetServingType @@ -321,7 +324,7 @@ func (tsv *TabletServer) StartService(target querypb.Target, dbcfgs *dbconfigs.D if err := tsv.InitDBConfig(target, dbcfgs, mysqld); err != nil { return err } - _, err := tsv.sm.SetServingType(target.TabletType, StateServing, nil) + _, err := tsv.sm.SetServingType(target.TabletType, time.Time{}, StateServing, nil) return err } diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index c0a5a27f445..a01b614f072 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -62,7 +62,7 @@ func TestBeginOnReplica(t *testing.T) { db.AddQueryPattern(".*", &sqltypes.Result{}) target := querypb.Target{TabletType: topodatapb.TabletType_REPLICA} - _, err := tsv.SetServingType(topodatapb.TabletType_REPLICA, true, nil) + _, err := tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, nil) require.NoError(t, err) options := querypb.ExecuteOptions{ @@ -103,7 +103,7 @@ func TestTabletServerMasterToReplica(t *testing.T) { require.NoError(t, err) ch := make(chan bool) go func() { - tsv.SetServingType(topodatapb.TabletType_REPLICA, true, []topodatapb.TabletType{topodatapb.TabletType_MASTER}) + tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, []topodatapb.TabletType{topodatapb.TabletType_MASTER}) ch <- true }() @@ -126,13 +126,13 @@ func TestTabletServerRedoLogIsKeptBetweenRestarts(t *testing.T) { _, tsv, db := newTestTxExecutor(t) defer tsv.StopService() defer db.Close() - tsv.SetServingType(topodatapb.TabletType_REPLICA, true, nil) + tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, nil) turnOnTxEngine := func() { - tsv.SetServingType(topodatapb.TabletType_MASTER, true, nil) + tsv.SetServingType(topodatapb.TabletType_MASTER, time.Time{}, true, nil) } turnOffTxEngine := func() { - tsv.SetServingType(topodatapb.TabletType_REPLICA, true, nil) + tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, nil) } tpc := tsv.te.twoPC diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go index f3aa4f26697..e4becf4f05c 100644 --- a/go/vt/vttablet/tabletservermock/controller.go +++ b/go/vt/vttablet/tabletservermock/controller.go @@ -133,7 +133,7 @@ func (tqsc *Controller) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DB } // SetServingType is part of the tabletserver.Controller interface -func (tqsc *Controller) SetServingType(tabletType topodatapb.TabletType, serving bool, alsoAllow []topodatapb.TabletType) (bool, error) { +func (tqsc *Controller) SetServingType(tabletType topodatapb.TabletType, terTime time.Time, serving bool, alsoAllow []topodatapb.TabletType) (bool, error) { tqsc.mu.Lock() defer tqsc.mu.Unlock() From addfd0e7abebbab50c5f14eacd3a8149c89a4117 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 4 Jul 2020 19:42:27 -0700 Subject: [PATCH 013/100] vttablet: streamHealth wired up Signed-off-by: Sugu Sougoumarane --- .../tabletmanager/tablet_health_test.go | 3 +- go/vt/vttablet/endtoend/misc_test.go | 17 ---- .../vttablet/tabletserver/health_streamer.go | 23 +++-- .../tabletserver/repltracker/repltracker.go | 6 +- go/vt/vttablet/tabletserver/tabletserver.go | 83 +------------------ 5 files changed, 24 insertions(+), 108 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go index 0fad0dbc81a..e4bb846c8a5 100644 --- a/go/test/endtoend/tabletmanager/tablet_health_test.go +++ b/go/test/endtoend/tabletmanager/tablet_health_test.go @@ -420,8 +420,7 @@ func TestNoMysqlHealthCheck(t *testing.T) { err = json2.Unmarshal([]byte(result), &streamHealthResponse) require.NoError(t, err) realTimeStats := streamHealthResponse.GetRealtimeStats() - secondsBehindMaster := realTimeStats.GetSecondsBehindMaster() - assert.True(t, secondsBehindMaster == 7200) + assert.Equal(t, "replication is not running", realTimeStats.HealthError) // restart replication, wait until health check goes small // (a value of zero is default and won't be in structure) diff --git a/go/vt/vttablet/endtoend/misc_test.go b/go/vt/vttablet/endtoend/misc_test.go index 41498a8e283..10e71b93d34 100644 --- a/go/vt/vttablet/endtoend/misc_test.go +++ b/go/vt/vttablet/endtoend/misc_test.go @@ -427,23 +427,6 @@ func TestStreamHealth(t *testing.T) { } } -func TestStreamHealth_Expired(t *testing.T) { - var health *querypb.StreamHealthResponse - framework.Server.BroadcastHealth(0, nil, time.Millisecond) - time.Sleep(5 * time.Millisecond) - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) - defer cancel() - if err := framework.Server.StreamHealth(ctx, func(shr *querypb.StreamHealthResponse) error { - health = shr - return io.EOF - }); err != nil { - t.Fatal(err) - } - if health != nil { - t.Errorf("Health: %v, want %v", health, nil) - } -} - func TestQueryStats(t *testing.T) { client := framework.NewClient() vstart := framework.DebugVars() diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index d0eef1d27e6..84bfd20c703 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -17,21 +17,27 @@ limitations under the License. package tabletserver import ( + "io" "sync" "time" "github.com/gogo/protobuf/proto" "golang.org/x/net/context" "vitess.io/vitess/go/timer" + "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) -// blpFunc is a legaacy feature. -// TODO(sougou): remove after legacy resharding worflows are removed. -var blpFunc = vreplication.StatusSummary +var ( + // blpFunc is a legaacy feature. + // TODO(sougou): remove after legacy resharding worflows are removed. + blpFunc = vreplication.StatusSummary + + errUnintialized = "tabletserver uninitialized" +) // healthStreamer streams health information to callers. type healthStreamer struct { @@ -68,7 +74,7 @@ func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStat Target: &querypb.Target{}, TabletAlias: &alias, RealtimeStats: &querypb.RealtimeStats{ - HealthError: "tabletserver uninitialized", + HealthError: errUnintialized, }, }, } @@ -87,7 +93,11 @@ func (hs *healthStreamer) Stream(ctx context.Context, callback func(*querypb.Str case <-ctx.Done(): return nil case shr := <-ch: + log.Infof("sending: %v", shr) if err := callback(shr); err != nil { + if err == io.EOF { + return nil + } return err } } @@ -124,10 +134,6 @@ func (hs *healthStreamer) Broadcast() { hs.mu.Lock() defer hs.mu.Unlock() - if len(hs.clients) == 0 { - return - } - var healthy bool lag, err := hs.replStatusFunc() if err != nil { @@ -157,6 +163,7 @@ func (hs *healthStreamer) Broadcast() { func (hs *healthStreamer) ChangeState(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) { hs.mu.Lock() defer hs.mu.Unlock() + log.Infof("State changed: %v %v %v", tabletType, terTimestamp, serving) hs.state.Target.TabletType = tabletType if tabletType == topodatapb.TabletType_MASTER { diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 720e0380598..86fac4a4d6e 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -90,9 +90,13 @@ func (rt *ReplTracker) MakeNonMaster() { defer rt.mu.Unlock() rt.isMaster = false - if rt.mode == tabletenv.Heartbeat { + switch rt.mode { + case tabletenv.Heartbeat: rt.hw.Close() rt.hr.Open() + case tabletenv.Polling: + // Run the status once to pre-initialize values. + rt.poller.Status() } } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 874aef24ecf..bc77602c9a5 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -19,7 +19,6 @@ package tabletserver import ( "bytes" "fmt" - "io" "net/http" "os" "os/signal" @@ -108,13 +107,6 @@ type TabletServer struct { // sm manages state transitions. sm *stateManager - // streamHealthMutex protects all the following fields - streamHealthMutex sync.Mutex - streamHealthIndex int - streamHealthMap map[int]chan<- *querypb.StreamHealthResponse - lastStreamHealthResponse *querypb.StreamHealthResponse - lastStreamHealthExpiration time.Time - // alias is used for identifying this tabletserver in healthcheck responses. alias topodatapb.TabletAlias } @@ -148,7 +140,6 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to TerseErrors: config.TerseErrors, enableHotRowProtection: config.HotRowProtection.Mode != tabletenv.Disable, topoServer: topoServer, - streamHealthMap: make(map[int]chan<- *querypb.StreamHealthResponse), alias: alias, } @@ -213,6 +204,7 @@ func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.D tsv.rt.InitDBConfig(target, mysqld) tsv.txThrottler.InitDBConfig(target) tsv.vstreamer.InitDBConfig(target.Keyspace) + tsv.hs.InitDBConfig(target) return nil } @@ -1356,82 +1348,13 @@ func convertErrorCode(err error) vtrpcpb.Code { } // StreamHealth streams the health status to callback. -// At the beginning, if TabletServer has a valid health -// state, that response is immediately sent. func (tsv *TabletServer) StreamHealth(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) error { - tsv.streamHealthMutex.Lock() - shr := tsv.lastStreamHealthResponse - shrExpiration := tsv.lastStreamHealthExpiration - tsv.streamHealthMutex.Unlock() - // Send current state immediately. - if shr != nil && time.Now().Before(shrExpiration) { - if err := callback(shr); err != nil { - if err == io.EOF { - return nil - } - return err - } - } - - // Broadcast periodic updates. - id, ch := tsv.streamHealthRegister() - defer tsv.streamHealthUnregister(id) - - for { - select { - case <-ctx.Done(): - return nil - case shr = <-ch: - } - if err := callback(shr); err != nil { - if err == io.EOF { - return nil - } - return err - } - } -} - -func (tsv *TabletServer) streamHealthRegister() (int, chan *querypb.StreamHealthResponse) { - tsv.streamHealthMutex.Lock() - defer tsv.streamHealthMutex.Unlock() - - id := tsv.streamHealthIndex - tsv.streamHealthIndex++ - ch := make(chan *querypb.StreamHealthResponse, 10) - tsv.streamHealthMap[id] = ch - return id, ch -} - -func (tsv *TabletServer) streamHealthUnregister(id int) { - tsv.streamHealthMutex.Lock() - defer tsv.streamHealthMutex.Unlock() - delete(tsv.streamHealthMap, id) + return tsv.hs.Stream(ctx, callback) } // BroadcastHealth will broadcast the current health to all listeners func (tsv *TabletServer) BroadcastHealth(terTimestamp int64, stats *querypb.RealtimeStats, maxCache time.Duration) { - target := tsv.sm.Target() - shr := &querypb.StreamHealthResponse{ - Target: &target, - TabletAlias: &tsv.alias, - Serving: tsv.IsServing(), - - TabletExternallyReparentedTimestamp: terTimestamp, - RealtimeStats: stats, - } - - tsv.streamHealthMutex.Lock() - defer tsv.streamHealthMutex.Unlock() - for _, c := range tsv.streamHealthMap { - // Do not block on any write. - select { - case c <- shr: - default: - } - } - tsv.lastStreamHealthResponse = shr - tsv.lastStreamHealthExpiration = time.Now().Add(maxCache) + tsv.hs.Broadcast() } // HeartbeatLag returns the current lag as calculated by the heartbeat From 020b2e7df5c88c9d72bd1a4b9858503b33011b44 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 4 Jul 2020 21:59:08 -0700 Subject: [PATCH 014/100] vttablet: status page uses healthStreamer Signed-off-by: Sugu Sougoumarane --- go/cmd/vttablet/status.go | 68 --------------- go/test/endtoend/clustertest/vtctld_test.go | 4 - .../vttablet/tabletserver/health_streamer.go | 15 +++- go/vt/vttablet/tabletserver/state_manager.go | 7 -- .../tabletserver/state_manager_test.go | 2 - go/vt/vttablet/tabletserver/status.go | 87 ++++++++++++++++--- go/vt/vttablet/tabletserver/tabletserver.go | 2 - 7 files changed, 85 insertions(+), 100 deletions(-) diff --git a/go/cmd/vttablet/status.go b/go/cmd/vttablet/status.go index fd4d6564cec..323e2a2405e 100644 --- a/go/cmd/vttablet/status.go +++ b/go/cmd/vttablet/status.go @@ -17,13 +17,9 @@ limitations under the License. package main import ( - "html/template" - - "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/servenv" _ "vitess.io/vitess/go/vt/status" "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/vttablet/tabletmanager" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tabletserver" ) @@ -88,62 +84,9 @@ var ( -` - - // healthTemplate is just about the tablet health - healthTemplate = ` -
Current status: {{.CurrentHTML}}
-

Polling health information from {{github_com_vitessio_vitess_health_html_name}}. ({{.Config}})

-

Health History

- - - - - - {{range .Records}} - - - - - {{end}} -
TimeHealthcheck Result
{{.Time.Format "Jan 2, 2006 at 15:04:05 (MST)"}}{{.HTML}}
-
-
healthy
-
serving traffic.
- -
unhappy
-
will serve traffic only if there are no fully healthy tablets.
- -
unhealthy
-
will not serve traffic.
-
` ) -type healthStatus struct { - Records []interface{} - Config template.HTML - current *tabletmanager.HealthRecord -} - -func (hs *healthStatus) CurrentClass() string { - if hs.current != nil { - return hs.current.Class() - } - return "unknown" -} - -func (hs *healthStatus) CurrentHTML() template.HTML { - if hs.current != nil { - return hs.current.HTML() - } - return template.HTML("unknown") -} - -func healthHTMLName() template.HTML { - return health.DefaultAggregator.HTMLName() -} - func addStatusParts(qsc tabletserver.Controller) { servenv.AddStatusPart("Tablet", tabletTemplate, func() interface{} { return map[string]interface{}{ @@ -152,17 +95,6 @@ func addStatusParts(qsc tabletserver.Controller) { "DisallowQueryService": tm.DisallowQueryService(), } }) - servenv.AddStatusFuncs(template.FuncMap{ - "github_com_vitessio_vitess_health_html_name": healthHTMLName, - }) - servenv.AddStatusPart("Health", healthTemplate, func() interface{} { - latest, _ := tm.History.Latest().(*tabletmanager.HealthRecord) - return &healthStatus{ - Records: tm.History.Records(), - Config: tabletmanager.ConfigHTML(), - current: latest, - } - }) qsc.AddStatusPart() vreplication.AddStatusPart() } diff --git a/go/test/endtoend/clustertest/vtctld_test.go b/go/test/endtoend/clustertest/vtctld_test.go index d338bd3597d..4aee80ea235 100644 --- a/go/test/endtoend/clustertest/vtctld_test.go +++ b/go/test/endtoend/clustertest/vtctld_test.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "net/http" "reflect" - "regexp" "strings" "testing" @@ -105,9 +104,6 @@ func testTabletStatus(t *testing.T) { require.Nil(t, err) result := string(respByte) log.Infof("Tablet status response: %v", result) - matched, err := regexp.Match(`Polling health information from.+MySQLReplicationLag`, []byte(result)) - require.Nil(t, err) - assert.True(t, matched) assert.True(t, strings.Contains(result, `Alias: %v, %s -> %s", sm.target.TabletType, tabletType, stateInfo(sm.state), stateInfo(state)) sm.target.TabletType = tabletType sm.state = state - sm.history.Add(&historyRecord{ - Time: time.Now(), - ServingState: stateInfo(state), - TabletType: sm.target.TabletType.String(), - }) sm.notify(tabletType, sm.terTimestamp, sm.state == StateServing && sm.wantState == StateServing) } diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index bc6845c57fa..ae8b431bd9d 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/history" "vitess.io/vitess/go/sync2" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -490,7 +489,6 @@ func newTestStateManager(t *testing.T) *stateManager { transitioning: sync2.NewSemaphore(1, 0), checkMySQLThrottler: sync2.NewSemaphore(1, 0), - history: history.New(10), timebombDuration: time.Duration(10 * time.Millisecond), } } diff --git a/go/vt/vttablet/tabletserver/status.go b/go/vt/vttablet/tabletserver/status.go index 2e9cfec53e6..d00562e06b9 100644 --- a/go/vt/vttablet/tabletserver/status.go +++ b/go/vt/vttablet/tabletserver/status.go @@ -17,7 +17,12 @@ limitations under the License. package tabletserver import ( + "fmt" + "strings" "time" + + "vitess.io/vitess/go/sync2" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) // This file contains the status web page export for tabletserver @@ -78,22 +83,32 @@ var ( ` queryserviceStatusTemplate = ` -

State: {{.State}}

-

Queryservice History

+
Current status: {{.Latest.Status}}
+

Health History

- - + + {{range .History}} - + + - {{end}}
TimeTarget Tablet TypeServing StateStatusTablet Type
{{.Time.Format "Jan 2, 2006 at 15:04:05 (MST)"}}{{.Status}} {{.TabletType}}{{.ServingState}}
+
+
healthy
+
serving traffic.
+ +
unhappy
+
will serve traffic only if there are no fully healthy tablets.
+ +
unhealthy
+
will not serve traffic.
+
QPS: {{.CurrentQPS}}
@@ -174,8 +189,8 @@ google.setOnLoadCallback(drawQPSChart); ) type queryserviceStatus struct { - State string History []interface{} + Latest *historyRecord CurrentQPS float64 } @@ -192,10 +207,19 @@ func (tsv *TabletServer) AddStatusHeader() { // AddStatusPart registers the status part for the status page. func (tsv *TabletServer) AddStatusPart() { - tsv.exporter.AddStatusPart("Queryservice", queryserviceStatusTemplate, func() interface{} { + // Save the threshold values for reporting. + degradedThreshold.Set(tsv.config.Healthcheck.DegradedThresholdSeconds.Get()) + unhealthyThreshold.Set(tsv.config.Healthcheck.UnhealthyThresholdSeconds.Get()) + + tsv.exporter.AddStatusPart("Health", queryserviceStatusTemplate, func() interface{} { status := queryserviceStatus{ - State: tsv.sm.StateByName(), - History: tsv.sm.history.Records(), + History: tsv.hs.history.Records(), + } + latest := tsv.hs.history.Latest() + if latest != nil { + status.Latest = latest.(*historyRecord) + } else { + status.Latest = &historyRecord{} } rates := tsv.stats.QPSRates.Get() if qps, ok := rates["All"]; ok && len(qps) > 0 { @@ -205,10 +229,45 @@ func (tsv *TabletServer) AddStatusPart() { }) } +var degradedThreshold sync2.AtomicDuration +var unhealthyThreshold sync2.AtomicDuration + type historyRecord struct { - Time time.Time - TabletType string - ServingState string + Time time.Time + serving bool + tabletType topodatapb.TabletType + lag time.Duration + err string +} + +func (r *historyRecord) Class() string { + if r.serving { + if r.lag > degradedThreshold.Get() { + return "unhappy" + } + return "healthy" + } + return "unhealthy" +} + +func (r *historyRecord) Status() string { + if r.serving { + if r.lag > degradedThreshold.Get() { + return fmt.Sprintf("replication delayed: %v", r.lag) + } + return "healthy" + } + if r.lag > unhealthyThreshold.Get() { + return fmt.Sprintf("not serving: replication delay %v", r.lag) + } + if r.err == "" { + return "not serving" + } + return fmt.Sprintf("not serving: %v", r.err) +} + +func (r *historyRecord) TabletType() string { + return strings.ToLower(r.tabletType.String()) } // IsDuplicate implements history.Deduplicable @@ -217,5 +276,5 @@ func (r *historyRecord) IsDuplicate(other interface{}) bool { if !ok { return false } - return r.TabletType == rother.TabletType && r.ServingState == rother.ServingState + return r.tabletType == rother.tabletType && r.serving == rother.serving && r.err == rother.err } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index bc77602c9a5..d05d379c56c 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -30,7 +30,6 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/acl" - "vitess.io/vitess/go/history" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" @@ -170,7 +169,6 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to transitioning: sync2.NewSemaphore(1, 0), checkMySQLThrottler: sync2.NewSemaphore(1, 0), - history: history.New(10), timebombDuration: time.Duration(config.OltpReadPool.TimeoutSeconds * 10), } From 9dee129e0727011bf5fd6b48d084a0d3020d267e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 5 Jul 2020 14:18:52 -0700 Subject: [PATCH 015/100] vttablet: VREngine retries if Open fails Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/healthcheck.go | 4 +- go/vt/vttablet/tabletmanager/state_change.go | 6 +- .../tabletmanager/vreplication/engine.go | 51 +++++++-- .../tabletmanager/vreplication/engine_test.go | 105 ++++++++++++------ .../vreplication/framework_test.go | 5 +- .../testlib/migrate_served_from_test.go | 4 +- .../testlib/migrate_served_types_test.go | 24 +--- go/vt/wrangler/traffic_switcher_env_test.go | 8 +- 8 files changed, 122 insertions(+), 85 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/healthcheck.go b/go/vt/vttablet/tabletmanager/healthcheck.go index 67874d1bcc5..283aaace780 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck.go +++ b/go/vt/vttablet/tabletmanager/healthcheck.go @@ -273,9 +273,7 @@ func (tm *TabletManager) runHealthCheckLocked() { // came up. This is because the mysql could have been in read-only mode, etc. // So, start the engine if it's not already running. if tablet.Type == topodatapb.TabletType_MASTER && tm.VREngine != nil && !tm.VREngine.IsOpen() { - if err := tm.VREngine.Open(tm.BatchCtx); err == nil { - log.Info("VReplication engine successfully started") - } + tm.VREngine.Open(tm.BatchCtx) } // save the health record diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index a02b7440866..4b1133e591b 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -336,11 +336,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable // See if we need to start or stop vreplication. if tm.VREngine != nil { if newTablet.Type == topodatapb.TabletType_MASTER { - if err := tm.VREngine.Open(tm.BatchCtx); err != nil { - log.Errorf("Could not start VReplication engine: %v. Will keep retrying at health check intervals.", err) - } else { - log.Info("VReplication engine started") - } + tm.VREngine.Open(tm.BatchCtx) } else { tm.VREngine.Close() } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index 53b6ea6a4b1..0e16d239a0b 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -25,6 +25,7 @@ import ( "time" "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/withddl" @@ -37,6 +38,7 @@ import ( "vitess.io/vitess/go/vt/mysqlctl" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/topo" ) @@ -84,6 +86,7 @@ type Engine struct { // mu synchronizes isOpen, controllers and wg. mu sync.Mutex isOpen bool + retrying bool controllers map[int]*controller // wg is used by in-flight functions that can run for long periods. wg sync.WaitGroup @@ -151,26 +154,52 @@ func NewTestEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, db } // Open starts the Engine service. -func (vre *Engine) Open(ctx context.Context) error { +func (vre *Engine) Open(ctx context.Context) { vre.mu.Lock() defer vre.mu.Unlock() if vre.ts == nil { log.Info("ts is nil: disabling vreplication engine") - return nil + return } if vre.isOpen { - return nil + return } + vre.isOpen = true log.Infof("Starting VReplication engine") vre.ctx, vre.cancel = context.WithCancel(ctx) - vre.isOpen = true if err := vre.initAll(); err != nil { - go vre.Close() - return err + vre.retrying = true + vre.wg.Add(1) + go vre.retry(vre.ctx, err) + return } vre.updateStats() - return nil +} + +var openRetryInterval = 1 * time.Second + +func (vre *Engine) retry(ctx context.Context, err error) { + defer vre.wg.Done() + + log.Errorf("Error starting vreplication engine: %v, will keep retrying.", err) + for { + timer := time.NewTimer(openRetryInterval) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + vre.mu.Lock() + if err := vre.initAll(); err == nil { + vre.updateStats() + vre.retrying = false + vre.mu.Unlock() + return + } + vre.mu.Unlock() + } } func (vre *Engine) initAll() error { @@ -187,7 +216,8 @@ func (vre *Engine) initAll() error { for _, row := range rows { ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil, vre) if err != nil { - return err + log.Errorf("Controller could not be initialized for stream: %v", row) + continue } vre.controllers[int(ct.id)] = ct } @@ -240,7 +270,10 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { vre.mu.Lock() defer vre.mu.Unlock() if !vre.isOpen { - return nil, errors.New("vreplication engine is closed") + return nil, vterrors.New(vtrpcpb.Code_UNAVAILABLE, "vreplication engine is closed") + } + if vre.retrying { + return nil, vterrors.New(vtrpcpb.Code_UNAVAILABLE, "engine is still trying to open") } defer vre.updateStats() diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go index b6fb0fa3d03..89e9ae5207c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go @@ -17,12 +17,15 @@ limitations under the License. package vreplication import ( + "errors" "fmt" "reflect" "strings" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/context" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" @@ -40,12 +43,8 @@ func TestEngineOpen(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} - // Test Insert - vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) - if vre.IsOpen() { - t.Errorf("IsOpen: %v, want false", vre.IsOpen()) - } + require.False(t, vre.IsOpen()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -60,23 +59,69 @@ func TestEngineOpen(t *testing.T) { dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil) dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil) dbClient.ExpectRequest("commit", nil, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) defer vre.Close() - if !vre.IsOpen() { - t.Errorf("IsOpen: %v, want true", vre.IsOpen()) - } + assert.True(t, vre.IsOpen()) // Verify stats - if !reflect.DeepEqual(globalStats.controllers, vre.controllers) { - t.Errorf("stats are mismatched: %v, want %v", globalStats.controllers, vre.controllers) - } + assert.Equal(t, globalStats.controllers, vre.controllers) ct := vre.controllers[1] - if ct == nil || ct.id != 1 { - t.Errorf("ct: %v, id should be 1", ct) + assert.True(t, ct != nil && ct.id == 1) +} + +func TestEngineOpenRetry(t *testing.T) { + defer func() { globalStats = &vrStats{} }() + + defer func(saved time.Duration) { openRetryInterval = saved }(openRetryInterval) + openRetryInterval = 10 * time.Millisecond + + defer deleteTablet(addTablet(100)) + resetBinlogClient() + dbClient := binlogplayer.NewMockDBClient(t) + dbClientFactory := func() binlogplayer.DBClient { return dbClient } + mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} + + vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) + + // Fail twice to ensure the retry retries at least once. + dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", nil, errors.New("err")) + dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", nil, errors.New("err")) + dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "id|state|source", + "int64|varchar|varchar", + ), + ), nil) + + isRetrying := func() bool { + vre.mu.Lock() + defer vre.mu.Unlock() + return vre.retrying } + + vre.Open(context.Background()) + + assert.True(t, isRetrying()) + func() { + for i := 0; i < 10; i++ { + time.Sleep(10 * time.Millisecond) + if !isRetrying() { + return + } + } + t.Error("retrying did not become false") + }() + + vre.Close() + + dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", nil, errors.New("err")) + vre.Open(context.Background()) + start := time.Now() + // Close should cause the retry to exit immediately. + vre.Close() + elapsed := time.Since(start) + assert.Greater(t, int64(openRetryInterval), int64(elapsed)) } func TestEngineExec(t *testing.T) { @@ -93,9 +138,7 @@ func TestEngineExec(t *testing.T) { vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) defer vre.Close() dbClient.ExpectRequest("use _vt", &sqltypes.Result{}, nil) @@ -253,9 +296,7 @@ func TestEngineBadInsert(t *testing.T) { vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) defer vre.Close() dbClient.ExpectRequest("use _vt", &sqltypes.Result{}, nil) @@ -283,9 +324,7 @@ func TestEngineSelect(t *testing.T) { vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) defer vre.Close() dbClient.ExpectRequest("use _vt", &sqltypes.Result{}, nil) @@ -318,9 +357,7 @@ func TestWaitForPos(t *testing.T) { vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1083"), @@ -354,9 +391,7 @@ func TestWaitForPosError(t *testing.T) { } dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) err = vre.WaitForPos(context.Background(), 1, "BadFlavor/0-1-1084") want = `parse error: unknown GTIDSet flavor "BadFlavor"` @@ -390,9 +425,7 @@ func TestWaitForPosCancel(t *testing.T) { vre := NewTestEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName(), nil) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/0-1-1083"), @@ -439,9 +472,7 @@ func TestCreateDBAndTable(t *testing.T) { tableNotFound := mysql.SQLError{Num: 1146, Message: "table not found"} dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", nil, &tableNotFound) - if err := vre.Open(context.Background()); err != nil { - t.Fatal(err) - } + vre.Open(context.Background()) defer vre.Close() // Missing db. Statement should get retried after creating everything. diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 9f18e8ac656..a4394f42805 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -109,10 +109,7 @@ func TestMain(m *testing.M) { "extb": env.Dbcfgs, } playerEngine = NewTestEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, vrepldb, externalConfig) - if err := playerEngine.Open(context.Background()); err != nil { - fmt.Fprintf(os.Stderr, "%v", err) - return 1 - } + playerEngine.Open(context.Background()) defer playerEngine.Close() if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), binlogplayer.CreateVReplicationTable()); err != nil { diff --git a/go/vt/wrangler/testlib/migrate_served_from_test.go b/go/vt/wrangler/testlib/migrate_served_from_test.go index 9b834206a98..7105f298419 100644 --- a/go/vt/wrangler/testlib/migrate_served_from_test.go +++ b/go/vt/wrangler/testlib/migrate_served_from_test.go @@ -108,9 +108,7 @@ func TestMigrateServedFrom(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } destMaster.TM.VREngine = vreplication.NewTestEngine(ts, "", destMaster.FakeMysqlDaemon, dbClientFactory, dbClient.DBName(), nil) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := destMaster.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + destMaster.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), diff --git a/go/vt/wrangler/testlib/migrate_served_types_test.go b/go/vt/wrangler/testlib/migrate_served_types_test.go index ff5e855d897..74d574be9c9 100644 --- a/go/vt/wrangler/testlib/migrate_served_types_test.go +++ b/go/vt/wrangler/testlib/migrate_served_types_test.go @@ -156,9 +156,7 @@ func TestMigrateServedTypes(t *testing.T) { dest1Master.TM.VREngine = vreplication.NewTestEngine(ts, "", dest1Master.FakeMysqlDaemon, dbClientFactory1, dbClient1.DBName(), nil) // select * from _vt.vreplication during Open dbClient1.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := dest1Master.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + dest1Master.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient1.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), @@ -184,9 +182,7 @@ func TestMigrateServedTypes(t *testing.T) { dest2Master.TM.VREngine = vreplication.NewTestEngine(ts, "", dest2Master.FakeMysqlDaemon, dbClientFactory2, dbClient2.DBName(), nil) // select * from _vt.vreplication during Open dbClient2.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := dest2Master.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + dest2Master.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient2.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), @@ -420,9 +416,7 @@ func TestMultiShardMigrateServedTypes(t *testing.T) { dest1Master.TM.VREngine = vreplication.NewTestEngine(ts, "", dest1Master.FakeMysqlDaemon, dbClientFactory1, "db", nil) // select * from _vt.vreplication during Open dbClient1.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := dest1Master.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + dest1Master.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient1.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), @@ -437,9 +431,7 @@ func TestMultiShardMigrateServedTypes(t *testing.T) { dest2Master.TM.VREngine = vreplication.NewTestEngine(ts, "", dest2Master.FakeMysqlDaemon, dbClientFactory2, "db", nil) // select * from _vt.vreplication during Open dbClient2.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := dest2Master.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + dest2Master.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient2.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ @@ -508,9 +500,7 @@ func TestMultiShardMigrateServedTypes(t *testing.T) { dest3Master.TM.VREngine = vreplication.NewTestEngine(ts, "", dest3Master.FakeMysqlDaemon, dbClientFactory1, "db", nil) // select * from _vt.vreplication during Open dbClient1.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := dest3Master.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + dest3Master.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient1.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ sqltypes.NewVarBinary("MariaDB/5-456-892"), @@ -525,9 +515,7 @@ func TestMultiShardMigrateServedTypes(t *testing.T) { dest4Master.TM.VREngine = vreplication.NewTestEngine(ts, "", dest4Master.FakeMysqlDaemon, dbClientFactory2, "db", nil) // select * from _vt.vreplication during Open dbClient2.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) - if err := dest4Master.TM.VREngine.Open(context.Background()); err != nil { - t.Fatal(err) - } + dest4Master.TM.VREngine.Open(context.Background()) // select pos, state, message from _vt.vreplication dbClient2.ExpectRequest("select pos, state, message from _vt.vreplication where id=1", &sqltypes.Result{Rows: [][]sqltypes.Value{{ diff --git a/go/vt/wrangler/traffic_switcher_env_test.go b/go/vt/wrangler/traffic_switcher_env_test.go index 2c6e5258abd..eff923af02e 100644 --- a/go/vt/wrangler/traffic_switcher_env_test.go +++ b/go/vt/wrangler/traffic_switcher_env_test.go @@ -336,9 +336,7 @@ func (tme *testMigraterEnv) createDBClients(ctx context.Context, t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbclient } // Replace existing engine with a new one master.TM.VREngine = vreplication.NewTestEngine(tme.ts, "", master.FakeMysqlDaemon, dbClientFactory, dbclient.DBName(), nil) - if err := master.TM.VREngine.Open(ctx); err != nil { - t.Fatal(err) - } + master.TM.VREngine.Open(ctx) } for _, master := range tme.targetMasters { dbclient := newFakeDBClient() @@ -346,9 +344,7 @@ func (tme *testMigraterEnv) createDBClients(ctx context.Context, t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbclient } // Replace existing engine with a new one master.TM.VREngine = vreplication.NewTestEngine(tme.ts, "", master.FakeMysqlDaemon, dbClientFactory, dbclient.DBName(), nil) - if err := master.TM.VREngine.Open(ctx); err != nil { - t.Fatal(err) - } + master.TM.VREngine.Open(ctx) } tme.allDBClients = append(tme.dbSourceClients, tme.dbTargetClients...) } From 5f968fc371632c31f6eba446c26f8ce1d28e8758 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 5 Jul 2020 16:19:42 -0700 Subject: [PATCH 016/100] tm: preparing to delete healthcheck Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/state_change.go | 66 ++----------------- go/vt/vttablet/tabletmanager/tm_init.go | 33 +++++----- .../vttablet/tabletserver/health_streamer.go | 8 +-- 3 files changed, 24 insertions(+), 83 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 4b1133e591b..7b69bece6e1 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -191,39 +191,11 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err) updateBlacklistedTables = false } else { - if oldTablet.Type == topodatapb.TabletType_RESTORE { - // always start as NON-SERVING after a restore because - // healthcheck has not been initialized yet - allowQuery = false - // setting disallowQueryService permanently turns off query service - // since we want it to be temporary (until tablet is healthy) we don't set it - // disallowQueryReason is only used for logging - disallowQueryReason = "after restore from backup" - } else { - if newTablet.Type == topodatapb.TabletType_MASTER { - if len(shardInfo.SourceShards) > 0 { - allowQuery = false - disallowQueryReason = "master tablet with filtered replication on" - disallowQueryService = disallowQueryReason - } - } else { - var replicationDelay time.Duration - var healthErr error - if tm.HealthReporter != nil { - replicationDelay, healthErr = tm.HealthReporter.Report(true, true) - } - if healthErr != nil { - allowQuery = false - disallowQueryReason = "unable to get health" - } else { - tm.mutex.Lock() - tm._replicationDelay = replicationDelay - tm.mutex.Unlock() - if tm._replicationDelay > unhealthyThreshold { - allowQuery = false - disallowQueryReason = "replica tablet with unhealthy replication lag" - } - } + if newTablet.Type == topodatapb.TabletType_MASTER { + if len(shardInfo.SourceShards) > 0 { + allowQuery = false + disallowQueryReason = "master tablet with filtered replication on" + disallowQueryService = disallowQueryReason } } srvKeyspace, err := tm.TopoServer.GetSrvKeyspace(ctx, newTablet.Alias.Cell, newTablet.Keyspace) @@ -269,7 +241,6 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable } } - broadcastHealth := false if allowQuery { // Query service should be running. if oldTablet.Type == topodatapb.TabletType_REPLICA && @@ -277,23 +248,13 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable // When promoting from replica to master, allow both master and replica // queries to be served during gracePeriod. if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, []topodatapb.TabletType{oldTablet.Type}); err == nil { - // If successful, broadcast to vtgate and then wait. - tm.broadcastHealth() time.Sleep(*gracePeriod) } else { log.Errorf("Can't start query service for MASTER+REPLICA mode: %v", err) } } - if stateChanged, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, nil); err == nil { - // If the state changed, broadcast to vtgate. - // (e.g. this happens when the tablet was already master, but it just - // changed from NOT_SERVING to SERVING due to - // "vtctl MigrateServedFrom ... master".) - if stateChanged { - broadcastHealth = true - } - } else { + if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, nil); err != nil { log.Errorf("Cannot start query service: %v", err) } } else { @@ -307,15 +268,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable } log.Infof("Disabling query service on type change, reason: %v", disallowQueryReason) - if stateChanged, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, false, nil); err == nil { - // If the state changed, broadcast to vtgate. - // (e.g. this happens when the tablet was already master, but it just - // changed from SERVING to NOT_SERVING because filtered replication was - // enabled.) - if stateChanged { - broadcastHealth = true - } - } else { + if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, false, nil); err != nil { log.Errorf("SetServingType(serving=false) failed: %v", err) } } @@ -341,11 +294,6 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable tm.VREngine.Close() } } - - // Broadcast health changes to vtgate immediately. - if broadcastHealth { - tm.broadcastHealth() - } } func (tm *TabletManager) publishState(ctx context.Context) { diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 790c1a1bbc2..859252a8b84 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -341,10 +341,6 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { servenv.OnTerm(tm.VREngine.Close) } - if err := tm.handleRestore(tm.BatchCtx); err != nil { - return err - } - // The following initializations don't need to be done // in any specific order. tm.startShardSync() @@ -359,7 +355,16 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { } servenv.OnRun(tm.registerTabletManager) - // Temporary glue code to keep things working. + restoring, err := tm.handleRestore(tm.BatchCtx) + if err != nil { + return err + } + if restoring { + // If restore was triggered, it will take care + // of updating the tablet state. + return nil + } + if err := tm.lock(tm.BatchCtx); err != nil { return err } @@ -645,11 +650,11 @@ func (tm *TabletManager) initTablet(ctx context.Context) error { return nil } -func (tm *TabletManager) handleRestore(ctx context.Context) error { +func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { tablet := tm.Tablet() // Sanity check for inconsistent flags if tm.Cnf == nil && *restoreFromBackup { - return fmt.Errorf("you cannot enable -restore_from_backup without a my.cnf file") + return false, fmt.Errorf("you cannot enable -restore_from_backup without a my.cnf file") } // two cases then: @@ -663,11 +668,8 @@ func (tm *TabletManager) handleRestore(ctx context.Context) error { if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), *waitForBackupInterval, false /* deleteBeforeRestore */); err != nil { log.Exitf("RestoreFromBackup failed: %v", err) } - - // after the restore is done, start health check - tm.initHealthCheck() }() - return nil + return true, nil } // optionally populate metadata records @@ -676,18 +678,15 @@ func (tm *TabletManager) handleRestore(ctx context.Context) error { if tm.Cnf != nil { // we are managing mysqld // we'll use batchCtx here because we are still initializing and can't proceed unless this succeeds if err := tm.MysqlDaemon.Wait(ctx, tm.Cnf); err != nil { - return err + return false, err } } err := mysqlctl.PopulateMetadataTables(tm.MysqlDaemon, localMetadata, topoproto.TabletDbName(tablet)) if err != nil { - return vterrors.Wrap(err, "failed to -init_populate_metadata") + return false, vterrors.Wrap(err, "failed to -init_populate_metadata") } } - - // synchronously start health check if needed - tm.initHealthCheck() - return nil + return false, nil } func (tm *TabletManager) exportStats() { diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index ea904937053..cd53abeeb5e 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -86,6 +86,7 @@ func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStat func (hs *healthStreamer) InitDBConfig(target querypb.Target) { hs.state.Target = &target + hs.ticks.Start(hs.Broadcast) } func (hs *healthStreamer) Stream(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) error { @@ -114,9 +115,6 @@ func (hs *healthStreamer) register() chan *querypb.StreamHealthResponse { ch := make(chan *querypb.StreamHealthResponse, 1) hs.clients[ch] = struct{}{} - // Start is idempotent. - hs.ticks.Start(hs.Broadcast) - // Send the current state immediately. ch <- proto.Clone(hs.state).(*querypb.StreamHealthResponse) return ch @@ -127,10 +125,6 @@ func (hs *healthStreamer) unregister(ch chan *querypb.StreamHealthResponse) { defer hs.mu.Unlock() delete(hs.clients, ch) - - if len(hs.clients) == 0 { - hs.ticks.Stop() - } } func (hs *healthStreamer) Broadcast() { From b4f22079212c0f7f52c3658c6c556c5aa325ac7d Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 6 Jul 2020 00:20:59 -0700 Subject: [PATCH 017/100] vttablet: trying a different heathcheck Signed-off-by: Sugu Sougoumarane --- .../vttablet/tabletserver/health_streamer.go | 57 ++------ .../tabletserver/health_streamer_test.go | 69 ++------- go/vt/vttablet/tabletserver/state_manager.go | 131 ++++++++++++------ .../tabletserver/state_manager_test.go | 76 ++++++---- go/vt/vttablet/tabletserver/status.go | 8 +- go/vt/vttablet/tabletserver/tabletserver.go | 17 ++- 6 files changed, 172 insertions(+), 186 deletions(-) diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index cd53abeeb5e..c62e47844d1 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -24,7 +24,6 @@ import ( "github.com/gogo/protobuf/proto" "golang.org/x/net/context" "vitess.io/vitess/go/history" - "vitess.io/vitess/go/timer" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" @@ -41,36 +40,19 @@ var ( // healthStreamer streams health information to callers. type healthStreamer struct { - interval time.Duration - degradedThreshold time.Duration - unhealthyThreshold time.Duration - - replStatusFunc func() (time.Duration, error) - stats *tabletenv.Stats - ticks *timer.Timer + stats *tabletenv.Stats mu sync.Mutex clients map[chan *querypb.StreamHealthResponse]struct{} state *querypb.StreamHealthResponse - // serving reflects the state of stateManager. - // We may still broadcast as not serving if there are - // replication errors, or if lag is above threshold. - serving bool history *history.History } -func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStatusFunc func() (time.Duration, error)) *healthStreamer { - hc := env.Config().Healthcheck +func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias) *healthStreamer { return &healthStreamer{ - interval: hc.IntervalSeconds.Get(), - degradedThreshold: hc.DegradedThresholdSeconds.Get(), - unhealthyThreshold: hc.UnhealthyThresholdSeconds.Get(), - - replStatusFunc: replStatusFunc, - stats: env.Stats(), - clients: make(map[chan *querypb.StreamHealthResponse]struct{}), - ticks: timer.NewTimer(hc.IntervalSeconds.Get()), + stats: env.Stats(), + clients: make(map[chan *querypb.StreamHealthResponse]struct{}), state: &querypb.StreamHealthResponse{ Target: &querypb.Target{}, @@ -86,7 +68,6 @@ func newHealthStreamer(env tabletenv.Env, alias topodatapb.TabletAlias, replStat func (hs *healthStreamer) InitDBConfig(target querypb.Target) { hs.state.Target = &target - hs.ticks.Start(hs.Broadcast) } func (hs *healthStreamer) Stream(ctx context.Context, callback func(*querypb.StreamHealthResponse) error) error { @@ -127,22 +108,24 @@ func (hs *healthStreamer) unregister(ch chan *querypb.StreamHealthResponse) { delete(hs.clients, ch) } -func (hs *healthStreamer) Broadcast() { +func (hs *healthStreamer) ChangeState(tabletType topodatapb.TabletType, terTimestamp time.Time, lag time.Duration, err error, serving bool) { hs.mu.Lock() defer hs.mu.Unlock() - var healthy bool - lag, err := hs.replStatusFunc() + hs.state.Target.TabletType = tabletType + if tabletType == topodatapb.TabletType_MASTER { + hs.state.TabletExternallyReparentedTimestamp = terTimestamp.Unix() + } else { + hs.state.TabletExternallyReparentedTimestamp = 0 + } if err != nil { hs.state.RealtimeStats.HealthError = err.Error() hs.state.RealtimeStats.SecondsBehindMaster = 0 - healthy = false } else { hs.state.RealtimeStats.HealthError = "" hs.state.RealtimeStats.SecondsBehindMaster = uint32(lag.Seconds()) - healthy = lag <= hs.unhealthyThreshold } - hs.state.Serving = hs.serving && healthy + hs.state.Serving = serving hs.state.RealtimeStats.SecondsBehindMasterFilteredReplication, hs.state.RealtimeStats.BinlogPlayersCount = blpFunc() hs.state.RealtimeStats.Qps = hs.stats.QPSRates.TotalRate() @@ -160,20 +143,6 @@ func (hs *healthStreamer) Broadcast() { serving: shr.Serving, tabletType: shr.Target.TabletType, lag: lag, - err: shr.RealtimeStats.HealthError, + err: err, }) } - -func (hs *healthStreamer) ChangeState(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) { - hs.mu.Lock() - defer hs.mu.Unlock() - - hs.state.Target.TabletType = tabletType - if tabletType == topodatapb.TabletType_MASTER { - hs.state.TabletExternallyReparentedTimestamp = terTimestamp.Unix() - } else { - hs.state.TabletExternallyReparentedTimestamp = 0 - } - hs.serving = serving - hs.ticks.Trigger() -} diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index d17a4b704b0..8410767d694 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -30,14 +30,13 @@ import ( func TestHealthStreamerBroadcast(t *testing.T) { config := tabletenv.NewDefaultConfig() - config.Healthcheck.IntervalSeconds = tabletenv.Seconds(0.01) env := tabletenv.NewEnv(config, "ReplTrackerTest") alias := topodatapb.TabletAlias{ Cell: "cell", Uid: 1, } blpFunc = testBlpFunc - hs := newHealthStreamer(env, alias, replFunc) + hs := newHealthStreamer(env, alias) target := querypb.Target{} hs.InitDBConfig(target) @@ -54,13 +53,14 @@ func TestHealthStreamerBroadcast(t *testing.T) { } assert.Equal(t, want, shr) - // The next fetch will broadcast newly obtained info. + hs.ChangeState(topodatapb.TabletType_REPLICA, time.Time{}, 0, nil, false) shr = <-ch want = &querypb.StreamHealthResponse{ - Target: &querypb.Target{}, + Target: &querypb.Target{ + TabletType: topodatapb.TabletType_REPLICA, + }, TabletAlias: &alias, RealtimeStats: &querypb.RealtimeStats{ - SecondsBehindMaster: 1, SecondsBehindMasterFilteredReplication: 1, BinlogPlayersCount: 2, }, @@ -69,7 +69,7 @@ func TestHealthStreamerBroadcast(t *testing.T) { // Test master and timestamp. now := time.Now() - hs.ChangeState(topodatapb.TabletType_MASTER, now, true) + hs.ChangeState(topodatapb.TabletType_MASTER, now, 0, nil, true) shr = <-ch want = &querypb.StreamHealthResponse{ Target: &querypb.Target{ @@ -79,7 +79,6 @@ func TestHealthStreamerBroadcast(t *testing.T) { Serving: true, TabletExternallyReparentedTimestamp: now.Unix(), RealtimeStats: &querypb.RealtimeStats{ - SecondsBehindMaster: 1, SecondsBehindMasterFilteredReplication: 1, BinlogPlayersCount: 2, }, @@ -87,7 +86,7 @@ func TestHealthStreamerBroadcast(t *testing.T) { assert.Equal(t, want, shr) // Test non-serving, and 0 timestamp for non-master. - hs.ChangeState(topodatapb.TabletType_REPLICA, now, false) + hs.ChangeState(topodatapb.TabletType_REPLICA, now, 1*time.Second, nil, false) shr = <-ch want = &querypb.StreamHealthResponse{ Target: &querypb.Target{ @@ -103,10 +102,7 @@ func TestHealthStreamerBroadcast(t *testing.T) { assert.Equal(t, want, shr) // Test Health error. - hs.mu.Lock() - hs.replStatusFunc = replFuncErr - hs.mu.Unlock() - hs.ChangeState(topodatapb.TabletType_REPLICA, now, true) + hs.ChangeState(topodatapb.TabletType_REPLICA, now, 0, errors.New("repl err"), false) shr = <-ch want = &querypb.StreamHealthResponse{ Target: &querypb.Target{ @@ -120,43 +116,6 @@ func TestHealthStreamerBroadcast(t *testing.T) { }, } assert.Equal(t, want, shr) - - // Test Unhealthy threshold - hs.mu.Lock() - hs.replStatusFunc = replFuncUnhealthy - hs.mu.Unlock() - shr = <-ch - want = &querypb.StreamHealthResponse{ - Target: &querypb.Target{ - TabletType: topodatapb.TabletType_REPLICA, - }, - TabletAlias: &alias, - RealtimeStats: &querypb.RealtimeStats{ - SecondsBehindMaster: 10800, - SecondsBehindMasterFilteredReplication: 1, - BinlogPlayersCount: 2, - }, - } - assert.Equal(t, want, shr) - - // Test everything back to normal. - hs.mu.Lock() - hs.replStatusFunc = replFunc - hs.mu.Unlock() - shr = <-ch - want = &querypb.StreamHealthResponse{ - Target: &querypb.Target{ - TabletType: topodatapb.TabletType_REPLICA, - }, - Serving: true, - TabletAlias: &alias, - RealtimeStats: &querypb.RealtimeStats{ - SecondsBehindMaster: 1, - SecondsBehindMasterFilteredReplication: 1, - BinlogPlayersCount: 2, - }, - } - assert.Equal(t, want, shr) } func testStream(t *testing.T, hs *healthStreamer) (<-chan *querypb.StreamHealthResponse, context.CancelFunc) { @@ -171,18 +130,6 @@ func testStream(t *testing.T, hs *healthStreamer) (<-chan *querypb.StreamHealthR return ch, cancel } -func replFunc() (time.Duration, error) { - return 1 * time.Second, nil -} - -func replFuncUnhealthy() (time.Duration, error) { - return 3 * time.Hour, nil -} - -func replFuncErr() (time.Duration, error) { - return 0, errors.New("repl err") -} - func testBlpFunc() (int64, int32) { return 1, 2 } diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 8f5b3a96761..1088f52a518 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -23,6 +23,7 @@ import ( "time" "vitess.io/vitess/go/sync2" + "vitess.io/vitess/go/timer" "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -47,22 +48,6 @@ const ( // transitionRetryInterval is for tests. var transitionRetryInterval = 1 * time.Second -// stateName names every state. The number of elements must -// match the number of states. Names can overlap. -var stateName = []string{ - "NOT_SERVING", - "NOT_SERVING", - "SERVING", -} - -// stateDetail matches every state and optionally more information about the reason -// why the state is serving / not serving. -var stateDetail = []string{ - "Not Connected", - "Not Serving", - "", -} - // stateManager manages state transition for all the TabletServer // subcomponents. type stateManager struct { @@ -72,6 +57,7 @@ type stateManager struct { // because we need TryAcquire, which is not supported by sync.Mutex. // If an acquire is successful, we must either Release explicitly // or invoke execTransition, which will release once it's done. + // There are no ordering restrictions on using TryAcquire. transitioning *sync2.Semaphore // mu should be held to access the group of variables under it. @@ -90,9 +76,10 @@ type stateManager struct { // TODO(sougou): deprecate alsoAllow alsoAllow []topodatapb.TabletType terTimestamp time.Time + replHealthy bool requests sync.WaitGroup - lameduck sync2.AtomicInt32 + lameduck sync2.AtomicBool // Open must be done in forward order. // Close must be done in reverse order. @@ -108,13 +95,18 @@ type stateManager struct { messager subComponent // notify will be invoked by stateManager on every state change. - // The implementation is provided by healthStreamer.Status. - notify func(topodatapb.TabletType, time.Time, bool) + // The implementation is provided by healthStreamer.ChangeState. + notify func(topodatapb.TabletType, time.Time, time.Duration, error, bool) + + // hcticks starts on initialiazation and runs forever. + hcticks *timer.Timer // checkMySQLThrottler ensures that CheckMysql // doesn't get spammed. checkMySQLThrottler *sync2.Semaphore - timebombDuration time.Duration + + timebombDuration time.Duration + unhealthyThreshold time.Duration } type ( @@ -128,6 +120,7 @@ type ( MakeMaster() MakeNonMaster() Close() + Status() (time.Duration, error) } queryEngine interface { @@ -154,6 +147,16 @@ type ( } ) +// Init performs the second phase of initialization. +func (sm *stateManager) Init(env tabletenv.Env, target querypb.Target) { + sm.target = target + sm.transitioning = sync2.NewSemaphore(1, 0) + sm.checkMySQLThrottler = sync2.NewSemaphore(1, 0) + sm.timebombDuration = env.Config().OltpReadPool.TimeoutSeconds.Get() * 10 + sm.hcticks = timer.NewTimer(env.Config().Healthcheck.IntervalSeconds.Get()) + sm.unhealthyThreshold = env.Config().Healthcheck.UnhealthyThresholdSeconds.Get() +} + // SetServingType changes the state to the specified settings. // If a transition is in progress, it waits and then executes the // new request. If the transition fails, it returns an error, and @@ -164,12 +167,15 @@ type ( func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { defer sm.ExitLameduck() + // Start is idempotent. + sm.hcticks.Start(sm.Broadcast) + if tabletType == topodatapb.TabletType_RESTORE { // TODO(sougou): remove this code once tm can give us more accurate state requests. state = StateNotConnected } - log.Infof("Starting transition to %v %v", tabletType, stateName[state]) + log.Infof("Starting transition to %v %v", tabletType, stateName(state)) if sm.mustTransition(tabletType, terTimestamp, state, alsoAllow) { return true, sm.execTransition(tabletType, state) } @@ -217,7 +223,7 @@ func (sm *stateManager) execTransition(tabletType topodatapb.TabletType, state s sm.closeAll() } if err != nil { - sm.retryTransition(fmt.Sprintf("Error transitioning to the desired state: %v, %v, will keep retrying: %v", tabletType, stateName[state], err)) + sm.retryTransition(fmt.Sprintf("Error transitioning to the desired state: %v, %v, will keep retrying: %v", tabletType, stateName(state), err)) } return err } @@ -242,17 +248,16 @@ func (sm *stateManager) retryTransition(message string) { } func (sm *stateManager) recheckState() bool { - if !sm.transitioning.TryAcquire() { - return false - } sm.mu.Lock() defer sm.mu.Unlock() if sm.wantState == sm.state && sm.wantTabletType == sm.target.TabletType { sm.retrying = false - sm.transitioning.Release() return true } + if !sm.transitioning.TryAcquire() { + return false + } go sm.execTransition(sm.wantTabletType, sm.wantState) return false } @@ -292,6 +297,8 @@ func (sm *stateManager) StopService() { defer close(sm.setTimeBomb()) log.Info("Stopping TabletServer") + // Stop replica tracking because StopService is used by all tests. + sm.hcticks.Stop() sm.SetServingType(sm.Target().TabletType, time.Time{}, StateNotConnected, nil) } @@ -302,8 +309,9 @@ func (sm *stateManager) StartRequest(ctx context.Context, target *querypb.Target sm.mu.Lock() defer sm.mu.Unlock() - if sm.state != StateServing { - return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "operation not allowed in state %s", stateName[sm.state]) + if sm.state != StateServing || !sm.replHealthy { + // This specific error string needs to be returned for vtgate buffering to work. + return vterrors.New(vtrpcpb.Code_FAILED_PRECONDITION, "operation not allowed in state NOT_SERVING") } shuttingDown := sm.wantState != StateServing @@ -457,6 +465,8 @@ func (sm *stateManager) unserveCommon() { } func (sm *stateManager) closeAll() { + defer close(sm.setTimeBomb()) + sm.unserveCommon() sm.txThrottler.Close() sm.qe.Close() @@ -493,10 +503,29 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving if tabletType == topodatapb.TabletType_UNKNOWN { tabletType = sm.wantTabletType } - log.Infof("TabletServer transition: %v -> %v, %s -> %s", sm.target.TabletType, tabletType, stateInfo(sm.state), stateInfo(state)) + log.Infof("TabletServer transition: %v -> %v, %s -> %s", sm.target.TabletType, tabletType, stateName(sm.state), stateName(state)) sm.target.TabletType = tabletType sm.state = state - sm.notify(tabletType, sm.terTimestamp, sm.state == StateServing && sm.wantState == StateServing) + // Broadcast also obtains a lock. Trigger in a goroutine to avoid a deadlock. + go sm.hcticks.Trigger() +} + +// Broadcast fetches the replication status and broadcasts +// the state to all subscribed. +func (sm *stateManager) Broadcast() { + lag, err := sm.rt.Status() + + sm.mu.Lock() + defer sm.mu.Unlock() + + if err != nil { + sm.replHealthy = false + } else { + sm.replHealthy = lag <= sm.unhealthyThreshold + } + isServing := sm.isServingLocked() + + sm.notify(sm.target.TabletType, sm.terTimestamp, lag, err, isServing) } // EnterLameduck causes tabletserver to enter the lameduck state. This @@ -504,22 +533,34 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving // otherwise remains the same. Any subsequent calls to SetServingType will // cause the tabletserver to exit this mode. func (sm *stateManager) EnterLameduck() { - sm.lameduck.Set(1) + sm.lameduck.Set(true) } // ExitLameduck causes the tabletserver to exit the lameduck mode. func (sm *stateManager) ExitLameduck() { - sm.lameduck.Set(0) + sm.lameduck.Set(false) } // IsServing returns true if TabletServer is in SERVING state. func (sm *stateManager) IsServing() bool { - return sm.StateByName() == "SERVING" + sm.mu.Lock() + defer sm.mu.Unlock() + return sm.isServingLocked() +} + +func (sm *stateManager) isServingLocked() bool { + return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.lameduck.Get() } func (sm *stateManager) State() servingState { sm.mu.Lock() defer sm.mu.Unlock() + // We should not change these state numbers without + // an announcement. Even though this is not perfect, + // this behavior keeps things backward compatible. + if !sm.replHealthy { + return StateNotConnected + } return sm.state } @@ -530,19 +571,21 @@ func (sm *stateManager) Target() querypb.Target { return target } -// StateByName returns the name of the current TabletServer state. -func (sm *stateManager) StateByName() string { - if sm.lameduck.Get() != 0 { - return "NOT_SERVING" +// IsServingString returns the name of the current TabletServer state. +func (sm *stateManager) IsServingString() string { + if sm.IsServing() { + return "SERVING" } - return stateName[sm.State()] + return "NOT_SERVING" } -// stateInfo returns a string representation of the state and optional detail -// about the reason for the state transition -func stateInfo(state servingState) string { - if state == StateServing { - return "SERVING" +// stateName returns a string representation of the state. +func stateName(state servingState) string { + switch state { + case StateServing: + return "Serving" + case StateNotServing: + return "Not Serving" } - return fmt.Sprintf("%s (%s)", stateName[state], stateDetail[state]) + return "Not connected to mysql" } diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index ae8b431bd9d..3a9d4326877 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -33,24 +33,29 @@ import ( var testNow = time.Now() func TestStateManagerStateByName(t *testing.T) { - states := []servingState{ - StateNotConnected, - StateNotServing, - StateServing, - } - // Don't reuse stateName. - names := []string{ - "NOT_SERVING", - "NOT_SERVING", - "SERVING", - } sm := &stateManager{} - for i, state := range states { - sm.state = state - require.Equal(t, names[i], sm.StateByName(), "StateByName") - } + + sm.replHealthy = true + sm.wantState = StateServing + sm.state = StateNotConnected + assert.Equal(t, "NOT_SERVING", sm.IsServingString()) + + sm.state = StateNotServing + assert.Equal(t, "NOT_SERVING", sm.IsServingString()) + + sm.state = StateServing + assert.Equal(t, "SERVING", sm.IsServingString()) + + sm.wantState = StateNotServing + assert.Equal(t, "NOT_SERVING", sm.IsServingString()) + sm.wantState = StateServing + sm.EnterLameduck() - require.Equal(t, "NOT_SERVING", sm.StateByName(), "StateByName") + assert.Equal(t, "NOT_SERVING", sm.IsServingString()) + sm.ExitLameduck() + + sm.replHealthy = false + assert.Equal(t, "NOT_SERVING", sm.IsServingString()) } func TestStateManagerServeMaster(t *testing.T) { @@ -60,7 +65,7 @@ func TestStateManagerServeMaster(t *testing.T) { require.NoError(t, err) assert.True(t, stateChanged) - assert.Equal(t, int32(0), sm.lameduck.Get()) + assert.Equal(t, false, sm.lameduck.Get()) assert.Equal(t, testNow, sm.terTimestamp) verifySubcomponent(t, 1, sm.watcher, testStateClosed) @@ -358,6 +363,13 @@ func TestStateManagerValidations(t *testing.T) { err := sm.StartRequest(ctx, target, false) assert.Contains(t, err.Error(), "operation not allowed") + sm.replHealthy = false + sm.state = StateServing + sm.wantState = StateServing + err = sm.StartRequest(ctx, target, false) + assert.Contains(t, err.Error(), "operation not allowed") + + sm.replHealthy = true sm.state = StateServing sm.wantState = StateNotServing err = sm.StartRequest(ctx, target, false) @@ -411,6 +423,7 @@ func TestStateManagerWaitForRequests(t *testing.T) { sm.target = *target sm.timebombDuration = 10 * time.Second + sm.replHealthy = true _, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.NoError(t, err) @@ -448,13 +461,19 @@ func TestStateManagerNotify(t *testing.T) { var ( gotType topodatapb.TabletType gotts time.Time + gotlag time.Duration + goterr error gotServing bool ) - sm.notify = func(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) { + ch := make(chan struct{}) + sm.notify = func(tabletType topodatapb.TabletType, terTimestamp time.Time, lag time.Duration, err error, serving bool) { gotType = tabletType gotts = terTimestamp + gotlag = lag + goterr = err gotServing = serving + ch <- struct{}{} } stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) require.NoError(t, err) @@ -462,8 +481,13 @@ func TestStateManagerNotify(t *testing.T) { assert.Equal(t, topodatapb.TabletType_MASTER, sm.target.TabletType) assert.Equal(t, StateServing, sm.state) + + <-ch + sm.hcticks.Stop() assert.Equal(t, topodatapb.TabletType_MASTER, gotType) assert.Equal(t, testNow, gotts) + assert.Equal(t, 1*time.Second, gotlag) + assert.Equal(t, nil, goterr) assert.True(t, gotServing) } @@ -475,7 +499,7 @@ func verifySubcomponent(t *testing.T, order int64, component interface{}, state func newTestStateManager(t *testing.T) *stateManager { order.Set(0) - return &stateManager{ + sm := &stateManager{ se: &testSchemaEngine{}, rt: &testReplTracker{}, vstreamer: &testSubcomponent{}, @@ -485,12 +509,12 @@ func newTestStateManager(t *testing.T) *stateManager { txThrottler: &testTxThrottler{}, te: &testTxEngine{}, messager: &testSubcomponent{}, - notify: func(topodatapb.TabletType, time.Time, bool) {}, - - transitioning: sync2.NewSemaphore(1, 0), - checkMySQLThrottler: sync2.NewSemaphore(1, 0), - timebombDuration: time.Duration(10 * time.Millisecond), + notify: func(topodatapb.TabletType, time.Time, time.Duration, error, bool) {}, } + config := tabletenv.NewDefaultConfig() + env := tabletenv.NewEnv(config, "StateManagerTest") + sm.Init(env, querypb.Target{}) + return sm } func (sm *stateManager) isTransitioning() bool { @@ -570,6 +594,10 @@ func (te *testReplTracker) Close() { te.state = testStateClosed } +func (te *testReplTracker) Status() (time.Duration, error) { + return 1 * time.Second, nil +} + type testQueryEngine struct { testOrderState isReachable bool diff --git a/go/vt/vttablet/tabletserver/status.go b/go/vt/vttablet/tabletserver/status.go index d00562e06b9..2c42b915142 100644 --- a/go/vt/vttablet/tabletserver/status.go +++ b/go/vt/vttablet/tabletserver/status.go @@ -237,7 +237,7 @@ type historyRecord struct { serving bool tabletType topodatapb.TabletType lag time.Duration - err string + err error } func (r *historyRecord) Class() string { @@ -260,10 +260,10 @@ func (r *historyRecord) Status() string { if r.lag > unhealthyThreshold.Get() { return fmt.Sprintf("not serving: replication delay %v", r.lag) } - if r.err == "" { - return "not serving" + if r.err != nil { + return fmt.Sprintf("not serving: %v", r.err) } - return fmt.Sprintf("not serving: %v", r.err) + return "not serving" } func (r *historyRecord) TabletType() string { diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index d05d379c56c..e03d294420b 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -153,7 +153,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsv.txThrottler = txthrottler.NewTxThrottler(tsv.config, topoServer) tsv.te = NewTxEngine(tsv) tsv.messager = messager.NewEngine(tsv, tsv.se, tsv.vstreamer) - tsv.hs = newHealthStreamer(tsv, alias, tsv.rt.Status) + tsv.hs = newHealthStreamer(tsv, alias) tsv.sm = &stateManager{ se: tsv.se, @@ -166,19 +166,15 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to te: tsv.te, messager: tsv.messager, notify: tsv.hs.ChangeState, - - transitioning: sync2.NewSemaphore(1, 0), - checkMySQLThrottler: sync2.NewSemaphore(1, 0), - timebombDuration: time.Duration(config.OltpReadPool.TimeoutSeconds * 10), } tsv.exporter.NewGaugeFunc("TabletState", "Tablet server state", func() int64 { return int64(tsv.sm.State()) }) - tsv.exporter.Publish("TabletStateName", stats.StringFunc(tsv.sm.StateByName)) + tsv.exporter.Publish("TabletStateName", stats.StringFunc(tsv.sm.IsServingString)) // TabletServerState exports the same information as the above two stats (TabletState / TabletStateName), // but exported with TabletStateName as a label for Prometheus, which doesn't support exporting strings as stat values. tsv.exporter.NewGaugesFuncWithMultiLabels("TabletServerState", "Tablet server state labeled by state name", []string{"name"}, func() map[string]int64 { - return map[string]int64{tsv.sm.StateByName(): 1} + return map[string]int64{tsv.sm.IsServingString(): 1} }) tsv.exporter.NewGaugeDurationFunc("QueryTimeout", "Tablet server query timeout", tsv.QueryTimeout.Get) @@ -193,8 +189,9 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to // to complete the creation of TabletServer. func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DBConfigs, mysqld mysqlctl.MysqlDaemon) error { if tsv.sm.State() != StateNotConnected { - return vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "InitDBConfig failed, current state: %s", tsv.sm.StateByName()) + return vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "InitDBConfig failed, current state: %s", tsv.sm.IsServingString()) } + tsv.sm.Init(tsv, target) tsv.sm.target = target tsv.config.DB = dbcfgs @@ -314,6 +311,8 @@ func (tsv *TabletServer) StartService(target querypb.Target, dbcfgs *dbconfigs.D if err := tsv.InitDBConfig(target, dbcfgs, mysqld); err != nil { return err } + // StartService is only used for testing. So, we cheat by aggressively setting replication to healthy. + tsv.sm.replHealthy = true _, err := tsv.sm.SetServingType(target.TabletType, time.Time{}, StateServing, nil) return err } @@ -1352,7 +1351,7 @@ func (tsv *TabletServer) StreamHealth(ctx context.Context, callback func(*queryp // BroadcastHealth will broadcast the current health to all listeners func (tsv *TabletServer) BroadcastHealth(terTimestamp int64, stats *querypb.RealtimeStats, maxCache time.Duration) { - tsv.hs.Broadcast() + tsv.sm.Broadcast() } // HeartbeatLag returns the current lag as calculated by the heartbeat From efcf8b32c92c7e24f7f6e635558f42a2cbe7aaee Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 6 Jul 2020 00:53:38 -0700 Subject: [PATCH 018/100] vttablet: delete old broken health tests Signed-off-by: Sugu Sougoumarane --- .../tabletmanager/healthcheck_test.go | 827 ------------------ go/vt/vttablet/tabletmanager/state_change.go | 4 +- go/vt/vttablet/tabletserver/state_manager.go | 2 +- 3 files changed, 4 insertions(+), 829 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/healthcheck_test.go b/go/vt/vttablet/tabletmanager/healthcheck_test.go index 954f001f5d7..f8e66cf3ead 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck_test.go +++ b/go/vt/vttablet/tabletmanager/healthcheck_test.go @@ -21,7 +21,6 @@ import ( "fmt" "html/template" "reflect" - "strings" "testing" "time" @@ -31,13 +30,8 @@ import ( "vitess.io/vitess/go/sync2" "vitess.io/vitess/go/vt/binlog" "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/health" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vttablet/tabletserver" "vitess.io/vitess/go/vt/vttablet/tabletservermock" @@ -166,827 +160,6 @@ func createTestTM(ctx context.Context, t *testing.T, preStart func(*TabletManage return tm } -// TestHealthCheckControlsQueryService verifies that a tablet going healthy -// starts the query service, and going unhealthy stops it. -func TestHealthCheckControlsQueryService(t *testing.T) { - // we need an actual grace period set, so lameduck is enabled - *gracePeriod = 10 * time.Millisecond - defer func() { - *gracePeriod = 0 - }() - - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - // Consume the first health broadcast triggered by TabletManager.Start(): - // (REPLICA, NOT_SERVING) goes to (REPLICA, SERVING). And we - // should be serving. - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - - // First health check, should keep us as replica and serving. - before := time.Now() - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 12 * time.Second - tm.runHealthCheck() - ti, err := tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("First health check failed to go to replica: %v", ti.Type) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: %v", tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 12); err != nil { - t.Fatal(err) - } - - // now make the tablet unhealthy - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 13 * time.Second - tm.HealthReporter.(*fakeHealthCheck).reportError = fmt.Errorf("tablet is unhealthy") - before = time.Now() - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("Unhappy health check failed to stay as replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_REPLICA) - } - - // first we get the lameduck broadcast, with no error and old - // replication delay - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "", 12); err != nil { - t.Fatal(err) - } - - // then query service is disabled since we are unhealthy now. - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // and the associated broadcast - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "tablet is unhealthy", 13); err != nil { - t.Fatal(err) - } - - // and nothing more. - if err := expectBroadcastDataEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } - if err := expectStateChangesEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } -} - -// TestErrReplicationNotRunningIsHealthy verifies that a tablet whose -// healthcheck reports health.ErrReplicationNotRunning is still considered -// healthy with high replication lag. -func TestErrReplicationNotRunningIsHealthy(t *testing.T) { - unhealthyThreshold = 10 * time.Minute - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - // Consume the first health broadcast triggered by TabletManager.Start(): - // (REPLICA, NOT_SERVING) goes to (REPLICA, SERVING). And we - // should be serving. - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - - // health check returning health.ErrReplicationNotRunning, should - // keep us as replica and serving - before := time.Now() - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 12 * time.Second - tm.HealthReporter.(*fakeHealthCheck).reportError = health.ErrReplicationNotRunning - tm.runHealthCheck() - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: %v", tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 10*60); err != nil { - t.Fatal(err) - } - - // and nothing more. - if err := expectBroadcastDataEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } - if err := expectStateChangesEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } -} - -// TestQueryServiceNotStarting verifies that if a tablet cannot start the -// query service, it should not go healthy. -func TestQueryServiceNotStarting(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, func(a *TabletManager) { - // The SetServingType that will fail is part of Start() - // so we have to do this here. - a.QueryServiceControl.(*tabletservermock.Controller).SetServingTypeError = fmt.Errorf("test cannot start query service") - }) - - // we should not be serving. - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - - // There is no broadcast data to consume, we're just not - // healthy from startup - - // Now we can run another health check, it will stay unhealthy forever. - before := time.Now() - tm.runHealthCheck() - ti, err := tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("Happy health check which cannot start query service should stay replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - bd := <-tm.QueryServiceControl.(*tabletservermock.Controller).BroadcastData - if bd.RealtimeStats.HealthError != "test cannot start query service" { - t.Errorf("unexpected HealthError: %v", *bd) - } - if tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: %v", tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType) - } - - if err := expectBroadcastDataEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } - if err := expectStateChangesEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } -} - -// TestQueryServiceStopped verifies that if a healthy tablet's query -// service is shut down, the tablet goes unhealthy -func TestQueryServiceStopped(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - // Consume the first health broadcast triggered by TabletManager.Start(): - // (REPLICA, NOT_SERVING) goes to (REPLICA, SERVING). And we - // should be serving. - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - - // first health check, should keep us in replica / healthy - before := time.Now() - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 14 * time.Second - tm.runHealthCheck() - ti, err := tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("First health check failed to stay in replica: %v", ti.Type) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - want := topodatapb.TabletType_REPLICA - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != want { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, want) - } - - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 14); err != nil { - t.Fatal(err) - } - - // shut down query service and prevent it from starting again - // (this is to simulate mysql going away, tablet server detecting it - // and shutting itself down). Intercept the message - tm.QueryServiceControl.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, false, nil) - tm.QueryServiceControl.(*tabletservermock.Controller).SetServingTypeError = fmt.Errorf("test cannot start query service") - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // health check should now fail - before = time.Now() - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 15 * time.Second - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("Happy health check which cannot start query service should stay replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - want = topodatapb.TabletType_REPLICA - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != want { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, want) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "test cannot start query service", 15); err != nil { - t.Fatal(err) - } - // NOTE: No more broadcasts or state changes since SetServingTypeError is set - // on the mocked controller and this disables its SetServingType(). - - if err := expectBroadcastDataEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } - if err := expectStateChangesEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } -} - -// TestTabletControl verifies the shard's TabletControl record can disable -// query service in a tablet. -func TestTabletControl(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - // Consume the first health broadcast triggered by TabletManager.Start(): - // (REPLICA, NOT_SERVING) goes to (REPLICA, SERVING). And we - // should be serving. - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // first health check, should keep us in replica, just broadcast - before := time.Now() - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 16 * time.Second - tm.runHealthCheck() - ti, err := tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("First health check failed to go to replica: %v", ti.Type) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_REPLICA) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 16); err != nil { - t.Fatal(err) - } - - // now update the shard - - si, err := tm.TopoServer.GetShard(ctx, "test_keyspace", "0") - if err != nil { - t.Fatalf("GetShard failed: %v", err) - } - - ctx, unlock, lockErr := tm.TopoServer.LockKeyspace(ctx, "test_keyspace", "UpdateDisableQueryService") - if lockErr != nil { - t.Fatalf("Couldn't lock keyspace for test") - } - defer unlock(&err) - - // Let's generate the keyspace graph we have partition information for this cell - err = topotools.RebuildKeyspaceLocked(ctx, logutil.NewConsoleLogger(), tm.TopoServer, "test_keyspace", []string{tm.tabletAlias.GetCell()}) - if err != nil { - t.Fatalf("RebuildKeyspaceLocked failed: %v", err) - } - - err = tm.TopoServer.UpdateDisableQueryService(ctx, "test_keyspace", []*topo.ShardInfo{si}, topodatapb.TabletType_REPLICA, nil, true) - if err != nil { - t.Fatalf("UpdateShardFields failed: %v", err) - } - - // now refresh the tablet state, as the resharding process would do - tm.RefreshState(ctx) - - // check we shutdown query service - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - - // check UpdateStream is still running - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - - // Consume the health broadcast which was triggered due to the QueryService - // state change from SERVING to NOT_SERVING. - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "", 16); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // check running a health check will not start it again - before = time.Now() - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 17 * time.Second - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("Health check failed to go to replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_REPLICA) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "", 17); err != nil { - t.Fatal(err) - } - // NOTE: No state change here since nothing has changed. - - // go unhealthy, check we go to error state and QS is not running - tm.HealthReporter.(*fakeHealthCheck).reportError = fmt.Errorf("tablet is unhealthy") - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 18 * time.Second - before = time.Now() - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("Unhealthy health check should stay replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "tablet is unhealthy", 18); err != nil { - t.Fatal(err) - } - // NOTE: No state change here since QueryService is already NOT_SERVING. - want := topodatapb.TabletType_REPLICA - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != want { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, want) - } - - // go back healthy, check QS is still not running - tm.HealthReporter.(*fakeHealthCheck).reportError = nil - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 19 * time.Second - before = time.Now() - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_REPLICA { - t.Errorf("Healthy health check should go to replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if !tm.UpdateStream.IsEnabled() { - t.Errorf("UpdateStream should be running") - } - if tm._healthyTime.Sub(before) < 0 { - t.Errorf("runHealthCheck did not update tm._healthyTime") - } - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "", 19); err != nil { - t.Fatal(err) - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_REPLICA) - } - - // now clear TabletControl, run health check, make sure we go - // back healthy and serving. - err = tm.TopoServer.UpdateDisableQueryService(ctx, "test_keyspace", []*topo.ShardInfo{si}, topodatapb.TabletType_REPLICA, nil, false) - if err != nil { - t.Fatalf("UpdateDisableQueryService failed: %v", err) - } - - // now refresh the tablet state, as the resharding process would do - tm.RefreshState(ctx) - - // QueryService changed back from SERVING to NOT_SERVING since RefreshState() - // re-read the topology and saw that REPLICA is still not allowed to serve. - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 19); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - if err := expectBroadcastDataEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } - if err := expectStateChangesEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } -} - -// TestQueryServiceChangeImmediateHealthcheckResponse verifies that a change -// of the QueryService state or the tablet type will result into a broadcast -// of a StreamHealthResponse message. -func TestStateChangeImmediateHealthBroadcast(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - // Consume the first health broadcast triggered by TabletManager.Start(): - // (REPLICA, NOT_SERVING) goes to (REPLICA, SERVING). And we - // should be serving. - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // Run health check to turn into a healthy replica - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 12 * time.Second - tm.runHealthCheck() - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_REPLICA { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_REPLICA) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 12); err != nil { - t.Fatal(err) - } - - // Change to master. - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 19 * time.Second - if err := tm.ChangeType(ctx, topodatapb.TabletType_MASTER); err != nil { - t.Fatalf("TabletExternallyReparented failed: %v", err) - } - // Wait for shard_sync to finish - startTime := time.Now() - for { - if time.Since(startTime) > 10*time.Second /* timeout */ { - si, err := tm.TopoServer.GetShard(ctx, tm.Tablet().Keyspace, tm.Tablet().Shard) - if err != nil { - t.Fatalf("GetShard(%v, %v) failed: %v", tm.Tablet().Keyspace, tm.Tablet().Shard, err) - } - if !topoproto.TabletAliasEqual(si.MasterAlias, tm.Tablet().Alias) { - t.Fatalf("ShardInfo should have MasterAlias %v but has %v", topoproto.TabletAliasString(tm.Tablet().Alias), topoproto.TabletAliasString(si.MasterAlias)) - } - } - si, err := tm.TopoServer.GetShard(ctx, tm.Tablet().Keyspace, tm.Tablet().Shard) - if err != nil { - t.Fatalf("GetShard(%v, %v) failed: %v", tm.Tablet().Keyspace, tm.Tablet().Shard, err) - } - if topoproto.TabletAliasEqual(si.MasterAlias, tm.Tablet().Alias) { - break - } else { - time.Sleep(100 * time.Millisecond /* interval at which to re-check the shard record */) - } - } - - ti, err := tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_MASTER { - t.Errorf("TER failed to go to master: %v", ti.Type) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_MASTER { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_MASTER) - } - - // Consume the health broadcast (no replication delay as we are master) - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 12); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_MASTER); err != nil { - t.Fatal(err) - } - - // Run health check to make sure we stay good - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 20 * time.Second - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_MASTER { - t.Errorf("First health check failed to go to master: %v", ti.Type) - } - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should be running") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_MASTER { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_MASTER) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 20); err != nil { - t.Fatal(err) - } - - // Simulate a vertical split resharding where we set - // SourceShards in the topo and enable filtered replication. - _, err = tm.TopoServer.UpdateShardFields(ctx, "test_keyspace", "0", func(si *topo.ShardInfo) error { - si.SourceShards = []*topodatapb.Shard_SourceShard{ - { - Uid: 1, - Keyspace: "source_keyspace", - Shard: "0", - Tables: []string{ - "table1", - }, - }, - } - return nil - }) - if err != nil { - t.Fatalf("UpdateShardFields failed: %v", err) - } - - // Refresh the tablet state, as vtworker would do. - // Since we change the QueryService state, we'll also trigger a health broadcast. - tm.RefreshState(ctx) - - // (Destination) MASTER with enabled filtered replication mustn't serve anymore. - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - // Consume health broadcast sent out due to QueryService state change from - // (MASTER, SERVING) to (MASTER, NOT_SERVING). - // TODO(sougou); this test case does not reflect reality. Need to fix. - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "", 20); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_MASTER); err != nil { - t.Fatal(err) - } - - // Running a healthcheck won't put the QueryService back to SERVING. - tm.runHealthCheck() - ti, err = tm.TopoServer.GetTablet(ctx, tabletAlias) - if err != nil { - t.Fatalf("GetTablet failed: %v", err) - } - if ti.Type != topodatapb.TabletType_MASTER { - t.Errorf("Health check failed to go to replica: %v", ti.Type) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - if got := tm.QueryServiceControl.(*tabletservermock.Controller).CurrentTarget.TabletType; got != topodatapb.TabletType_MASTER { - t.Errorf("invalid tabletserver target: got = %v, want = %v", got, topodatapb.TabletType_MASTER) - } - if _, err := expectBroadcastData(tm.QueryServiceControl, false, "", 20); err != nil { - t.Fatal(err) - } - // NOTE: No state change here since nothing has changed. - - // Simulate migration to destination master i.e. remove SourceShards. - _, err = tm.TopoServer.UpdateShardFields(ctx, "test_keyspace", "0", func(si *topo.ShardInfo) error { - si.SourceShards = nil - return nil - }) - if err != nil { - t.Fatalf("UpdateShardFields failed: %v", err) - } - - // Refresh the tablet state, as vtctl MigrateServedFrom would do. - // This should also trigger a health broadcast since the QueryService state - // changes from NOT_SERVING to SERVING. - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 23 * time.Second - tm.RefreshState(ctx) - - // QueryService changed from NOT_SERVING to SERVING. - if !tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should not be running") - } - // Since we didn't run healthcheck again yet, the broadcast data contains the - // cached replication lag of 0. This is because - // RefreshState on MASTER always sets the replicationDelay to 0 - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "", 20); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_MASTER); err != nil { - t.Fatal(err) - } - - if err := expectBroadcastDataEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } - if err := expectStateChangesEmpty(tm.QueryServiceControl); err != nil { - t.Fatal(err) - } -} - -// TestOldHealthCheck verifies that a healthcheck that is too old will -// return an error -func TestOldHealthCheck(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - healthCheckInterval = 20 * time.Second - tm._healthy = nil - - // last health check time is now, we're good - tm._healthyTime = time.Now() - if _, healthy := tm.Healthy(); healthy != nil { - t.Errorf("Healthy returned unexpected error: %v", healthy) - } - - // last health check time is 2x interval ago, we're good - tm._healthyTime = time.Now().Add(-2 * healthCheckInterval) - if _, healthy := tm.Healthy(); healthy != nil { - t.Errorf("Healthy returned unexpected error: %v", healthy) - } - - // last health check time is 4x interval ago, we're not good - tm._healthyTime = time.Now().Add(-4 * healthCheckInterval) - if _, healthy := tm.Healthy(); healthy == nil || !strings.Contains(healthy.Error(), "last health check is too old") { - t.Errorf("Healthy returned wrong error: %v", healthy) - } -} - -// TestBackupStateChange verifies that after backup we check -// the replication delay before setting REPLICA tablet to SERVING -func TestBackupStateChange(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - degradedThreshold = 7 * time.Second - unhealthyThreshold = 15 * time.Second - - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 16 * time.Second - - // change to BACKUP, query service will turn off - if err := tm.ChangeType(ctx, topodatapb.TabletType_BACKUP); err != nil { - t.Fatal(err) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should NOT be running") - } - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_BACKUP); err != nil { - t.Fatal(err) - } - // change back to REPLICA, query service should not start - // because replication delay > unhealthyThreshold - if err := tm.ChangeType(ctx, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should NOT be running") - } - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // run healthcheck - // now query service should still be OFF - tm.runHealthCheck() - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should NOT be running") - } -} - -// TestRestoreStateChange verifies that after restore we check -// the replication delay before setting REPLICA tablet to SERVING -func TestRestoreStateChange(t *testing.T) { - ctx := context.Background() - tm := createTestTM(ctx, t, nil) - - degradedThreshold = 7 * time.Second - unhealthyThreshold = 15 * time.Second - - if _, err := expectBroadcastData(tm.QueryServiceControl, true, "healthcheck not run yet", 0); err != nil { - t.Fatal(err) - } - if err := expectStateChange(tm.QueryServiceControl, true, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - tm.HealthReporter.(*fakeHealthCheck).reportReplicationDelay = 16 * time.Second - - // change to RESTORE, query service will turn off - if err := tm.ChangeType(ctx, topodatapb.TabletType_RESTORE); err != nil { - t.Fatal(err) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should NOT be running") - } - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_RESTORE); err != nil { - t.Fatal(err) - } - // change back to REPLICA, query service should not start - // because replication delay > unhealthyThreshold - if err := tm.ChangeType(ctx, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should NOT be running") - } - if err := expectStateChange(tm.QueryServiceControl, false, topodatapb.TabletType_REPLICA); err != nil { - t.Fatal(err) - } - - // run healthcheck - // now query service should still be OFF - tm.runHealthCheck() - if tm.QueryServiceControl.IsServing() { - t.Errorf("Query service should NOT be running") - } -} - // expectBroadcastData checks that runHealthCheck() broadcasted the expected // stats (going the value for secondsBehindMaster). func expectBroadcastData(qsc tabletserver.Controller, serving bool, healthError string, secondsBehindMaster uint32) (*tabletservermock.BroadcastData, error) { diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 7b69bece6e1..5c67e1d2415 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -30,6 +30,7 @@ import ( "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -173,7 +174,8 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable defer span.Finish() allowQuery := topo.IsRunningQueryService(newTablet.Type) - terTime := tm.masterTermStartTime() + // TODO(sougou): find a better way to compute this. + terTime := logutil.ProtoToTime(newTablet.MasterTermStartTime) // Read the shard to get SourceShards / TabletControlMap if // we're going to use it. diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 1088f52a518..c336370cb02 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -175,7 +175,7 @@ func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTime state = StateNotConnected } - log.Infof("Starting transition to %v %v", tabletType, stateName(state)) + log.Infof("Starting transition to %v %v, timestamp: %v", tabletType, stateName(state), terTimestamp) if sm.mustTransition(tabletType, terTimestamp, state, alsoAllow) { return true, sm.execTransition(tabletType, state) } From 290ea361d76fe2e1ef2d99f561d6fa54bdc4ddbc Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 6 Jul 2020 01:24:02 -0700 Subject: [PATCH 019/100] vttablet: make BACKUP StateNotConnected Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletserver/state_manager.go | 2 +- go/vt/vttablet/tabletserver/state_manager_test.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index c336370cb02..c9f5601f63d 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -170,7 +170,7 @@ func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTime // Start is idempotent. sm.hcticks.Start(sm.Broadcast) - if tabletType == topodatapb.TabletType_RESTORE { + if tabletType == topodatapb.TabletType_RESTORE || tabletType == topodatapb.TabletType_BACKUP { // TODO(sougou): remove this code once tm can give us more accurate state requests. state = StateNotConnected } diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index 3a9d4326877..8878cb648a4 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -295,7 +295,7 @@ func TestStateManagerTransitionFailRetry(t *testing.T) { assert.Equal(t, StateServing, sm.State()) } -func TestStateManagerRestoreType(t *testing.T) { +func TestStateManagerNotConnectedType(t *testing.T) { sm := newTestStateManager(t) sm.EnterLameduck() stateChanged, err := sm.SetServingType(topodatapb.TabletType_RESTORE, testNow, StateNotServing, nil) @@ -303,7 +303,13 @@ func TestStateManagerRestoreType(t *testing.T) { assert.True(t, stateChanged) assert.Equal(t, topodatapb.TabletType_RESTORE, sm.target.TabletType) - // RESTORE can only be in StateNotConnected. + assert.Equal(t, StateNotConnected, sm.state) + + stateChanged, err = sm.SetServingType(topodatapb.TabletType_BACKUP, testNow, StateNotServing, nil) + require.NoError(t, err) + assert.True(t, stateChanged) + + assert.Equal(t, topodatapb.TabletType_BACKUP, sm.target.TabletType) assert.Equal(t, StateNotConnected, sm.state) } From 2f22ad38363bc59a679eb509e106d51141c6202c Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 6 Jul 2020 14:54:10 -0700 Subject: [PATCH 020/100] vttablet: fix tabletserver repl health & tests Signed-off-by: Sugu Sougoumarane --- .../tabletmanager/tablet_health_test.go | 5 ++- go/vt/vttablet/tabletserver/state_manager.go | 31 +++++++++++++------ .../tabletserver/state_manager_test.go | 2 +- go/vt/vttablet/tabletserver/tabletserver.go | 1 - 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go index e4bb846c8a5..bb37e8c1d82 100644 --- a/go/test/endtoend/tabletmanager/tablet_health_test.go +++ b/go/test/endtoend/tabletmanager/tablet_health_test.go @@ -407,11 +407,10 @@ func TestNoMysqlHealthCheck(t *testing.T) { require.NoError(t, err) checkHealth(t, mTablet.HTTPPort, false) - // the replica will now be healthy, but report a very high replication - // lag, because it can't figure out what it exactly is. + // the replica will not be healthy because it cannot compute replication lag. err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) require.NoError(t, err) - assert.Equal(t, "SERVING", rTablet.VttabletProcess.GetTabletStatus()) + assert.Equal(t, "NOT_SERVING", rTablet.VttabletProcess.GetTabletStatus()) checkHealth(t, rTablet.HTTPPort, false) result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", rTablet.Alias) diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index c9f5601f63d..9abf42c49f7 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -71,15 +71,15 @@ type stateManager struct { wantState servingState wantTabletType topodatapb.TabletType state servingState + replHealthy bool + lameduck bool target querypb.Target retrying bool // TODO(sougou): deprecate alsoAllow alsoAllow []topodatapb.TabletType terTimestamp time.Time - replHealthy bool requests sync.WaitGroup - lameduck sync2.AtomicBool // Open must be done in forward order. // Close must be done in reverse order. @@ -505,6 +505,11 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving } log.Infof("TabletServer transition: %v -> %v, %s -> %s", sm.target.TabletType, tabletType, stateName(sm.state), stateName(state)) sm.target.TabletType = tabletType + if sm.state == StateNotConnected { + // If we're transitioning out of StateNotConnected, we have + // to also ensure replication status is healthy. + _, _ = sm.refreshReplHealthLocked() + } sm.state = state // Broadcast also obtains a lock. Trigger in a goroutine to avoid a deadlock. go sm.hcticks.Trigger() @@ -513,19 +518,21 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving // Broadcast fetches the replication status and broadcasts // the state to all subscribed. func (sm *stateManager) Broadcast() { - lag, err := sm.rt.Status() - sm.mu.Lock() defer sm.mu.Unlock() + lag, err := sm.refreshReplHealthLocked() + sm.notify(sm.target.TabletType, sm.terTimestamp, lag, err, sm.isServingLocked()) +} + +func (sm *stateManager) refreshReplHealthLocked() (time.Duration, error) { + lag, err := sm.rt.Status() if err != nil { sm.replHealthy = false } else { sm.replHealthy = lag <= sm.unhealthyThreshold } - isServing := sm.isServingLocked() - - sm.notify(sm.target.TabletType, sm.terTimestamp, lag, err, isServing) + return lag, err } // EnterLameduck causes tabletserver to enter the lameduck state. This @@ -533,12 +540,16 @@ func (sm *stateManager) Broadcast() { // otherwise remains the same. Any subsequent calls to SetServingType will // cause the tabletserver to exit this mode. func (sm *stateManager) EnterLameduck() { - sm.lameduck.Set(true) + sm.mu.Lock() + defer sm.mu.Unlock() + sm.lameduck = true } // ExitLameduck causes the tabletserver to exit the lameduck mode. func (sm *stateManager) ExitLameduck() { - sm.lameduck.Set(false) + sm.mu.Lock() + defer sm.mu.Unlock() + sm.lameduck = false } // IsServing returns true if TabletServer is in SERVING state. @@ -549,7 +560,7 @@ func (sm *stateManager) IsServing() bool { } func (sm *stateManager) isServingLocked() bool { - return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.lameduck.Get() + return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.lameduck } func (sm *stateManager) State() servingState { diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index 8878cb648a4..d290b8d9f32 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -65,7 +65,7 @@ func TestStateManagerServeMaster(t *testing.T) { require.NoError(t, err) assert.True(t, stateChanged) - assert.Equal(t, false, sm.lameduck.Get()) + assert.Equal(t, false, sm.lameduck) assert.Equal(t, testNow, sm.terTimestamp) verifySubcomponent(t, 1, sm.watcher, testStateClosed) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index e03d294420b..b2229183486 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -312,7 +312,6 @@ func (tsv *TabletServer) StartService(target querypb.Target, dbcfgs *dbconfigs.D return err } // StartService is only used for testing. So, we cheat by aggressively setting replication to healthy. - tsv.sm.replHealthy = true _, err := tsv.sm.SetServingType(target.TabletType, time.Time{}, StateServing, nil) return err } From ea790b5cb79cc8a89bdf75cc27ccaab7ad8d3c6e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Mon, 6 Jul 2020 20:41:28 -0700 Subject: [PATCH 021/100] vttablet: relpManager initial code Signed-off-by: Sugu Sougoumarane --- .../tabletmanager/replication_reporter.go | 53 +------------------ .../vttablet/tabletmanager/rpc_replication.go | 52 ++++++++++++++++-- go/vt/vttablet/tabletmanager/state_change.go | 2 + go/vt/vttablet/tabletmanager/tm_init.go | 10 ++-- ...plication_watcher.go => binlog_watcher.go} | 44 +++++++-------- go/vt/vttablet/tabletserver/tabletserver.go | 4 +- 6 files changed, 78 insertions(+), 87 deletions(-) rename go/vt/vttablet/tabletserver/{replication_watcher.go => binlog_watcher.go} (68%) diff --git a/go/vt/vttablet/tabletmanager/replication_reporter.go b/go/vt/vttablet/tabletmanager/replication_reporter.go index 975a2254a30..08218109e68 100644 --- a/go/vt/vttablet/tabletmanager/replication_reporter.go +++ b/go/vt/vttablet/tabletmanager/replication_reporter.go @@ -17,19 +17,15 @@ limitations under the License. package tabletmanager import ( - "fmt" "html/template" "time" - "vitess.io/vitess/go/vt/vterrors" - "golang.org/x/net/context" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl" - "vitess.io/vitess/go/vt/topo/topoproto" ) var enableReplicationReporter bool @@ -66,7 +62,7 @@ func (r *replicationReporter) Report(isReplicaType, shouldQueryServiceBeRunning } else { log.Infof("replication is stopped. Trying to reconnect to master...") ctx, cancel := context.WithTimeout(r.tm.BatchCtx, 5*time.Second) - if err := repairReplication(ctx, r.tm); err != nil { + if err := r.tm.repairReplication(ctx); err != nil { log.Infof("Failed to reconnect to master: %v", err) } cancel() @@ -106,53 +102,6 @@ func (r *replicationReporter) HTMLName() template.HTML { return template.HTML("MySQLReplicationLag") } -// repairReplication tries to connect this server to whoever is -// the current master of the shard, and start replicating. -func repairReplication(ctx context.Context, tm *TabletManager) error { - if *mysqlctl.DisableActiveReparents { - return fmt.Errorf("can't repair replication with --disable_active_reparents") - } - - ts := tm.TopoServer - tablet := tm.Tablet() - - si, err := ts.GetShard(ctx, tablet.Keyspace, tablet.Shard) - if err != nil { - return err - } - if !si.HasMaster() { - return fmt.Errorf("no master tablet for shard %v/%v", tablet.Keyspace, tablet.Shard) - } - - if topoproto.TabletAliasEqual(si.MasterAlias, tablet.Alias) { - // The shard record says we are master, but we disagree; we wouldn't - // reach this point unless we were told to check replication. - // Hopefully someone is working on fixing that, but in any case, - // we should not try to reparent to ourselves. - return fmt.Errorf("shard %v/%v record claims tablet %v is master, but its type is %v", tablet.Keyspace, tablet.Shard, topoproto.TabletAliasString(tablet.Alias), tablet.Type) - } - - // If Orchestrator is configured and if Orchestrator is actively reparenting, we should not repairReplication - if tm.orc != nil { - re, err := tm.orc.InActiveShardRecovery(tablet) - if err != nil { - return err - } - if re { - return fmt.Errorf("orchestrator actively reparenting shard %v, skipping repairReplication", si) - } - - // Before repairing replication, tell Orchestrator to enter maintenance mode for this tablet and to - // lock any other actions on this tablet by Orchestrator. - if err := tm.orc.BeginMaintenance(tm.Tablet(), "vttablet has been told to StopReplication"); err != nil { - log.Warningf("Orchestrator BeginMaintenance failed: %v", err) - return vterrors.Wrap(err, "orchestrator BeginMaintenance failed, skipping repairReplication") - } - } - - return tm.setMasterRepairReplication(ctx, si.MasterAlias, 0, "", true) -} - func registerReplicationReporter(tm *TabletManager) { if enableReplicationReporter { health.DefaultAggregator.Register("replication_reporter", diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index cb2781865f5..d64ec8077f9 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -90,7 +90,7 @@ func (tm *TabletManager) stopReplicationLocked(ctx context.Context) error { // Remember that we were told to stop, so we don't try to // restart ourselves (in replication_reporter). - tm.setReplicationStopped(true) + tm.replManager.setReplicationStopped(true) // Also tell Orchestrator we're stopped on purpose for some Vitess task. // Do this in the background, as it's best-effort. @@ -162,7 +162,7 @@ func (tm *TabletManager) StartReplication(ctx context.Context) error { } defer tm.unlock() - tm.setReplicationStopped(false) + tm.replManager.setReplicationStopped(false) // Tell Orchestrator we're no longer stopped on purpose. // Do this in the background, as it's best-effort. @@ -213,7 +213,7 @@ func (tm *TabletManager) ResetReplication(ctx context.Context) error { } defer tm.unlock() - tm.setReplicationStopped(true) + tm.replManager.setReplicationStopped(true) return tm.MysqlDaemon.ResetReplication(ctx) } @@ -225,7 +225,7 @@ func (tm *TabletManager) InitMaster(ctx context.Context) (string, error) { defer tm.unlock() // Initializing as master implies undoing any previous "do not replicate". - tm.setReplicationStopped(false) + tm.replManager.setReplicationStopped(false) // we need to insert something in the binlogs, so we can get the // current position. Let's just use the mysqlctl.CreateReparentJournal commands. @@ -296,7 +296,7 @@ func (tm *TabletManager) InitReplica(ctx context.Context, parent *topodatapb.Tab return err } - tm.setReplicationStopped(false) + tm.replManager.setReplicationStopped(false) // If using semi-sync, we need to enable it before connecting to master. // If we were a master type, we need to switch back to replica settings. @@ -841,6 +841,48 @@ func (tm *TabletManager) handleRelayLogError(err error) error { return err } +// repairReplication tries to connect this server to whoever is +// the current master of the shard, and start replicating. +func (tm *TabletManager) repairReplication(ctx context.Context) error { + tablet := tm.Tablet() + + si, err := tm.TopoServer.GetShard(ctx, tablet.Keyspace, tablet.Shard) + if err != nil { + return err + } + if !si.HasMaster() { + return fmt.Errorf("no master tablet for shard %v/%v", tablet.Keyspace, tablet.Shard) + } + + if topoproto.TabletAliasEqual(si.MasterAlias, tablet.Alias) { + // The shard record says we are master, but we disagree; we wouldn't + // reach this point unless we were told to check replication. + // Hopefully someone is working on fixing that, but in any case, + // we should not try to reparent to ourselves. + return fmt.Errorf("shard %v/%v record claims tablet %v is master, but its type is %v", tablet.Keyspace, tablet.Shard, topoproto.TabletAliasString(tablet.Alias), tablet.Type) + } + + // If Orchestrator is configured and if Orchestrator is actively reparenting, we should not repairReplication + if tm.orc != nil { + re, err := tm.orc.InActiveShardRecovery(tablet) + if err != nil { + return err + } + if re { + return fmt.Errorf("orchestrator actively reparenting shard %v, skipping repairReplication", si) + } + + // Before repairing replication, tell Orchestrator to enter maintenance mode for this tablet and to + // lock any other actions on this tablet by Orchestrator. + if err := tm.orc.BeginMaintenance(tm.Tablet(), "vttablet has been told to StopReplication"); err != nil { + log.Warningf("Orchestrator BeginMaintenance failed: %v", err) + return vterrors.Wrap(err, "orchestrator BeginMaintenance failed, skipping repairReplication") + } + } + + return tm.setMasterRepairReplication(ctx, si.MasterAlias, 0, "", true) +} + // SlaveStatus is deprecated func (tm *TabletManager) SlaveStatus(ctx context.Context) (*replicationdatapb.Status, error) { return tm.ReplicationStatus(ctx) diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 5c67e1d2415..51b8f7eee86 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -243,6 +243,8 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable } } + tm.replManager.SetTabletType(newTablet.Type) + if allowQuery { // Query service should be running. if oldTablet.Type == topodatapb.TabletType_REPLICA && diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 859252a8b84..c66112e49b0 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -73,12 +73,6 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -const ( - // replicationStoppedFile is the name of the file whose existence informs - // vttablet to NOT try to repair replication. - replicationStoppedFile = "do_not_replicate" -) - var ( // The following flags initialize the tablet record. tabletHostname = flag.String("tablet_hostname", "", "if not empty, this hostname will be assumed instead of trying to resolve it") @@ -137,6 +131,9 @@ type TabletManager struct { UpdateStream binlog.UpdateStreamControl VREngine *vreplication.Engine + // replManager manages replication. + replManager *replManager + // HealthReporter initiates healthchecks. HealthReporter health.Reporter @@ -293,6 +290,7 @@ func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32) ( func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { tm.setTablet(tablet) tm.DBConfigs.DBName = topoproto.TabletDbName(tablet) + tm.replManager = newReplManager(tm.BatchCtx, tm, healthCheckInterval) tm.History = history.New(historyLength) tm.tabletAlias = tablet.Alias demoteType, err := topoproto.ParseTabletType(*demoteMasterType) diff --git a/go/vt/vttablet/tabletserver/replication_watcher.go b/go/vt/vttablet/tabletserver/binlog_watcher.go similarity index 68% rename from go/vt/vttablet/tabletserver/replication_watcher.go rename to go/vt/vttablet/tabletserver/binlog_watcher.go index 06b7f829550..59b8e0c90c4 100644 --- a/go/vt/vttablet/tabletserver/replication_watcher.go +++ b/go/vt/vttablet/tabletserver/binlog_watcher.go @@ -29,15 +29,15 @@ import ( ) // VStreamer defines the functions of VStreamer -// that the replicationWatcher needs. +// that the BinlogWatcher needs. type VStreamer interface { Stream(ctx context.Context, startPos string, tablePKs []*binlogdatapb.TableLastPK, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error } -// ReplicationWatcher is a tabletserver service that watches the +// BinlogWatcher is a tabletserver service that watches the // replication stream. It will trigger schema reloads if a DDL // is encountered. -type ReplicationWatcher struct { +type BinlogWatcher struct { env tabletenv.Env watchReplication bool vs VStreamer @@ -46,40 +46,40 @@ type ReplicationWatcher struct { wg sync.WaitGroup } -// NewReplicationWatcher creates a new ReplicationWatcher. -func NewReplicationWatcher(env tabletenv.Env, vs VStreamer, config *tabletenv.TabletConfig) *ReplicationWatcher { - return &ReplicationWatcher{ +// NewBinlogWatcher creates a new BinlogWatcher. +func NewBinlogWatcher(env tabletenv.Env, vs VStreamer, config *tabletenv.TabletConfig) *BinlogWatcher { + return &BinlogWatcher{ env: env, vs: vs, watchReplication: config.WatchReplication, } } -// Open starts the ReplicationWatcher service. -func (rpw *ReplicationWatcher) Open() { - if rpw.cancel != nil || !rpw.watchReplication { +// Open starts the BinlogWatcher service. +func (blw *BinlogWatcher) Open() { + if blw.cancel != nil || !blw.watchReplication { return } ctx, cancel := context.WithCancel(tabletenv.LocalContext()) - rpw.cancel = cancel - rpw.wg.Add(1) - go rpw.process(ctx) + blw.cancel = cancel + blw.wg.Add(1) + go blw.process(ctx) } -// Close stops the ReplicationWatcher service. -func (rpw *ReplicationWatcher) Close() { - if rpw.cancel == nil { +// Close stops the BinlogWatcher service. +func (blw *BinlogWatcher) Close() { + if blw.cancel == nil { return } - rpw.cancel() - rpw.cancel = nil - rpw.wg.Wait() + blw.cancel() + blw.cancel = nil + blw.wg.Wait() } -func (rpw *ReplicationWatcher) process(ctx context.Context) { - defer rpw.env.LogError() - defer rpw.wg.Done() +func (blw *BinlogWatcher) process(ctx context.Context) { + defer blw.env.LogError() + defer blw.wg.Done() filter := &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{ @@ -89,7 +89,7 @@ func (rpw *ReplicationWatcher) process(ctx context.Context) { for { // VStreamer will reload the schema when it encounters a DDL. - err := rpw.vs.Stream(ctx, "current", nil, filter, func(events []*binlogdatapb.VEvent) error { + err := blw.vs.Stream(ctx, "current", nil, filter, func(events []*binlogdatapb.VEvent) error { return nil }) select { diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index b2229183486..828dc560c02 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -96,7 +96,7 @@ type TabletServer struct { rt *repltracker.ReplTracker vstreamer *vstreamer.Engine tracker *schema.Tracker - watcher *ReplicationWatcher + watcher *BinlogWatcher qe *QueryEngine txThrottler *txthrottler.TxThrottler te *TxEngine @@ -148,7 +148,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsv.rt = repltracker.NewReplTracker(tsv, alias) tsv.vstreamer = vstreamer.NewEngine(tsv, srvTopoServer, tsv.se, alias.Cell) tsv.tracker = schema.NewTracker(tsv, tsv.vstreamer, tsv.se) - tsv.watcher = NewReplicationWatcher(tsv, tsv.vstreamer, tsv.config) + tsv.watcher = NewBinlogWatcher(tsv, tsv.vstreamer, tsv.config) tsv.qe = NewQueryEngine(tsv, tsv.se) tsv.txThrottler = txthrottler.NewTxThrottler(tsv.config, topoServer) tsv.te = NewTxEngine(tsv) From 109aab6de0eb318ef51a791f2780ce552ce5de02 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 7 Jul 2020 10:43:27 -0700 Subject: [PATCH 022/100] vttablet: fix tabletmanager tests Signed-off-by: Sugu Sougoumarane --- go/cmd/vttablet/healthz.go | 42 ---- .../tabletmanager/tablet_health_test.go | 210 +----------------- go/vt/vttablet/tabletmanager/replmanager.go | 151 +++++++++++++ go/vt/vttablet/tabletmanager/tm_init.go | 18 -- .../vttablet/tabletserver/health_streamer.go | 12 + go/vt/vttablet/tabletserver/tabletserver.go | 19 ++ 6 files changed, 186 insertions(+), 266 deletions(-) delete mode 100644 go/cmd/vttablet/healthz.go create mode 100644 go/vt/vttablet/tabletmanager/replmanager.go diff --git a/go/cmd/vttablet/healthz.go b/go/cmd/vttablet/healthz.go deleted file mode 100644 index 22d2985ef1a..00000000000 --- a/go/cmd/vttablet/healthz.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package main - -import ( - "fmt" - "net/http" - - "vitess.io/vitess/go/vt/servenv" -) - -// This file registers a /healthz URL that reports the health of the tm. - -var okMessage = []byte("ok\n") - -func init() { - servenv.OnRun(func() { - http.Handle("/healthz", http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - if _, err := tm.Healthy(); err != nil { - http.Error(rw, fmt.Sprintf("500 internal server error: tablet manager not healthy: %v", err), http.StatusInternalServerError) - return - } - - rw.Header().Set("Content-Length", fmt.Sprintf("%v", len(okMessage))) - rw.WriteHeader(http.StatusOK) - rw.Write(okMessage) - })) - }) -} diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go index bb37e8c1d82..2c2391e37ec 100644 --- a/go/test/endtoend/tabletmanager/tablet_health_test.go +++ b/go/test/endtoend/tabletmanager/tablet_health_test.go @@ -23,7 +23,6 @@ import ( "net/http" "strings" "testing" - "time" "github.com/stretchr/testify/require" @@ -62,10 +61,14 @@ func TestTabletReshuffle(t *testing.T) { // mycnf_server_id prevents vttablet from reading the mycnf // Pointing to masterTablet's socket file + // We have to disable active reparenting to prevent the tablet from trying to fix replication. + // We also have to disable replication reporting because we're pointed at the master. clusterInstance.VtTabletExtraArgs = []string{ "-lock_tables_timeout", "5s", "-mycnf_server_id", fmt.Sprintf("%d", rTablet.TabletUID), "-db_socket", fmt.Sprintf("%s/mysql.sock", masterTablet.VttabletProcess.Directory), + "-disable_active_reparents", + "-enable_replication_reporter=false", } defer func() { clusterInstance.VtTabletExtraArgs = []string{} }() @@ -249,211 +252,6 @@ func TestHealthCheckDrainedStateDoesNotShutdownQueryService(t *testing.T) { checkHealth(t, rdonlyTablet.HTTPPort, false) } -func TestIgnoreHealthError(t *testing.T) { - // This test verify the tablet health by Ignoring the error - // For this case we need a healthy tablet in a shard without any master. - // When we try to make a connection to such tablet we get "no slave status" error. - // We will then ignore this error and verify if the status report the tablet as Healthy. - - // Create a new shard - defer cluster.PanicHandler(t) - newShard := &cluster.Shard{ - Name: "1", - } - - // Start mysql process - tablet := clusterInstance.NewVttabletInstance("replica", 0, "") - tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, clusterInstance.TmpDirectory) - err := tablet.MysqlctlProcess.Start() - require.NoError(t, err) - - // start vttablet process - tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, - tablet.GrpcPort, - tablet.TabletUID, - clusterInstance.Cell, - newShard.Name, - clusterInstance.Keyspaces[0].Name, - clusterInstance.VtctldProcess.Port, - tablet.Type, - clusterInstance.TopoProcess.Port, - clusterInstance.Hostname, - clusterInstance.TmpDirectory, - clusterInstance.VtTabletExtraArgs, - clusterInstance.EnableSemiSync) - tablet.Alias = tablet.VttabletProcess.TabletPath - newShard.Vttablets = append(newShard.Vttablets, tablet) - - clusterInstance.Keyspaces[0].Shards = append(clusterInstance.Keyspaces[0].Shards, *newShard) - - // Init Tablet - err = clusterInstance.VtctlclientProcess.InitTablet(tablet, cell, keyspaceName, hostname, newShard.Name) - require.NoError(t, err) - - // create database - err = tablet.VttabletProcess.CreateDB(keyspaceName) - require.NoError(t, err) - - // Start Vttablet, it should be NOT_SERVING as there is no master - err = clusterInstance.StartVttablet(tablet, "NOT_SERVING", false, cell, keyspaceName, hostname, newShard.Name) - require.NoError(t, err) - - // Force it healthy. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("IgnoreHealthError", tablet.Alias, ".*no slave status.*") - require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", tablet.Alias) - require.NoError(t, err) - err = tablet.VttabletProcess.WaitForTabletType("SERVING") - require.NoError(t, err) - checkHealth(t, tablet.HTTPPort, false) - - // Turn off the force-healthy. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("IgnoreHealthError", tablet.Alias, "") - require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", tablet.Alias) - require.NoError(t, err) - err = tablet.VttabletProcess.WaitForTabletType("NOT_SERVING") - require.NoError(t, err) - checkHealth(t, tablet.HTTPPort, true) - - // Tear down custom processes - killTablets(t, tablet) -} - -func TestNoMysqlHealthCheck(t *testing.T) { - // This test starts a vttablet with no mysql port, while mysql is down. - // It makes sure vttablet will start properly and be unhealthy. - // Then we start mysql, and make sure vttablet becomes healthy. - defer cluster.PanicHandler(t) - ctx := context.Background() - - clusterInstance.VtTabletExtraArgs = []string{ - "-publish_retry_interval", "1s", - } - defer func() { clusterInstance.VtTabletExtraArgs = []string{} }() - - rTablet := clusterInstance.NewVttabletInstance("replica", 0, "") - mTablet := clusterInstance.NewVttabletInstance("replica", 0, "") - - // Start Mysql Processes and return connection - masterConn, err := cluster.StartMySQLAndGetConnection(ctx, mTablet, username, clusterInstance.TmpDirectory) - require.NoError(t, err) - defer masterConn.Close() - - replicaConn, err := cluster.StartMySQLAndGetConnection(ctx, rTablet, username, clusterInstance.TmpDirectory) - require.NoError(t, err) - defer replicaConn.Close() - - // Create database in mysql - exec(t, masterConn, fmt.Sprintf("create database vt_%s", keyspaceName)) - exec(t, replicaConn, fmt.Sprintf("create database vt_%s", keyspaceName)) - - //Get the gtid to ensure we bring master and replica at same position - qr := exec(t, masterConn, "SELECT @@GLOBAL.gtid_executed") - gtid := string(qr.Rows[0][0].Raw()) - - // Ensure master ans salve are at same position - exec(t, replicaConn, "STOP SLAVE") - exec(t, replicaConn, "RESET MASTER") - exec(t, replicaConn, "RESET SLAVE") - exec(t, replicaConn, fmt.Sprintf("SET GLOBAL gtid_purged='%s'", gtid)) - exec(t, replicaConn, fmt.Sprintf("CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_USER='vt_repl', MASTER_AUTO_POSITION = 1", hostname, mTablet.MySQLPort)) - exec(t, replicaConn, "START SLAVE") - - // now shutdown all mysqld - err = rTablet.MysqlctlProcess.Stop() - require.NoError(t, err) - err = mTablet.MysqlctlProcess.Stop() - require.NoError(t, err) - - //Init Tablets - err = clusterInstance.VtctlclientProcess.InitTablet(mTablet, cell, keyspaceName, hostname, shardName) - require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.InitTablet(rTablet, cell, keyspaceName, hostname, shardName) - require.NoError(t, err) - - // Start vttablet process, should be in NOT_SERVING state as mysqld is not running - err = clusterInstance.StartVttablet(mTablet, "NOT_SERVING", false, cell, keyspaceName, hostname, shardName) - require.NoError(t, err) - err = clusterInstance.StartVttablet(rTablet, "NOT_SERVING", false, cell, keyspaceName, hostname, shardName) - require.NoError(t, err) - - // Check Health should fail as Mysqld is not found - checkHealth(t, mTablet.HTTPPort, true) - checkHealth(t, rTablet.HTTPPort, true) - - // Tell replica to not try to repair replication in healthcheck. - // The StopReplication will ultimately fail because mysqld is not running, - // But vttablet should remember that it's not supposed to fix replication. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rTablet.Alias) - assert.Error(t, err, "Fail as mysqld not running") - - //The above notice to not fix replication should survive tablet restart. - err = rTablet.VttabletProcess.TearDown() - require.NoError(t, err) - err = rTablet.VttabletProcess.Setup() - require.NoError(t, err) - - // restart mysqld - rTablet.MysqlctlProcess.InitMysql = false - err = rTablet.MysqlctlProcess.Start() - require.NoError(t, err) - mTablet.MysqlctlProcess.InitMysql = false - err = mTablet.MysqlctlProcess.Start() - require.NoError(t, err) - - // the master should still be healthy - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", mTablet.Alias) - require.NoError(t, err) - checkHealth(t, mTablet.HTTPPort, false) - - // the replica will not be healthy because it cannot compute replication lag. - err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) - require.NoError(t, err) - assert.Equal(t, "NOT_SERVING", rTablet.VttabletProcess.GetTabletStatus()) - checkHealth(t, rTablet.HTTPPort, false) - - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", rTablet.Alias) - require.NoError(t, err) - var streamHealthResponse querypb.StreamHealthResponse - err = json2.Unmarshal([]byte(result), &streamHealthResponse) - require.NoError(t, err) - realTimeStats := streamHealthResponse.GetRealtimeStats() - assert.Equal(t, "replication is not running", realTimeStats.HealthError) - - // restart replication, wait until health check goes small - // (a value of zero is default and won't be in structure) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartSlave", rTablet.Alias) - require.NoError(t, err) - - timeout := time.Now().Add(10 * time.Second) - for time.Now().Before(timeout) { - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", rTablet.Alias) - require.NoError(t, err) - var streamHealthResponse querypb.StreamHealthResponse - err = json2.Unmarshal([]byte(result), &streamHealthResponse) - require.NoError(t, err) - realTimeStats := streamHealthResponse.GetRealtimeStats() - secondsBehindMaster := realTimeStats.GetSecondsBehindMaster() - if secondsBehindMaster < 30 { - break - } else { - time.Sleep(100 * time.Millisecond) - } - } - - // wait for the tablet to fix its mysql port - result, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", rTablet.Alias) - require.NoError(t, err) - var tablet topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tablet) - require.NoError(t, err) - assert.Equal(t, int32(rTablet.MySQLPort), tablet.MysqlPort, "mysql port in tablet record") - - // Tear down custom processes - killTablets(t, rTablet, mTablet) -} - func killTablets(t *testing.T, tablets ...*cluster.Vttablet) { for _, tablet := range tablets { //Stop Mysqld diff --git a/go/vt/vttablet/tabletmanager/replmanager.go b/go/vt/vttablet/tabletmanager/replmanager.go new file mode 100644 index 00000000000..080cfcb0b36 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/replmanager.go @@ -0,0 +1,151 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletmanager + +import ( + "os" + "path" + "sync" + "time" + + "golang.org/x/net/context" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/timer" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" +) + +const ( + // replicationStoppedFile is the name of the file whose existence informs + // vttablet to NOT try to repair replication. + replicationStoppedFile = "do_not_replicate" +) + +// replManager manages replication. +type replManager struct { + ctx context.Context + tm *TabletManager + markerFile string + ticks *timer.Timer + + // replStopped is tri-state. + // A nil value signifies that the value is not set. + mu sync.Mutex + replStopped *bool +} + +func newReplManager(ctx context.Context, tm *TabletManager, interval time.Duration) *replManager { + return &replManager{ + ctx: ctx, + tm: tm, + markerFile: markerFile(tm.Cnf), + ticks: timer.NewTimer(interval), + } +} + +func (rm *replManager) SetTabletType(tabletType topodatapb.TabletType) { + if *mysqlctl.DisableActiveReparents { + return + } + if !topo.IsReplicaType(tabletType) { + rm.ticks.Stop() + return + } + if rm.replicationStopped() { + // Stop just to be safe. + rm.ticks.Stop() + return + } + // Do an explicit check and then start the timer. + rm.check() + rm.ticks.Start(rm.check) +} + +func (rm *replManager) check() { + status, err := rm.tm.MysqlDaemon.ReplicationStatus() + if err != nil { + if err != mysql.ErrNotReplica { + return + } + } else { + // TODO(sougou): this was ported from previous behavior. + // Need to check if this is should be &&. + if status.SQLThreadRunning || status.IOThreadRunning { + return + } + } + + log.Infof("Replication is stopped. Trying to reconnect to master...") + ctx, cancel := context.WithTimeout(rm.ctx, 5*time.Second) + defer cancel() + if err := rm.tm.repairReplication(ctx); err != nil { + log.Infof("Failed to reconnect to master: %v", err) + } +} + +// setReplicationStopped performs a best effort attempt of +// remembering a decision to stop replication. +func (rm *replManager) setReplicationStopped(stopped bool) { + if *mysqlctl.DisableActiveReparents { + return + } + + rm.mu.Lock() + defer rm.mu.Unlock() + + rm.replStopped = &stopped + + if rm.markerFile == "" { + return + } + if stopped { + rm.ticks.Stop() + if file, err := os.Create(rm.markerFile); err == nil { + file.Close() + } + } else { + rm.ticks.Start(rm.check) + os.Remove(rm.markerFile) + } +} + +func (rm *replManager) replicationStopped() bool { + rm.mu.Lock() + defer rm.mu.Unlock() + + if rm.replStopped != nil { + // Return cached value. + return *rm.replStopped + } + if rm.markerFile == "" { + return false + } + + _, err := os.Stat(rm.markerFile) + replicationStopped := err == nil + rm.replStopped = &replicationStopped + return replicationStopped +} + +func markerFile(cnf *mysqlctl.Mycnf) string { + if cnf == nil { + return "" + } + return path.Join(cnf.TabletDir(), replicationStoppedFile) +} diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index c66112e49b0..fd75561168b 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -754,24 +754,6 @@ func (tm *TabletManager) Tablet() *topodatapb.Tablet { return tablet } -// Healthy reads the result of the latest healthcheck, protected by mutex. -// If that status is too old, it means healthcheck hasn't run for a while, -// and is probably stuck, this is not good, we're not healthy. -func (tm *TabletManager) Healthy() (time.Duration, error) { - tm.mutex.Lock() - defer tm.mutex.Unlock() - - healthy := tm._healthy - if healthy == nil { - timeSinceLastCheck := time.Since(tm._healthyTime) - if timeSinceLastCheck > healthCheckInterval*3 { - healthy = fmt.Errorf("last health check is too old: %s > %s", timeSinceLastCheck, healthCheckInterval*3) - } - } - - return tm._replicationDelay, healthy -} - // BlacklistedTables returns the list of currently blacklisted tables. func (tm *TabletManager) BlacklistedTables() []string { tm.mutex.Lock() diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index c62e47844d1..102c60f3095 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -146,3 +146,15 @@ func (hs *healthStreamer) ChangeState(tabletType topodatapb.TabletType, terTimes err: err, }) } + +func (hs *healthStreamer) Healthy() string { + hs.mu.Lock() + defer hs.mu.Unlock() + if hs.state.Serving { + return "" + } + if hs.state.RealtimeStats.HealthError == "" { + return "uhealthy" + } + return hs.state.RealtimeStats.HealthError +} diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 828dc560c02..1fdbbb34e6a 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -178,6 +178,7 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to }) tsv.exporter.NewGaugeDurationFunc("QueryTimeout", "Tablet server query timeout", tsv.QueryTimeout.Get) + tsv.registerHealthzHealthHandler() tsv.registerDebugHealthHandler() tsv.registerQueryzHandler() tsv.registerStreamQueryzHandlers() @@ -1402,6 +1403,24 @@ func (tsv *TabletServer) Close(ctx context.Context) error { return nil } +var okMessage = []byte("ok\n") + +func (tsv *TabletServer) registerHealthzHealthHandler() { + tsv.exporter.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if err := acl.CheckAccessHTTP(r, acl.MONITORING); err != nil { + acl.SendError(w, err) + return + } + if state := tsv.hs.Healthy(); state != "" { + http.Error(w, fmt.Sprintf("500 internal server error: tablet manager not healthy: %v", state), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Length", fmt.Sprintf("%v", len(okMessage))) + w.WriteHeader(http.StatusOK) + w.Write(okMessage) + }) +} + func (tsv *TabletServer) registerDebugHealthHandler() { tsv.exporter.HandleFunc("/debug/health", func(w http.ResponseWriter, r *http.Request) { if err := acl.CheckAccessHTTP(r, acl.MONITORING); err != nil { From 6dd319dcc592f2f24451e763ccef222660e6a2e6 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 7 Jul 2020 13:48:53 -0700 Subject: [PATCH 023/100] vttablet: fix more tests Signed-off-by: Sugu Sougoumarane --- .../endtoend/sharding/mergesharding/mergesharding_base.go | 4 ---- go/test/endtoend/sharding/resharding/resharding_base.go | 4 ---- go/vt/vttablet/tabletmanager/replmanager.go | 6 +++++- go/vt/wrangler/testlib/reparent_utils_test.go | 2 ++ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go/test/endtoend/sharding/mergesharding/mergesharding_base.go b/go/test/endtoend/sharding/mergesharding/mergesharding_base.go index 761409c1604..c36bfb26c66 100644 --- a/go/test/endtoend/sharding/mergesharding/mergesharding_base.go +++ b/go/test/endtoend/sharding/mergesharding/mergesharding_base.go @@ -438,10 +438,6 @@ func TestMergesharding(t *testing.T, useVarbinaryShardingKeyType bool) { assert.Equal(t, streamHealthResponse.Serving, false) assert.NotNil(t, streamHealthResponse.RealtimeStats) - // check the destination master 3 is healthy, even though its query - // service is not running (if not healthy this would exception out) - sharding.VerifyTabletHealth(t, *shard3Master, hostname) - // now serve rdonly from the split shards, in cell1 only err = clusterInstance.VtctlclientProcess.ExecuteCommand( "MigrateServedTypes", shard3Ks, "rdonly") diff --git a/go/test/endtoend/sharding/resharding/resharding_base.go b/go/test/endtoend/sharding/resharding/resharding_base.go index 6a80ec1ea9b..231360f2456 100644 --- a/go/test/endtoend/sharding/resharding/resharding_base.go +++ b/go/test/endtoend/sharding/resharding/resharding_base.go @@ -650,10 +650,6 @@ func TestResharding(t *testing.T, useVarbinaryShardingKeyType bool) { } - // check the destination master 3 is healthy, even though its query - // service is not running (if not healthy this would exception out) - sharding.VerifyTabletHealth(t, *shard3Master, hostname) - // now serve rdonly from the split shards, in cell1 only err = clusterInstance.VtctlclientProcess.ExecuteCommand( "MigrateServedTypes", fmt.Sprintf("--cells=%s", cell1), diff --git a/go/vt/vttablet/tabletmanager/replmanager.go b/go/vt/vttablet/tabletmanager/replmanager.go index 080cfcb0b36..d3d8feeedbc 100644 --- a/go/vt/vttablet/tabletmanager/replmanager.go +++ b/go/vt/vttablet/tabletmanager/replmanager.go @@ -37,7 +37,11 @@ const ( replicationStoppedFile = "do_not_replicate" ) -// replManager manages replication. +// replManager manages runs a poller to ensure mysql is replicating from +// the master. If necessary, it invokes tm.repairReplication to get it +// fixed. On state change, SetTabletType must be called before changing +// the tabletserver state. This will ensure that replication is fixed +// upfront, allowing tabletserver to start off healthy. type replManager struct { ctx context.Context tm *TabletManager diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 04a5dce8d99..230030f0265 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -130,6 +130,8 @@ func TestReparentTablet(t *testing.T) { replica.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", + "FAKE SET MASTER", + "FAKE SET MASTER", } replica.StartActionLoop(t, wr) defer replica.StopActionLoop(t) From ff9dd142126de9a435b1469cda0e954a08bbd7c4 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Tue, 7 Jul 2020 22:25:51 -0700 Subject: [PATCH 024/100] vttablet: replManager unit tests Signed-off-by: Sugu Sougoumarane --- .../tabletmanager/tablet_health_test.go | 4 +- go/timer/timer.go | 6 ++ go/timer/timer_flaky_test.go | 50 +++++------- .../tabletmanager/healthcheck_test.go | 40 ---------- go/vt/vttablet/tabletmanager/replmanager.go | 11 ++- .../tabletmanager/replmanager_test.go | 80 +++++++++++++++++++ .../tabletmanager/state_change_test.go | 39 +++++++++ 7 files changed, 156 insertions(+), 74 deletions(-) create mode 100644 go/vt/vttablet/tabletmanager/replmanager_test.go diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go index 2c2391e37ec..0e68fe83cbd 100644 --- a/go/test/endtoend/tabletmanager/tablet_health_test.go +++ b/go/test/endtoend/tabletmanager/tablet_health_test.go @@ -134,7 +134,7 @@ func TestHealthCheck(t *testing.T) { // stop replication, make sure we don't go unhealthy. // TODO: replace with StopReplication once StopSlave has been removed - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopSlave", rTablet.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rTablet.Alias) require.NoError(t, err) err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) require.NoError(t, err) @@ -145,7 +145,7 @@ func TestHealthCheck(t *testing.T) { verifyStreamHealth(t, result) // then restart replication, make sure we stay healthy - err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rTablet.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", rTablet.Alias) require.NoError(t, err) err = clusterInstance.VtctlclientProcess.ExecuteCommand("RunHealthCheck", rTablet.Alias) require.NoError(t, err) diff --git a/go/timer/timer.go b/go/timer/timer.go index 327a2e4d5c0..7f64e6fa3cb 100644 --- a/go/timer/timer.go +++ b/go/timer/timer.go @@ -157,3 +157,9 @@ func (tm *Timer) Stop() { func (tm *Timer) Interval() time.Duration { return tm.interval.Get() } + +func (tm *Timer) Running() bool { + tm.mu.Lock() + defer tm.mu.Unlock() + return tm.running +} diff --git a/go/timer/timer_flaky_test.go b/go/timer/timer_flaky_test.go index b89998a83f3..bc46b54acda 100644 --- a/go/timer/timer_flaky_test.go +++ b/go/timer/timer_flaky_test.go @@ -17,70 +17,60 @@ limitations under the License. package timer import ( - "sync/atomic" "testing" "time" + + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/sync2" ) const ( - half = time.Duration(500e5) - quarter = time.Duration(250e5) - tenth = time.Duration(100e5) + half = 50 * time.Millisecond + quarter = 25 * time.Millisecond + tenth = 10 * time.Millisecond ) -var numcalls int32 +var numcalls sync2.AtomicInt64 func f() { - atomic.AddInt32(&numcalls, 1) + numcalls.Add(1) } func TestWait(t *testing.T) { - atomic.StoreInt32(&numcalls, 0) + numcalls.Set(0) timer := NewTimer(quarter) + assert.False(t, timer.Running()) timer.Start(f) defer timer.Stop() + assert.True(t, timer.Running()) time.Sleep(tenth) - if atomic.LoadInt32(&numcalls) != 0 { - t.Errorf("want 0, received %v", numcalls) - } + assert.Equal(t, int64(0), numcalls.Get()) time.Sleep(quarter) - if atomic.LoadInt32(&numcalls) != 1 { - t.Errorf("want 1, received %v", numcalls) - } + assert.Equal(t, int64(1), numcalls.Get()) time.Sleep(quarter) - if atomic.LoadInt32(&numcalls) != 2 { - t.Errorf("want 1, received %v", numcalls) - } + assert.Equal(t, int64(2), numcalls.Get()) } func TestReset(t *testing.T) { - atomic.StoreInt32(&numcalls, 0) + numcalls.Set(0) timer := NewTimer(half) timer.Start(f) defer timer.Stop() timer.SetInterval(quarter) time.Sleep(tenth) - if atomic.LoadInt32(&numcalls) != 0 { - t.Errorf("want 0, received %v", numcalls) - } + assert.Equal(t, int64(0), numcalls.Get()) time.Sleep(quarter) - if atomic.LoadInt32(&numcalls) != 1 { - t.Errorf("want 1, received %v", numcalls) - } + assert.Equal(t, int64(1), numcalls.Get()) } func TestIndefinite(t *testing.T) { - atomic.StoreInt32(&numcalls, 0) + numcalls.Set(0) timer := NewTimer(0) timer.Start(f) defer timer.Stop() timer.TriggerAfter(quarter) time.Sleep(tenth) - if atomic.LoadInt32(&numcalls) != 0 { - t.Errorf("want 0, received %v", numcalls) - } + assert.Equal(t, int64(0), numcalls.Get()) time.Sleep(quarter) - if atomic.LoadInt32(&numcalls) != 1 { - t.Errorf("want 1, received %v", numcalls) - } + assert.Equal(t, int64(1), numcalls.Get()) } diff --git a/go/vt/vttablet/tabletmanager/healthcheck_test.go b/go/vt/vttablet/tabletmanager/healthcheck_test.go index f8e66cf3ead..ce346bdb09e 100644 --- a/go/vt/vttablet/tabletmanager/healthcheck_test.go +++ b/go/vt/vttablet/tabletmanager/healthcheck_test.go @@ -24,14 +24,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - "golang.org/x/net/context" - - "vitess.io/vitess/go/sync2" - "vitess.io/vitess/go/vt/binlog" - "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" - "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vttablet/tabletserver" "vitess.io/vitess/go/vt/vttablet/tabletservermock" @@ -128,38 +120,6 @@ func (fhc *fakeHealthCheck) HTMLName() template.HTML { return template.HTML("fakeHealthCheck") } -func createTestTM(ctx context.Context, t *testing.T, preStart func(*TabletManager)) *TabletManager { - ts := memorytopo.NewServer("cell1") - tablet := &topodatapb.Tablet{ - Alias: tabletAlias, - Hostname: "host", - PortMap: map[string]int32{ - "vt": int32(1234), - }, - Keyspace: "test_keyspace", - Shard: "0", - Type: topodatapb.TabletType_REPLICA, - } - - tm := &TabletManager{ - BatchCtx: ctx, - TopoServer: ts, - MysqlDaemon: &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(-1)}, - DBConfigs: &dbconfigs.DBConfigs{}, - QueryServiceControl: tabletservermock.NewController(), - UpdateStream: binlog.NewUpdateStreamControlMock(), - } - if preStart != nil { - preStart(tm) - } - err := tm.Start(tablet) - require.NoError(t, err) - - tm.HealthReporter = &fakeHealthCheck{} - - return tm -} - // expectBroadcastData checks that runHealthCheck() broadcasted the expected // stats (going the value for secondsBehindMaster). func expectBroadcastData(qsc tabletserver.Controller, serving bool, healthError string, secondsBehindMaster uint32) (*tabletservermock.BroadcastData, error) { diff --git a/go/vt/vttablet/tabletmanager/replmanager.go b/go/vt/vttablet/tabletmanager/replmanager.go index d3d8feeedbc..190c58c6ece 100644 --- a/go/vt/vttablet/tabletmanager/replmanager.go +++ b/go/vt/vttablet/tabletmanager/replmanager.go @@ -76,6 +76,9 @@ func (rm *replManager) SetTabletType(tabletType topodatapb.TabletType) { rm.ticks.Stop() return } + if rm.ticks.Running() { + return + } // Do an explicit check and then start the timer. rm.check() rm.ticks.Start(rm.check) @@ -115,16 +118,20 @@ func (rm *replManager) setReplicationStopped(stopped bool) { rm.replStopped = &stopped + if stopped { + rm.ticks.Stop() + } else { + rm.ticks.Start(rm.check) + } + if rm.markerFile == "" { return } if stopped { - rm.ticks.Stop() if file, err := os.Create(rm.markerFile); err == nil { file.Close() } } else { - rm.ticks.Start(rm.check) os.Remove(rm.markerFile) } } diff --git a/go/vt/vttablet/tabletmanager/replmanager_test.go b/go/vt/vttablet/tabletmanager/replmanager_test.go new file mode 100644 index 00000000000..b9e33c4aba0 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/replmanager_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletmanager + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" + "vitess.io/vitess/go/vt/mysqlctl" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +func TestReplManagerSetTabletType(t *testing.T) { + defer func(saved bool) { *mysqlctl.DisableActiveReparents = saved }(*mysqlctl.DisableActiveReparents) + *mysqlctl.DisableActiveReparents = true + + tm := &TabletManager{} + tm.replManager = newReplManager(context.Background(), tm, 100*time.Millisecond) + + // DisableActiveReparents == true should result in no-op + *mysqlctl.DisableActiveReparents = true + tm.replManager.SetTabletType(topodatapb.TabletType_REPLICA) + assert.False(t, tm.replManager.ticks.Running()) + + // Master should stop the manager + *mysqlctl.DisableActiveReparents = false + tm.replManager.ticks.Start(nil) + tm.replManager.SetTabletType(topodatapb.TabletType_MASTER) + assert.False(t, tm.replManager.ticks.Running()) + + // If replcation is stopped, the manager should not start + tm.replManager.setReplicationStopped(true) + tm.replManager.ticks.Start(nil) + tm.replManager.SetTabletType(topodatapb.TabletType_REPLICA) + assert.False(t, tm.replManager.ticks.Running()) + + // If the manager is already running, rm.check should not be called + tm.replManager.setReplicationStopped(false) + tm.replManager.ticks.Start(nil) + tm.replManager.SetTabletType(topodatapb.TabletType_REPLICA) + assert.True(t, tm.replManager.ticks.Running()) + tm.replManager.ticks.Stop() +} + +func TestReplManagerSetReplicaStopped(t *testing.T) { + defer func(saved bool) { *mysqlctl.DisableActiveReparents = saved }(*mysqlctl.DisableActiveReparents) + *mysqlctl.DisableActiveReparents = true + + tm := &TabletManager{} + tm.replManager = newReplManager(context.Background(), tm, 100*time.Millisecond) + + // DisableActiveReparents == true should result in no-op + *mysqlctl.DisableActiveReparents = true + tm.replManager.setReplicationStopped(true) + assert.False(t, tm.replManager.ticks.Running()) + tm.replManager.setReplicationStopped(false) + assert.False(t, tm.replManager.ticks.Running()) + + *mysqlctl.DisableActiveReparents = false + tm.replManager.setReplicationStopped(false) + assert.True(t, tm.replManager.ticks.Running()) + tm.replManager.setReplicationStopped(true) + assert.False(t, tm.replManager.ticks.Running()) +} diff --git a/go/vt/vttablet/tabletmanager/state_change_test.go b/go/vt/vttablet/tabletmanager/state_change_test.go index 31d8ab6c910..4365b97b66a 100644 --- a/go/vt/vttablet/tabletmanager/state_change_test.go +++ b/go/vt/vttablet/tabletmanager/state_change_test.go @@ -23,6 +23,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" + "vitess.io/vitess/go/sync2" + "vitess.io/vitess/go/vt/binlog" + "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/vttablet/tabletservermock" ) func TestPublishState(t *testing.T) { @@ -70,3 +77,35 @@ func TestPublishState(t *testing.T) { require.NoError(t, err) assert.Equal(t, tab2, ttablet.Tablet) } + +func createTestTM(ctx context.Context, t *testing.T, preStart func(*TabletManager)) *TabletManager { + ts := memorytopo.NewServer("cell1") + tablet := &topodatapb.Tablet{ + Alias: tabletAlias, + Hostname: "host", + PortMap: map[string]int32{ + "vt": int32(1234), + }, + Keyspace: "test_keyspace", + Shard: "0", + Type: topodatapb.TabletType_REPLICA, + } + + tm := &TabletManager{ + BatchCtx: ctx, + TopoServer: ts, + MysqlDaemon: &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(-1)}, + DBConfigs: &dbconfigs.DBConfigs{}, + QueryServiceControl: tabletservermock.NewController(), + UpdateStream: binlog.NewUpdateStreamControlMock(), + } + if preStart != nil { + preStart(tm) + } + err := tm.Start(tablet) + require.NoError(t, err) + + tm.HealthReporter = &fakeHealthCheck{} + + return tm +} From 30f98a8359f07fb428a3bc51069a11bf3ad52f62 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 11 Jul 2020 13:58:34 -0700 Subject: [PATCH 025/100] vttablet: delete deprecated code Signed-off-by: Sugu Sougoumarane --- go/cmd/vttablet/vttablet.go | 2 - go/vt/health/health.go | 175 --------- go/vt/health/health_test.go | 59 --- go/vt/tableacl/tableacl.go | 23 -- go/vt/tableacl/tableacl_test.go | 47 --- go/vt/vtcombo/tablet_map.go | 2 - go/vt/vtctld/tablet_data_test.go | 24 +- go/vt/vttablet/endtoend/misc_test.go | 2 +- go/vt/vttablet/tabletmanager/healthcheck.go | 335 ------------------ .../tabletmanager/healthcheck_test.go | 175 --------- .../tabletmanager/heartbeat_reporter.go | 57 --- .../tabletmanager/replication_reporter.go | 113 ------ .../replication_reporter_test.go | 112 ------ go/vt/vttablet/tabletmanager/replmanager.go | 21 +- go/vt/vttablet/tabletmanager/rpc_actions.go | 16 +- go/vt/vttablet/tabletmanager/rpc_backup.go | 2 +- .../vttablet/tabletmanager/rpc_replication.go | 1 - .../{vreplication.go => rpc_vreplication.go} | 0 go/vt/vttablet/tabletmanager/state_change.go | 34 -- .../tabletmanager/state_change_test.go | 5 +- go/vt/vttablet/tabletmanager/tm_init.go | 82 +---- go/vt/vttablet/tabletserver/controller.go | 6 +- go/vt/vttablet/tabletserver/tabletserver.go | 8 +- go/vt/vttablet/tabletservermock/controller.go | 11 +- 24 files changed, 42 insertions(+), 1270 deletions(-) delete mode 100644 go/vt/health/health.go delete mode 100644 go/vt/health/health_test.go delete mode 100644 go/vt/vttablet/tabletmanager/healthcheck.go delete mode 100644 go/vt/vttablet/tabletmanager/healthcheck_test.go delete mode 100644 go/vt/vttablet/tabletmanager/heartbeat_reporter.go delete mode 100644 go/vt/vttablet/tabletmanager/replication_reporter.go delete mode 100644 go/vt/vttablet/tabletmanager/replication_reporter_test.go rename go/vt/vttablet/tabletmanager/{vreplication.go => rpc_vreplication.go} (100%) diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index aed5ca61193..a5ddad3913d 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -24,7 +24,6 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/vt/binlog" "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -97,7 +96,6 @@ func main() { QueryServiceControl: qsc, UpdateStream: binlog.NewUpdateStream(ts, tablet.Keyspace, tabletAlias.Cell, qsc.SchemaEngine()), VREngine: vreplication.NewEngine(config, ts, tabletAlias.Cell, mysqld), - HealthReporter: health.DefaultAggregator, } if err := tm.Start(tablet); err != nil { log.Exitf("failed to parse -tablet-path: %v", err) diff --git a/go/vt/health/health.go b/go/vt/health/health.go deleted file mode 100644 index 812fc810cc5..00000000000 --- a/go/vt/health/health.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package health - -import ( - "errors" - "fmt" - "html/template" - "sort" - "strings" - "sync" - "time" -) - -var ( - // DefaultAggregator is the global aggregator to use for real - // programs. Use a custom one for tests. - DefaultAggregator *Aggregator - - // ErrReplicationNotRunning is returned by health plugins when replication - // is not running and we can't figure out the replication delay. - // Note everything else should be operational, and the underlying - // MySQL instance should be capable of answering queries. - ErrReplicationNotRunning = errors.New("replication is not running") -) - -func init() { - DefaultAggregator = NewAggregator() -} - -// Reporter reports the health status of a tablet. -type Reporter interface { - // Report returns the replication delay gathered by this - // module (or 0 if it thinks it's not behind), assuming that - // it is a Replication type or not, and that its query service - // should be running or not. If Report returns an error it - // implies that the tablet is in a bad shape and not able to - // handle queries. - Report(isReplicaType, shouldQueryServiceBeRunning bool) (replicationDelay time.Duration, err error) - - // HTMLName returns a displayable name for the module. - // Can be used to be displayed in the status page. - HTMLName() template.HTML -} - -// FunctionReporter is a function that may act as a Reporter. -type FunctionReporter func(bool, bool) (time.Duration, error) - -// Report implements Reporter.Report -func (fc FunctionReporter) Report(isReplicaType, shouldQueryServiceBeRunning bool) (time.Duration, error) { - return fc(isReplicaType, shouldQueryServiceBeRunning) -} - -// HTMLName implements Reporter.HTMLName -func (fc FunctionReporter) HTMLName() template.HTML { - return template.HTML("FunctionReporter") -} - -// Aggregator aggregates the results of many Reporters. -// It also implements the Reporter interface. -type Aggregator struct { - // mu protects all fields below its declaration. - mu sync.Mutex - reporters map[string]Reporter -} - -// NewAggregator returns a new empty Aggregator -func NewAggregator() *Aggregator { - return &Aggregator{ - reporters: make(map[string]Reporter), - } -} - -type singleResult struct { - name string - delay time.Duration - err error -} - -// Report aggregates health statuses from all the reporters. If any -// errors occur during the reporting, they will be logged, but only -// the first error will be returned. -// The returned replication delay will be the highest of all the replication -// delays returned by the Reporter implementations (although typically -// only one implementation will actually return a meaningful one). -func (ag *Aggregator) Report(isReplicaType, shouldQueryServiceBeRunning bool) (time.Duration, error) { - wg := sync.WaitGroup{} - results := make([]singleResult, len(ag.reporters)) - index := 0 - ag.mu.Lock() - for name, rep := range ag.reporters { - wg.Add(1) - go func(index int, name string, rep Reporter) { - defer wg.Done() - results[index].name = name - results[index].delay, results[index].err = rep.Report(isReplicaType, shouldQueryServiceBeRunning) - }(index, name, rep) - index++ - } - ag.mu.Unlock() - wg.Wait() - - // merge and return the results - var result time.Duration - var err error - for _, s := range results { - switch s.err { - case ErrReplicationNotRunning: - // Return the ErrReplicationNotRunning sentinel - // value, only if there are no other errors. - err = ErrReplicationNotRunning - case nil: - if s.delay > result { - result = s.delay - } - default: - return 0, fmt.Errorf("%v: %v", s.name, s.err) - } - } - return result, err -} - -// Register registers rep with ag. -func (ag *Aggregator) Register(name string, rep Reporter) { - ag.mu.Lock() - defer ag.mu.Unlock() - if _, ok := ag.reporters[name]; ok { - panic("reporter named " + name + " is already registered") - } - ag.reporters[name] = rep -} - -// RegisterSimpleCheck registers a simple health check function. -func (ag *Aggregator) RegisterSimpleCheck(name string, check func() error) { - ag.Register(name, simpleReporter{html: template.HTML(name), check: check}) -} - -// HTMLName returns an aggregate name for all the reporters -func (ag *Aggregator) HTMLName() template.HTML { - ag.mu.Lock() - defer ag.mu.Unlock() - result := make([]string, 0, len(ag.reporters)) - for _, rep := range ag.reporters { - result = append(result, string(rep.HTMLName())) - } - sort.Strings(result) - return template.HTML(strings.Join(result, "  +  ")) -} - -type simpleReporter struct { - html template.HTML - check func() error -} - -func (s simpleReporter) HTMLName() template.HTML { - return s.html -} - -func (s simpleReporter) Report(bool, bool) (time.Duration, error) { - return 0, s.check() -} diff --git a/go/vt/health/health_test.go b/go/vt/health/health_test.go deleted file mode 100644 index e6d6ef9a8a4..00000000000 --- a/go/vt/health/health_test.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package health - -import ( - "errors" - "html/template" - "testing" - "time" -) - -func TestReporters(t *testing.T) { - tests := []struct { - isReplicaType bool - shouldQueryServiceBeRunning bool - delay1 time.Duration - delay2 time.Duration - err error - wantDelay time.Duration - strict bool - }{ - {true, true, 10 * time.Second, 5 * time.Second, nil, 10 * time.Second, false}, - {true, false, 10 * time.Second, 5 * time.Second, errors.New("oops"), 0, false}, - {true, false, 10 * time.Second, 5 * time.Second, ErrReplicationNotRunning, 10 * time.Second, true}, - } - for _, test := range tests { - ag := NewAggregator() - ag.Register("1", FunctionReporter(func(bool, bool) (time.Duration, error) { - return test.delay1, nil - })) - ag.Register("2", FunctionReporter(func(bool, bool) (time.Duration, error) { - return test.delay2, nil - })) - ag.RegisterSimpleCheck("simplecheck", func() error { return test.err }) - delay, err := ag.Report(test.isReplicaType, test.shouldQueryServiceBeRunning) - if delay != test.wantDelay || test.strict && err != test.err || (err == nil) != (test.err == nil) { - t.Errorf("ag.Report(%v, %v) = (%v, %v), want (%v, %v)", - test.isReplicaType, test.shouldQueryServiceBeRunning, delay, err, test.wantDelay, test.err) - } - wantHTML := template.HTML("FunctionReporter  +  FunctionReporter  +  simplecheck") - if got, want := ag.HTMLName(), wantHTML; got != want { - t.Errorf("ag.HTMLName() = %v, want %v", got, want) - } - } -} diff --git a/go/vt/tableacl/tableacl.go b/go/vt/tableacl/tableacl.go index 80380d55d5c..2db0fe686ab 100644 --- a/go/vt/tableacl/tableacl.go +++ b/go/vt/tableacl/tableacl.go @@ -28,11 +28,8 @@ import ( "github.com/golang/protobuf/proto" "github.com/tchap/go-patricia/patricia" "vitess.io/vitess/go/json2" - "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/tableacl/acl" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" tableaclpb "vitess.io/vitess/go/vt/proto/tableacl" ) @@ -319,23 +316,3 @@ func GetCurrentACLFactory() (acl.Factory, error) { } return nil, fmt.Errorf("aclFactory for given default: %s is not found", defaultACL) } - -func checkHealth(acl *tableACL) error { - if !acl.Valid() { - return errors.New("the tableacl is not valid") - } - return nil -} - -func init() { - servenv.OnRun(func() { - currentConfig := tabletenv.NewCurrentConfig() - if !currentConfig.StrictTableACL { - return - } - if currentConfig.EnableTableACLDryRun { - return - } - health.DefaultAggregator.RegisterSimpleCheck("tableacl", func() error { return checkHealth(¤tTableACL) }) - }) -} diff --git a/go/vt/tableacl/tableacl_test.go b/go/vt/tableacl/tableacl_test.go index 0b652c24bc2..923a6eccef2 100644 --- a/go/vt/tableacl/tableacl_test.go +++ b/go/vt/tableacl/tableacl_test.go @@ -23,11 +23,9 @@ import ( "os" "reflect" "testing" - "time" "github.com/golang/protobuf/proto" - "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/tableacl/acl" "vitess.io/vitess/go/vt/tableacl/simpleacl" @@ -248,48 +246,3 @@ func TestGetCurrentACLFactoryWithWrongDefault(t *testing.T) { t.Fatalf("there are more than one acl factories, but the default given does not match any of these.") } } - -func TestHealthWithACL(t *testing.T) { - tacl := tableACL{factory: &simpleacl.Factory{}} - ha := health.NewAggregator() - tests := []struct { - config *tableaclpb.Config - wantOK bool - }{ - { - config: nil, - wantOK: false, - }, - { - config: &tableaclpb.Config{ - TableGroups: []*tableaclpb.TableGroupSpec{{ - Name: "group01", - TableNamesOrPrefixes: []string{"test_table"}, - Readers: []string{"vt"}, - Writers: []string{"vt"}, - }}}, - wantOK: true, - }, - { - config: &tableaclpb.Config{}, - wantOK: true, - }, - } - ha.RegisterSimpleCheck("tableacl", func() error { return checkHealth(&tacl) }) - for _, test := range tests { - if test.config != nil { - if err := tacl.Set(test.config); err != nil { - t.Fatalf("tacl.Set(%#v) = %v, want %v", test.config, err, nil) - } - } - delay, err := ha.Report(true, true) - wantErr := error(nil) - if !test.wantOK { - wantErr = errors.New("") - } - wantDelay := time.Duration(0) - if delay != wantDelay || (err == nil) != test.wantOK { - t.Errorf("ha.Report(true, true) == (%v, %v), want (%v, %v)", delay, err, wantDelay, wantErr) - } - } -} diff --git a/go/vt/vtcombo/tablet_map.go b/go/vt/vtcombo/tablet_map.go index 5c2365534a0..6206a58c55b 100644 --- a/go/vt/vtcombo/tablet_map.go +++ b/go/vt/vtcombo/tablet_map.go @@ -28,7 +28,6 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/grpcclient" - "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/hook" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" @@ -97,7 +96,6 @@ func CreateTablet(ctx context.Context, ts *topo.Server, cell string, uid uint32, MysqlDaemon: mysqld, DBConfigs: dbcfgs, QueryServiceControl: controller, - HealthReporter: health.DefaultAggregator, } tablet := &topodatapb.Tablet{ Alias: alias, diff --git a/go/vt/vtctld/tablet_data_test.go b/go/vt/vtctld/tablet_data_test.go index 649bed081cb..fe9da760f32 100644 --- a/go/vt/vtctld/tablet_data_test.go +++ b/go/vt/vtctld/tablet_data_test.go @@ -89,10 +89,14 @@ func (s *streamHealthTabletServer) streamHealthUnregister(id int) error { } // BroadcastHealth will broadcast the current health to all listeners -func (s *streamHealthTabletServer) BroadcastHealth(terTimestamp int64, stats *querypb.RealtimeStats, maxCache time.Duration) { +func (s *streamHealthTabletServer) BroadcastHealth() { shr := &querypb.StreamHealthResponse{ - TabletExternallyReparentedTimestamp: terTimestamp, - RealtimeStats: stats, + TabletExternallyReparentedTimestamp: 42, + RealtimeStats: &querypb.RealtimeStats{ + HealthError: "testHealthError", + SecondsBehindMaster: 72, + CpuUsage: 1.1, + }, } s.streamHealthMutex.Lock() @@ -121,12 +125,6 @@ func TestTabletData(t *testing.T) { thc := newTabletHealthCache(ts) - stats := &querypb.RealtimeStats{ - HealthError: "testHealthError", - SecondsBehindMaster: 72, - CpuUsage: 1.1, - } - // Keep broadcasting until the first result goes through. stop := make(chan struct{}) go func() { @@ -135,7 +133,7 @@ func TestTabletData(t *testing.T) { case <-stop: return default: - shsq.BroadcastHealth(42, stats, time.Minute) + shsq.BroadcastHealth() } } }() @@ -149,6 +147,12 @@ func TestTabletData(t *testing.T) { if err != nil { t.Fatalf("thc.Get failed: %v", err) } + + stats := &querypb.RealtimeStats{ + HealthError: "testHealthError", + SecondsBehindMaster: 72, + CpuUsage: 1.1, + } if got, want := result.RealtimeStats, stats; !proto.Equal(got, want) { t.Errorf("RealtimeStats = %#v, want %#v", got, want) } diff --git a/go/vt/vttablet/endtoend/misc_test.go b/go/vt/vttablet/endtoend/misc_test.go index 10e71b93d34..770f81781ef 100644 --- a/go/vt/vttablet/endtoend/misc_test.go +++ b/go/vt/vttablet/endtoend/misc_test.go @@ -415,7 +415,7 @@ func TestHealth(t *testing.T) { func TestStreamHealth(t *testing.T) { var health *querypb.StreamHealthResponse - framework.Server.BroadcastHealth(0, nil, time.Minute) + framework.Server.BroadcastHealth() if err := framework.Server.StreamHealth(context.Background(), func(shr *querypb.StreamHealthResponse) error { health = shr return io.EOF diff --git a/go/vt/vttablet/tabletmanager/healthcheck.go b/go/vt/vttablet/tabletmanager/healthcheck.go deleted file mode 100644 index 283aaace780..00000000000 --- a/go/vt/vttablet/tabletmanager/healthcheck.go +++ /dev/null @@ -1,335 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tabletmanager - -// This file handles the health check. It is always enabled in production -// vttablets (but not in vtcombo, and not in unit tests by default). -// If we are unhealthy, we'll stop the query service. In any case, -// we report our replication delay so vtgate's discovery can use this tablet -// or not. -// -// Note: we used to go to SPARE when unhealthy, and back to the target -// tablet type when healhty. Now that we use the discovery module, -// health is handled by clients subscribing to the health stream, so -// we don't need to do that any more. - -import ( - "fmt" - "html/template" - "time" - - "vitess.io/vitess/go/timer" - "vitess.io/vitess/go/vt/health" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/topo" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -var ( - healthCheckInterval = 20 * time.Second - degradedThreshold = 30 * time.Second - unhealthyThreshold = 2 * time.Hour -) - -// HealthRecord records one run of the health checker. -type HealthRecord struct { - Time time.Time - Error error - IgnoredError error - IgnoreErrorExpr string - ReplicationDelay time.Duration -} - -// Class returns a human-readable one word version of the health state. -func (r *HealthRecord) Class() string { - switch { - case r.Error != nil: - return "unhealthy" - case r.ReplicationDelay > degradedThreshold: - return "unhappy" - default: - return "healthy" - } -} - -// HTML returns an HTML version to be displayed on UIs. -func (r *HealthRecord) HTML() template.HTML { - switch { - case r.Error != nil: - return template.HTML(fmt.Sprintf("unhealthy: %v", r.Error)) - case r.ReplicationDelay > degradedThreshold: - return template.HTML(fmt.Sprintf("unhappy: %v behind on replication", r.ReplicationDelay)) - default: - html := "healthy" - if r.ReplicationDelay > 0 { - html += fmt.Sprintf(": only %v behind on replication", r.ReplicationDelay) - } - if r.IgnoredError != nil { - html += fmt.Sprintf(" (ignored error: %v, matches expression: %v)", r.IgnoredError, r.IgnoreErrorExpr) - } - return template.HTML(html) - } -} - -// Degraded returns true if the replication delay is beyond degradedThreshold. -func (r *HealthRecord) Degraded() bool { - return r.ReplicationDelay > degradedThreshold -} - -// ErrorString returns Error as a string. -func (r *HealthRecord) ErrorString() string { - if r.Error == nil { - return "" - } - return r.Error.Error() -} - -// IgnoredErrorString returns IgnoredError as a string. -func (r *HealthRecord) IgnoredErrorString() string { - if r.IgnoredError == nil { - return "" - } - return r.IgnoredError.Error() -} - -// IsDuplicate implements history.Deduplicable -func (r *HealthRecord) IsDuplicate(other interface{}) bool { - rother, ok := other.(*HealthRecord) - if !ok { - return false - } - return r.ErrorString() == rother.ErrorString() && - r.IgnoredErrorString() == rother.IgnoredErrorString() && - r.IgnoreErrorExpr == rother.IgnoreErrorExpr && - r.Degraded() == rother.Degraded() -} - -// ConfigHTML returns a formatted summary of health checking config values. -func ConfigHTML() template.HTML { - return template.HTML(fmt.Sprintf( - "healthCheckInterval: %v; degradedThreshold: %v; unhealthyThreshold: %v", - healthCheckInterval, degradedThreshold, unhealthyThreshold)) -} - -// initHealthCheck will start the health check background go routine, -// and configure the healthcheck shutdown. It is only run by NewTabletManager -// for real vttablet tms (not by tests, nor vtcombo). -func (tm *TabletManager) initHealthCheck() { - if tm.HealthReporter == nil { - return - } - registerReplicationReporter(tm) - registerHeartbeatReporter(tm.QueryServiceControl) - - log.Infof("Starting periodic health check every %v", healthCheckInterval) - t := timer.NewTimer(healthCheckInterval) - servenv.OnTermSync(func() { - // When we enter lameduck mode, we want to not call - // the health check any more. After this returns, we - // are guaranteed to not call it. - log.Info("Stopping periodic health check timer") - t.Stop() - - // Now we can finish up and force ourselves to not healthy. - tm.terminateHealthChecks() - }) - t.Start(func() { - tm.runHealthCheck() - }) - t.Trigger() -} - -// runHealthCheck takes the action mutex, runs the health check, -// and if we need to change our state, do it. We never change our type, -// just the health we report (so we do not change the topo server at all). -// We do not interact with topo server, we use cached values for everything. -// -// This will not change the BinlogPlayerMap, but if it is not empty, -// we will think we should not be running the query service. -// -// This will not change the TabletControl record, but will use it -// to see if we should be running the query service. -func (tm *TabletManager) runHealthCheck() { - if err := tm.lock(tm.BatchCtx); err != nil { - log.Warningf("cannot lock actionMutex, not running HealthCheck") - return - } - defer tm.unlock() - - tm.runHealthCheckLocked() -} - -func (tm *TabletManager) runHealthCheckLocked() { - if tm.HealthReporter == nil { - return - } - tm.checkLock() - // read the current tablet record and tablet control - tablet := tm.Tablet() - terTime := tm.masterTermStartTime() - tm.mutex.Lock() - shouldBeServing := tm._disallowQueryService == "" - ignoreErrorExpr := tm._ignoreHealthErrorExpr - tm.mutex.Unlock() - - // run the health check - record := &HealthRecord{} - isReplicaType := true - if tablet.Type == topodatapb.TabletType_MASTER { - isReplicaType = false - } - - // Remember the health error as healthErr to be sure we don't - // accidentally overwrite it with some other err. - replicationDelay, healthErr := tm.HealthReporter.Report(isReplicaType, shouldBeServing) - if healthErr != nil && ignoreErrorExpr != nil && - ignoreErrorExpr.MatchString(healthErr.Error()) { - // we need to ignore this health error - record.IgnoredError = healthErr - record.IgnoreErrorExpr = ignoreErrorExpr.String() - healthErr = nil - } - if healthErr == health.ErrReplicationNotRunning { - // Replication is not running, so we just don't know the - // delay. Use a maximum delay, so we can let vtgate - // find the right replica, instead of erroring out. - // (this works as the check below is a strict > operator). - replicationDelay = unhealthyThreshold - healthErr = nil - } - if healthErr == nil { - if replicationDelay > unhealthyThreshold { - healthErr = fmt.Errorf("reported replication lag: %v higher than unhealthy threshold: %v", replicationDelay.Seconds(), unhealthyThreshold.Seconds()) - } - } - - // Figure out if we should be running QueryService, see if we are, - // and reconcile. - if healthErr != nil { - if tablet.Type != topodatapb.TabletType_DRAINED { - // We are not healthy and must shut down QueryService. - // At the moment, the only exception to this are "worker" tablets which - // still must serve queries e.g. as source tablet during a "SplitClone". - shouldBeServing = false - } - } - isServing := tm.QueryServiceControl.IsServing() - if shouldBeServing { - if !isServing { - // If starting queryservice fails, that's our - // new reason for being unhealthy. - // - // We don't care if the QueryService state actually - // changed because we'll broadcast the latest health - // status after this immediately anyway. - _ /* state changed */, healthErr = tm.QueryServiceControl.SetServingType(tablet.Type, terTime, true, nil) - } - } else { - if isServing { - // We are not healthy or should not be running - // the query service. - - // First enter lameduck during gracePeriod to - // limit client errors. - if topo.IsSubjectToLameduck(tablet.Type) && *gracePeriod > 0 { - tm.lameduck("health check failed") - } - - // We don't care if the QueryService state actually - // changed because we'll broadcast the latest health - // status after this immediately anyway. - log.Infof("Disabling query service because of health-check failure: %v", healthErr) - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, terTime, false, nil); err != nil { - log.Errorf("SetServingType(serving=false) failed: %v", err) - } - } - } - if tm.UpdateStream != nil { - if topo.IsRunningUpdateStream(tablet.Type) { - tm.UpdateStream.Enable() - } else { - tm.UpdateStream.Disable() - } - } - - // All master tablets have to run the VReplication engine. - // There is no guarantee that VREngine was successfully started when tabletmanager - // came up. This is because the mysql could have been in read-only mode, etc. - // So, start the engine if it's not already running. - if tablet.Type == topodatapb.TabletType_MASTER && tm.VREngine != nil && !tm.VREngine.IsOpen() { - tm.VREngine.Open(tm.BatchCtx) - } - - // save the health record - record.Time = time.Now() - record.Error = healthErr - record.ReplicationDelay = replicationDelay - tm.History.Add(record) - - // remember our health status - tm.mutex.Lock() - tm._healthy = healthErr - tm._healthyTime = time.Now() - tm._replicationDelay = replicationDelay - tm.mutex.Unlock() - - // send it to our observers - tm.broadcastHealth() -} - -// terminateHealthChecks is called when we enter lame duck mode. -// We will clean up our state, and set query service to lame duck mode. -// We only do something if we are in a serving state, and not a master. -func (tm *TabletManager) terminateHealthChecks() { - // No need to check for error, only a canceled batchCtx would fail this. - tm.lock(tm.BatchCtx) - defer tm.unlock() - log.Info("tm.terminateHealthChecks is starting") - - // read the current tablet record - tablet := tm.Tablet() - if !topo.IsSubjectToLameduck(tablet.Type) { - // If we're MASTER, SPARE, WORKER, etc. then we - // shouldn't enter lameduck. We do lameduck to not - // trigger errors on clients. - log.Infof("Tablet in state %v, not entering lameduck", tablet.Type) - return - } - - // Go lameduck for gracePeriod. - // We've already checked above that we're not MASTER. - - // Enter new lameduck mode for gracePeriod, then shut down - // queryservice. New lameduck mode means keep accepting - // queries, but advertise unhealthy. After we return from - // this synchronous OnTermSync hook, servenv may decide to - // wait even longer, for the rest of the time specified by its - // own "-lameduck-period" flag. During that extra period, - // queryservice will be in old lameduck mode, meaning stay - // alive but reject new queries. - tm.lameduck("terminating healthchecks") - - // Note we only do this now if we entered lameduck. In the - // master case for instance, we want to keep serving until - // vttablet dies entirely (where else is the client going to - // go?). After servenv lameduck, the queryservice is stopped - // from a servenv.OnClose() hook anyway. - log.Infof("Disabling query service after lameduck in terminating healthchecks") - tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), false, nil) -} diff --git a/go/vt/vttablet/tabletmanager/healthcheck_test.go b/go/vt/vttablet/tabletmanager/healthcheck_test.go deleted file mode 100644 index ce346bdb09e..00000000000 --- a/go/vt/vttablet/tabletmanager/healthcheck_test.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tabletmanager - -import ( - "errors" - "fmt" - "html/template" - "reflect" - "testing" - "time" - - "vitess.io/vitess/go/vt/vttablet/tabletserver" - "vitess.io/vitess/go/vt/vttablet/tabletservermock" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - - // needed so that grpc client is registered - _ "vitess.io/vitess/go/vt/vttablet/grpctmclient" -) - -func TestHealthRecordDeduplication(t *testing.T) { - now := time.Now() - later := now.Add(5 * time.Minute) - cases := []struct { - left, right *HealthRecord - duplicate bool - }{ - { - left: &HealthRecord{Time: now}, - right: &HealthRecord{Time: later}, - duplicate: true, - }, - { - left: &HealthRecord{Time: now, Error: errors.New("foo")}, - right: &HealthRecord{Time: now, Error: errors.New("foo")}, - duplicate: true, - }, - { - left: &HealthRecord{Time: now, ReplicationDelay: degradedThreshold / 2}, - right: &HealthRecord{Time: later, ReplicationDelay: degradedThreshold / 3}, - duplicate: true, - }, - { - left: &HealthRecord{Time: now, ReplicationDelay: degradedThreshold / 2}, - right: &HealthRecord{Time: later, ReplicationDelay: degradedThreshold * 2}, - duplicate: false, - }, - { - left: &HealthRecord{Time: now, Error: errors.New("foo"), ReplicationDelay: degradedThreshold * 2}, - right: &HealthRecord{Time: later, ReplicationDelay: degradedThreshold * 2}, - duplicate: false, - }, - } - - for _, c := range cases { - if got := c.left.IsDuplicate(c.right); got != c.duplicate { - t.Errorf("IsDuplicate %v and %v: got %v, want %v", c.left, c.right, got, c.duplicate) - } - } -} - -func TestHealthRecordClass(t *testing.T) { - cases := []struct { - r *HealthRecord - state string - }{ - { - r: &HealthRecord{}, - state: "healthy", - }, - { - r: &HealthRecord{Error: errors.New("foo")}, - state: "unhealthy", - }, - { - r: &HealthRecord{ReplicationDelay: degradedThreshold * 2}, - state: "unhappy", - }, - { - r: &HealthRecord{ReplicationDelay: degradedThreshold / 2}, - state: "healthy", - }, - } - - for _, c := range cases { - if got := c.r.Class(); got != c.state { - t.Errorf("class of %v: got %v, want %v", c.r, got, c.state) - } - } -} - -var tabletAlias = &topodatapb.TabletAlias{Cell: "cell1", Uid: 42} - -// fakeHealthCheck implements health.Reporter interface -type fakeHealthCheck struct { - reportReplicationDelay time.Duration - reportError error -} - -func (fhc *fakeHealthCheck) Report(isReplicaType, shouldQueryServiceBeRunning bool) (replicationDelay time.Duration, err error) { - return fhc.reportReplicationDelay, fhc.reportError -} - -func (fhc *fakeHealthCheck) HTMLName() template.HTML { - return template.HTML("fakeHealthCheck") -} - -// expectBroadcastData checks that runHealthCheck() broadcasted the expected -// stats (going the value for secondsBehindMaster). -func expectBroadcastData(qsc tabletserver.Controller, serving bool, healthError string, secondsBehindMaster uint32) (*tabletservermock.BroadcastData, error) { - bd := <-qsc.(*tabletservermock.Controller).BroadcastData - if got := bd.Serving; got != serving { - return nil, fmt.Errorf("unexpected BroadcastData.Serving, got: %v want: %v with bd: %+v", got, serving, bd) - } - if got := bd.RealtimeStats.HealthError; got != healthError { - return nil, fmt.Errorf("unexpected BroadcastData.HealthError, got: %v want: %v with bd: %+v", got, healthError, bd) - } - if got := bd.RealtimeStats.SecondsBehindMaster; got != secondsBehindMaster { - return nil, fmt.Errorf("unexpected BroadcastData.SecondsBehindMaster, got: %v want: %v with bd: %+v", got, secondsBehindMaster, bd) - } - return bd, nil -} - -// expectBroadcastDataEmpty closes the health broadcast channel and verifies -// that all broadcasted messages were consumed by expectBroadcastData(). -func expectBroadcastDataEmpty(qsc tabletserver.Controller) error { - c := qsc.(*tabletservermock.Controller).BroadcastData - close(c) - bd, ok := <-c - if ok { - return fmt.Errorf("BroadcastData channel should have been consumed, but was not: %v", bd) - } - return nil -} - -// expectStateChange verifies that the test changed the QueryService state -// to the expected state (serving or not, specific tablet type). -func expectStateChange(qsc tabletserver.Controller, serving bool, tabletType topodatapb.TabletType) error { - want := &tabletservermock.StateChange{ - Serving: serving, - TabletType: tabletType, - } - got := <-qsc.(*tabletservermock.Controller).StateChanges - if !reflect.DeepEqual(got, want) { - return fmt.Errorf("unexpected state change. got: %v want: %v", got, want) - } - return nil -} - -// expectStateChangesEmpty closes the StateChange channel and verifies -// that all sent state changes were consumed by expectStateChange(). -func expectStateChangesEmpty(qsc tabletserver.Controller) error { - c := qsc.(*tabletservermock.Controller).StateChanges - close(c) - sc, ok := <-c - if ok { - return fmt.Errorf("StateChanges channel should have been consumed, but was not: %v", sc) - } - return nil -} diff --git a/go/vt/vttablet/tabletmanager/heartbeat_reporter.go b/go/vt/vttablet/tabletmanager/heartbeat_reporter.go deleted file mode 100644 index 7697e376397..00000000000 --- a/go/vt/vttablet/tabletmanager/heartbeat_reporter.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tabletmanager - -import ( - "html/template" - "time" - - "vitess.io/vitess/go/vt/health" - "vitess.io/vitess/go/vt/vttablet/tabletserver" - "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" -) - -// Reporter is a wrapper around a heartbeat Reader, to be used as an interface from -// the health check system. -type Reporter struct { - controller tabletserver.Controller -} - -// RegisterReporter registers the heartbeat reader as a healthcheck reporter so that its -// measurements will be picked up in healthchecks. -func registerHeartbeatReporter(controller tabletserver.Controller) { - if tabletenv.NewCurrentConfig().ReplicationTracker.Mode != tabletenv.Heartbeat { - return - } - - reporter := &Reporter{controller} - health.DefaultAggregator.Register("heartbeat_reporter", reporter) -} - -// HTMLName is part of the health.Reporter interface. -func (r *Reporter) HTMLName() template.HTML { - return template.HTML("MySQLHeartbeat") -} - -// Report is part of the health.Reporter interface. It returns the last reported value -// written by the watchHeartbeat goroutine. If we're the master, it just returns 0. -func (r *Reporter) Report(isReplicaType, shouldQueryServiceBeRunning bool) (time.Duration, error) { - if !isReplicaType { - return 0, nil - } - return r.controller.HeartbeatLag() -} diff --git a/go/vt/vttablet/tabletmanager/replication_reporter.go b/go/vt/vttablet/tabletmanager/replication_reporter.go deleted file mode 100644 index 08218109e68..00000000000 --- a/go/vt/vttablet/tabletmanager/replication_reporter.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreedto in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tabletmanager - -import ( - "html/template" - "time" - - "golang.org/x/net/context" - - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/vt/health" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/mysqlctl" -) - -var enableReplicationReporter bool - -// replicationReporter implements health.Reporter -type replicationReporter struct { - // set at construction time - tm *TabletManager - now func() time.Time - - // store the last time we successfully got the lag, so if we - // can't get the lag any more, we can extrapolate. - lastKnownValue time.Duration - lastKnownTime time.Time -} - -// Report is part of the health.Reporter interface -func (r *replicationReporter) Report(isReplicaType, shouldQueryServiceBeRunning bool) (time.Duration, error) { - if !isReplicaType { - return 0, nil - } - - status, statusErr := r.tm.MysqlDaemon.ReplicationStatus() - if statusErr == mysql.ErrNotReplica || - (statusErr == nil && !status.SQLThreadRunning && !status.IOThreadRunning) { - // MySQL is up, but replica is either not configured or not running. - // Both SQL and IO threads are stopped, so it's probably either - // stopped on purpose, or stopped because of a mysqld restart. - if !r.tm.replicationStopped() { - // As far as we've been told, it isn't stopped on purpose, - // so let's try to start it. - if *mysqlctl.DisableActiveReparents { - log.Infof("replication is stopped. Running with --disable_active_reparents so will not try to reconnect to master...") - } else { - log.Infof("replication is stopped. Trying to reconnect to master...") - ctx, cancel := context.WithTimeout(r.tm.BatchCtx, 5*time.Second) - if err := r.tm.repairReplication(ctx); err != nil { - log.Infof("Failed to reconnect to master: %v", err) - } - cancel() - // Check status again. - status, statusErr = r.tm.MysqlDaemon.ReplicationStatus() - } - } - } - if statusErr != nil { - // mysqld is not running or replication is not configured. - // We can't report healthy. - return 0, statusErr - } - if !status.ReplicationRunning() { - // mysqld is running, but replication is not replicating (most likely, - // replication has been stopped). See if we can extrapolate. - if r.lastKnownTime.IsZero() { - // we can't. - return 0, health.ErrReplicationNotRunning - } - - // we can extrapolate with the worst possible - // value (that is we made no replication - // progress since last time, and just fell more behind). - elapsed := r.now().Sub(r.lastKnownTime) - return elapsed + r.lastKnownValue, nil - } - - // we got a real value, save it. - r.lastKnownValue = time.Duration(status.SecondsBehindMaster) * time.Second - r.lastKnownTime = r.now() - return r.lastKnownValue, nil -} - -// HTMLName is part of the health.Reporter interface -func (r *replicationReporter) HTMLName() template.HTML { - return template.HTML("MySQLReplicationLag") -} - -func registerReplicationReporter(tm *TabletManager) { - if enableReplicationReporter { - health.DefaultAggregator.Register("replication_reporter", - &replicationReporter{ - tm: tm, - now: time.Now, - }) - } -} diff --git a/go/vt/vttablet/tabletmanager/replication_reporter_test.go b/go/vt/vttablet/tabletmanager/replication_reporter_test.go deleted file mode 100644 index 8a70ad2091e..00000000000 --- a/go/vt/vttablet/tabletmanager/replication_reporter_test.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tabletmanager - -import ( - "errors" - "testing" - "time" - - "vitess.io/vitess/go/vt/health" - "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" -) - -func TestBasicMySQLReplicationLag(t *testing.T) { - mysqld := fakemysqldaemon.NewFakeMysqlDaemon(nil) - mysqld.Replicating = true - mysqld.SecondsBehindMaster = 10 - replicationStopped := true - - rep := &replicationReporter{ - tm: &TabletManager{MysqlDaemon: mysqld, _replicationStopped: &replicationStopped}, - now: time.Now, - } - dur, err := rep.Report(true, true) - if err != nil || dur != 10*time.Second { - t.Fatalf("wrong Report result: %v %v", dur, err) - } -} - -func TestNoKnownMySQLReplicationLag(t *testing.T) { - mysqld := fakemysqldaemon.NewFakeMysqlDaemon(nil) - mysqld.Replicating = false - replicationStopped := true - - rep := &replicationReporter{ - tm: &TabletManager{MysqlDaemon: mysqld, _replicationStopped: &replicationStopped}, - now: time.Now, - } - dur, err := rep.Report(true, true) - if err != health.ErrReplicationNotRunning { - t.Fatalf("wrong Report result: %v %v", dur, err) - } -} - -func TestExtrapolatedMySQLReplicationLag(t *testing.T) { - mysqld := fakemysqldaemon.NewFakeMysqlDaemon(nil) - mysqld.Replicating = true - mysqld.SecondsBehindMaster = 10 - replicationStopped := true - - now := time.Now() - rep := &replicationReporter{ - tm: &TabletManager{MysqlDaemon: mysqld, _replicationStopped: &replicationStopped}, - now: func() time.Time { return now }, - } - - // seed the last known value with a good value - dur, err := rep.Report(true, true) - if err != nil || dur != 10*time.Second { - t.Fatalf("wrong Report result: %v %v", dur, err) - } - - // now 20 seconds later, we're not replicating any more, - // we should get 20 more seconds in lag - now = now.Add(20 * time.Second) - mysqld.Replicating = false - dur, err = rep.Report(true, true) - if err != nil || dur != 30*time.Second { - t.Fatalf("wrong Report result: %v %v", dur, err) - } -} - -func TestNoExtrapolatedMySQLReplicationLag(t *testing.T) { - mysqld := fakemysqldaemon.NewFakeMysqlDaemon(nil) - mysqld.Replicating = true - mysqld.SecondsBehindMaster = 10 - replicationStopped := true - - now := time.Now() - rep := &replicationReporter{ - tm: &TabletManager{MysqlDaemon: mysqld, _replicationStopped: &replicationStopped}, - now: func() time.Time { return now }, - } - - // seed the last known value with a good value - dur, err := rep.Report(true, true) - if err != nil || dur != 10*time.Second { - t.Fatalf("wrong Report result: %v %v", dur, err) - } - - // now 20 seconds later, mysqld is down - now = now.Add(20 * time.Second) - mysqld.ReplicationStatusError = errors.New("mysql is down") - _, err = rep.Report(true, true) - if err != mysqld.ReplicationStatusError { - t.Fatalf("wrong Report error: %v", err) - } -} diff --git a/go/vt/vttablet/tabletmanager/replmanager.go b/go/vt/vttablet/tabletmanager/replmanager.go index 190c58c6ece..52b30988461 100644 --- a/go/vt/vttablet/tabletmanager/replmanager.go +++ b/go/vt/vttablet/tabletmanager/replmanager.go @@ -43,10 +43,11 @@ const ( // the tabletserver state. This will ensure that replication is fixed // upfront, allowing tabletserver to start off healthy. type replManager struct { - ctx context.Context - tm *TabletManager - markerFile string - ticks *timer.Timer + ctx context.Context + tm *TabletManager + markerFile string + ticks *timer.Timer + firstFailure bool // replStopped is tri-state. // A nil value signifies that the value is not set. @@ -98,12 +99,20 @@ func (rm *replManager) check() { } } - log.Infof("Replication is stopped. Trying to reconnect to master...") + if rm.firstFailure { + log.Infof("Replication is stopped, reconnecting to master.") + } ctx, cancel := context.WithTimeout(rm.ctx, 5*time.Second) defer cancel() if err := rm.tm.repairReplication(ctx); err != nil { - log.Infof("Failed to reconnect to master: %v", err) + if rm.firstFailure { + rm.firstFailure = false + log.Infof("Failed to reconnect to master: %v, will keep retrying.", err) + } + return } + log.Info("Successfully reconnected to master.") + rm.firstFailure = true } // setReplicationStopped performs a best effort attempt of diff --git a/go/vt/vttablet/tabletmanager/rpc_actions.go b/go/vt/vttablet/tabletmanager/rpc_actions.go index d94ac935c28..2be85e5808a 100644 --- a/go/vt/vttablet/tabletmanager/rpc_actions.go +++ b/go/vt/vttablet/tabletmanager/rpc_actions.go @@ -18,7 +18,6 @@ package tabletmanager import ( "fmt" - "regexp" "time" "vitess.io/vitess/go/vt/logutil" @@ -32,6 +31,7 @@ import ( tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) // This file contains the implementations of RPCTM methods. @@ -137,20 +137,10 @@ func (tm *TabletManager) RefreshState(ctx context.Context) error { // RunHealthCheck will manually run the health check on the tablet. func (tm *TabletManager) RunHealthCheck(ctx context.Context) { - tm.runHealthCheck() + tm.QueryServiceControl.BroadcastHealth() } // IgnoreHealthError sets the regexp for health check errors to ignore. func (tm *TabletManager) IgnoreHealthError(ctx context.Context, pattern string) error { - var expr *regexp.Regexp - if pattern != "" { - var err error - if expr, err = regexp.Compile(fmt.Sprintf("^%s$", pattern)); err != nil { - return err - } - } - tm.mutex.Lock() - tm._ignoreHealthErrorExpr = expr - tm.mutex.Unlock() - return nil + return vterrors.New(vtrpcpb.Code_INVALID_ARGUMENT, "deprecated") } diff --git a/go/vt/vttablet/tabletmanager/rpc_backup.go b/go/vt/vttablet/tabletmanager/rpc_backup.go index 3e497ac386f..cbb5f179f61 100644 --- a/go/vt/vttablet/tabletmanager/rpc_backup.go +++ b/go/vt/vttablet/tabletmanager/rpc_backup.go @@ -150,7 +150,7 @@ func (tm *TabletManager) RestoreFromBackup(ctx context.Context, logger logutil.L err = tm.restoreDataLocked(ctx, l, 0 /* waitForBackupInterval */, true /* deleteBeforeRestore */) // re-run health check to be sure to capture any replication delay - tm.runHealthCheckLocked() + tm.QueryServiceControl.BroadcastHealth() return err } diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index d64ec8077f9..906fcd316f6 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -626,7 +626,6 @@ func (tm *TabletManager) ReplicaWasRestarted(ctx context.Context, parent *topoda tablet.Type = topodatapb.TabletType_MASTER tablet.MasterTermStartTime = nil tm.updateState(ctx, tablet, "ReplicaWasRestarted") - tm.runHealthCheckLocked() return nil } diff --git a/go/vt/vttablet/tabletmanager/vreplication.go b/go/vt/vttablet/tabletmanager/rpc_vreplication.go similarity index 100% rename from go/vt/vttablet/tabletmanager/vreplication.go rename to go/vt/vttablet/tabletmanager/rpc_vreplication.go diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 51b8f7eee86..636511cc8e1 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -32,13 +32,11 @@ import ( "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl" - querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vttablet/tabletmanager/events" - "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tabletserver/rules" ) @@ -94,42 +92,10 @@ func (tm *TabletManager) loadBlacklistRules(ctx context.Context, tablet *topodat func (tm *TabletManager) lameduck(reason string) { log.Infof("TabletManager is entering lameduck, reason: %v", reason) tm.QueryServiceControl.EnterLameduck() - tm.broadcastHealth() time.Sleep(*gracePeriod) log.Infof("TabletManager is leaving lameduck") } -func (tm *TabletManager) broadcastHealth() { - // get the replication delays - tm.mutex.Lock() - replicationDelay := tm._replicationDelay - healthError := tm._healthy - healthyTime := tm._healthyTime - tm.mutex.Unlock() - - // send it to our observers - // FIXME(alainjobart,liguo) add CpuUsage - stats := &querypb.RealtimeStats{ - SecondsBehindMaster: uint32(replicationDelay.Seconds()), - } - stats.SecondsBehindMasterFilteredReplication, stats.BinlogPlayersCount = vreplication.StatusSummary() - stats.Qps = tm.QueryServiceControl.Stats().QPSRates.TotalRate() - if healthError != nil { - stats.HealthError = healthError.Error() - } else { - timeSinceLastCheck := time.Since(healthyTime) - if timeSinceLastCheck > healthCheckInterval*3 { - stats.HealthError = fmt.Sprintf("last health check is too old: %s > %s", timeSinceLastCheck, healthCheckInterval*3) - } - } - var ts int64 - terTime := tm.masterTermStartTime() - if !terTime.IsZero() { - ts = terTime.Unix() - } - go tm.QueryServiceControl.BroadcastHealth(ts, stats, healthCheckInterval*3) -} - // refreshTablet needs to be run after an action may have changed the current // state of the tablet. func (tm *TabletManager) refreshTablet(ctx context.Context, reason string) error { diff --git a/go/vt/vttablet/tabletmanager/state_change_test.go b/go/vt/vttablet/tabletmanager/state_change_test.go index 4365b97b66a..c762173a8b6 100644 --- a/go/vt/vttablet/tabletmanager/state_change_test.go +++ b/go/vt/vttablet/tabletmanager/state_change_test.go @@ -32,6 +32,8 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletservermock" ) +var tabletAlias = &topodatapb.TabletAlias{Cell: "cell1", Uid: 42} + func TestPublishState(t *testing.T) { defer func(saved time.Duration) { *publishRetryInterval = saved }(*publishRetryInterval) *publishRetryInterval = 1 * time.Millisecond @@ -104,8 +106,5 @@ func createTestTM(ctx context.Context, t *testing.T, preStart func(*TabletManage } err := tm.Start(tablet) require.NoError(t, err) - - tm.HealthReporter = &fakeHealthCheck{} - return tm } diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index fd75561168b..4e35448fb62 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -38,9 +38,6 @@ import ( "flag" "fmt" "math/rand" - "os" - "path" - "regexp" "sync" "time" @@ -56,7 +53,6 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/binlog" "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/health" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -134,9 +130,6 @@ type TabletManager struct { // replManager manages replication. replManager *replManager - // HealthReporter initiates healthchecks. - HealthReporter health.Reporter - // tabletAlias is saved away from tablet for read-only access tabletAlias *topodatapb.TabletAlias @@ -197,24 +190,6 @@ type TabletManager struct { // blacklisting. _blacklistedTables []string - // if the tm is healthy, this is nil. Otherwise it contains - // the reason we're not healthy. - _healthy error - - // this is the last time health check ran - _healthyTime time.Time - - // replication delay the last time we got it - _replicationDelay time.Duration - - // _ignoreHealthErrorExpr can be set by RPC to selectively disable certain - // healthcheck errors. It should only be accessed while holding actionMutex. - _ignoreHealthErrorExpr *regexp.Regexp - - // _replicationStopped remembers if we've been told to stop replicating. - // If it's nil, we'll try to check for the replicationStoppedFile. - _replicationStopped *bool - // _lockTablesConnection is used to get and release the table read locks to pause replication _lockTablesConnection *dbconnpool.DBConnection _lockTablesTimer *time.Timer @@ -227,13 +202,11 @@ type TabletManager struct { isPublishing bool } +var healthCheckInterval time.Duration + // InitConfig is a temp function to keep things working during the refactor. func InitConfig(config *tabletenv.TabletConfig) { healthCheckInterval = config.Healthcheck.IntervalSeconds.Get() - degradedThreshold = config.Healthcheck.DegradedThresholdSeconds.Get() - unhealthyThreshold = config.Healthcheck.UnhealthyThresholdSeconds.Get() - - enableReplicationReporter = config.ReplicationTracker.Mode == tabletenv.Polling } // BuildTabletFromInput builds a tablet record from input parameters. @@ -301,7 +274,6 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { log.Warningf("deprecated demote_master_type %v must match init_tablet_type %v", demoteType, tablet.Type) } tm.baseTabletType = tablet.Type - tm._healthy = fmt.Errorf("healthcheck not run yet") ctx, cancel := context.WithTimeout(tm.BatchCtx, *initTimeout) defer cancel() @@ -769,56 +741,6 @@ func (tm *TabletManager) DisallowQueryService() string { return tm._disallowQueryService } -func (tm *TabletManager) replicationStopped() bool { - tm.mutex.Lock() - defer tm.mutex.Unlock() - - // If we already know the value, don't bother checking the file. - if tm._replicationStopped != nil { - return *tm._replicationStopped - } - - // If there's no Cnf file, don't read state. - if tm.Cnf == nil { - return false - } - - // If the marker file exists, we're stopped. - // Treat any read error as if the file doesn't exist. - _, err := os.Stat(path.Join(tm.Cnf.TabletDir(), replicationStoppedFile)) - replicationStopped := err == nil - tm._replicationStopped = &replicationStopped - return replicationStopped -} - -func (tm *TabletManager) setReplicationStopped(stopped bool) { - tm.mutex.Lock() - defer tm.mutex.Unlock() - - tm._replicationStopped = &stopped - - // Make a best-effort attempt to persist the value across tablet restarts. - // We store a marker in the filesystem so it works regardless of whether - // mysqld is running, and so it's tied to this particular instance of the - // tablet data dir (the one that's paused at a known replication position). - if tm.Cnf == nil { - return - } - tabletDir := tm.Cnf.TabletDir() - if tabletDir == "" { - return - } - markerFile := path.Join(tabletDir, replicationStoppedFile) - if stopped { - file, err := os.Create(markerFile) - if err == nil { - file.Close() - } - } else { - os.Remove(markerFile) - } -} - func (tm *TabletManager) setServicesDesiredState(disallowQueryService string) { tm.mutex.Lock() tm._disallowQueryService = disallowQueryService diff --git a/go/vt/vttablet/tabletserver/controller.go b/go/vt/vttablet/tabletserver/controller.go index cc91b14bd5e..4ce32a9fe00 100644 --- a/go/vt/vttablet/tabletserver/controller.go +++ b/go/vt/vttablet/tabletserver/controller.go @@ -85,11 +85,7 @@ type Controller interface { SchemaEngine() *schema.Engine // BroadcastHealth sends the current health to all listeners - BroadcastHealth(terTimestamp int64, stats *querypb.RealtimeStats, maxCache time.Duration) - - // HeartbeatLag returns the current lag as calculated by the heartbeat - // package, if heartbeat is enabled. Otherwise returns 0. - HeartbeatLag() (time.Duration, error) + BroadcastHealth() // TopoServer returns the topo server. TopoServer() *topo.Server diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 1fdbbb34e6a..5201a8fc9a2 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -1350,16 +1350,10 @@ func (tsv *TabletServer) StreamHealth(ctx context.Context, callback func(*queryp } // BroadcastHealth will broadcast the current health to all listeners -func (tsv *TabletServer) BroadcastHealth(terTimestamp int64, stats *querypb.RealtimeStats, maxCache time.Duration) { +func (tsv *TabletServer) BroadcastHealth() { tsv.sm.Broadcast() } -// HeartbeatLag returns the current lag as calculated by the heartbeat -// package, if heartbeat is enabled. Otherwise returns 0. -func (tsv *TabletServer) HeartbeatLag() (time.Duration, error) { - return tsv.rt.Status() -} - // EnterLameduck causes tabletserver to enter the lameduck state. This // state causes health checks to fail, but the behavior of tabletserver // otherwise remains the same. Any subsequent calls to SetServingType will diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go index e4becf4f05c..0da4fba8315 100644 --- a/go/vt/vttablet/tabletservermock/controller.go +++ b/go/vt/vttablet/tabletservermock/controller.go @@ -202,22 +202,15 @@ func (tqsc *Controller) SchemaEngine() *schema.Engine { } // BroadcastHealth is part of the tabletserver.Controller interface -func (tqsc *Controller) BroadcastHealth(terTimestamp int64, stats *querypb.RealtimeStats, maxCache time.Duration) { +func (tqsc *Controller) BroadcastHealth() { tqsc.mu.Lock() defer tqsc.mu.Unlock() tqsc.BroadcastData <- &BroadcastData{ - TERTimestamp: terTimestamp, - RealtimeStats: *stats, - Serving: tqsc.queryServiceEnabled && (!tqsc.isInLameduck), + Serving: tqsc.queryServiceEnabled && (!tqsc.isInLameduck), } } -// HeartbeatLag is part of the tabletserver.Controller interface. -func (tqsc *Controller) HeartbeatLag() (time.Duration, error) { - return 0, nil -} - // TopoServer is part of the tabletserver.Controller interface. func (tqsc *Controller) TopoServer() *topo.Server { return tqsc.TS From e8dba28753839a345b85dc44643559cf1929f723 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 11 Jul 2020 14:24:03 -0700 Subject: [PATCH 026/100] vttablet: fix after rebase Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/rpc_replication.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index 906fcd316f6..7285fae9d9b 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -110,7 +110,7 @@ func (tm *TabletManager) stopIOThreadLocked(ctx context.Context) error { // Remember that we were told to stop, so we don't try to // restart ourselves (in replication_reporter). - tm.setReplicationStopped(true) + tm.replManager.setReplicationStopped(true) // Also tell Orchestrator we're stopped on purpose for some Vitess task. // Do this in the background, as it's best-effort. From 42fa9ad158610544f0482dd748ba2212b9b8337e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 12 Jul 2020 14:41:35 -0700 Subject: [PATCH 027/100] vttablet: move gracePeriod to TabletConfig Signed-off-by: Sugu Sougoumarane --- config/tablet/default.yaml | 6 +++++- go/vt/vttablet/tabletmanager/state_change.go | 8 ++++---- go/vt/vttablet/tabletmanager/tm_init.go | 1 + go/vt/vttablet/tabletserver/tabletenv/config.go | 16 +++++++++++++--- .../tabletserver/tabletenv/config_test.go | 8 ++++++++ go/vt/vttablet/tabletserver/tx_engine.go | 2 +- go/vt/vttablet/tabletserver/tx_engine_test.go | 4 ++-- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/config/tablet/default.yaml b/config/tablet/default.yaml index 5f4583aca36..ed9578582d0 100644 --- a/config/tablet/default.yaml +++ b/config/tablet/default.yaml @@ -83,6 +83,10 @@ healthcheck: degradedThresholdSeconds: 30 # degraded_threshold unhealthyThresholdSeconds: 7200 # unhealthy_threshold +gracePeriods: + transactionShutdownSeconds: 0 # transaction_shutdown_grace_period + transitionSeconds: 0 # serving_state_grace_period + replicationTracker: mode: disable # enable_replication_reporter heartbeatIntervalMilliseconds: 0 # heartbeat_enable, heartbeat_interval @@ -95,7 +99,7 @@ hotRowProtection: maxConcurrency: 5 # hot_row_protection_concurrent_transactions consolidator: enable|disable|notOnMaster # enable-consolidator, enable-consolidator-replicas -shutdownGracePeriodSeconds: 0 # transaction_shutdown_grace_period +transitionGracePeriodSeconds: 0 # serving_state_grace_period passthroughDML: false # queryserver-config-passthrough-dmls streamBufferSize: 32768 # queryserver-config-stream-buffer-size queryCacheSize: 5000 # queryserver-config-query-cache-size diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 636511cc8e1..7ae11252bd5 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -49,7 +49,7 @@ var ( // spare, or when being promoted to master). During this period, we expect // vtgate to gracefully redirect traffic elsewhere, before we begin actually // rejecting queries for that target type. - gracePeriod = flag.Duration("serving_state_grace_period", 0, "how long to pause after broadcasting health to vtgate, before enforcing a new serving state") + gracePeriod time.Duration publishRetryInterval = flag.Duration("publish_retry_interval", 30*time.Second, "how long vttablet waits to retry publishing the tablet record") ) @@ -92,7 +92,7 @@ func (tm *TabletManager) loadBlacklistRules(ctx context.Context, tablet *topodat func (tm *TabletManager) lameduck(reason string) { log.Infof("TabletManager is entering lameduck, reason: %v", reason) tm.QueryServiceControl.EnterLameduck() - time.Sleep(*gracePeriod) + time.Sleep(gracePeriod) log.Infof("TabletManager is leaving lameduck") } @@ -218,7 +218,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable // When promoting from replica to master, allow both master and replica // queries to be served during gracePeriod. if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, []topodatapb.TabletType{oldTablet.Type}); err == nil { - time.Sleep(*gracePeriod) + time.Sleep(gracePeriod) } else { log.Errorf("Can't start query service for MASTER+REPLICA mode: %v", err) } @@ -231,7 +231,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable // Query service should be stopped. if topo.IsSubjectToLameduck(oldTablet.Type) && newTablet.Type == topodatapb.TabletType_SPARE && - *gracePeriod > 0 { + gracePeriod > 0 { // When a non-MASTER serving type is going SPARE, // put query service in lameduck during gracePeriod. tm.lameduck(disallowQueryReason) diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 4e35448fb62..4cb73cb4b73 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -207,6 +207,7 @@ var healthCheckInterval time.Duration // InitConfig is a temp function to keep things working during the refactor. func InitConfig(config *tabletenv.TabletConfig) { healthCheckInterval = config.Healthcheck.IntervalSeconds.Get() + gracePeriod = config.GracePeriods.TransitionSeconds.Get() } // BuildTabletFromInput builds a tablet record from input parameters. diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index bae33bfa4ed..f68e8da3c0f 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -77,6 +77,7 @@ var ( healthCheckInterval time.Duration degradedThreshold time.Duration unhealthyThreshold time.Duration + transitionGracePeriod time.Duration enableReplicationReporter bool ) @@ -92,7 +93,7 @@ func init() { flag.IntVar(¤tConfig.MessagePostponeParallelism, "queryserver-config-message-postpone-cap", defaultConfig.MessagePostponeParallelism, "query server message postpone cap is the maximum number of messages that can be postponed at any given time. Set this number to substantially lower than transaction cap, so that the transaction pool isn't exhausted by the message subsystem.") flag.IntVar(&deprecatedFoundRowsPoolSize, "client-found-rows-pool-size", 0, "DEPRECATED: queryserver-config-transaction-cap will be used instead.") SecondsVar(¤tConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value") - SecondsVar(¤tConfig.ShutdownGracePeriodSeconds, "transaction_shutdown_grace_period", defaultConfig.ShutdownGracePeriodSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.") + SecondsVar(¤tConfig.GracePeriods.TransactionShutdownSeconds, "transaction_shutdown_grace_period", defaultConfig.GracePeriods.TransactionShutdownSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.") flag.IntVar(¤tConfig.Oltp.MaxRows, "queryserver-config-max-result-size", defaultConfig.Oltp.MaxRows, "query server max result size, maximum number of rows allowed to return from vttablet for non-streaming queries.") flag.IntVar(¤tConfig.Oltp.WarnRows, "queryserver-config-warn-result-size", defaultConfig.Oltp.WarnRows, "query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this") flag.IntVar(&deprecatedMaxDMLRows, "queryserver-config-max-dml-rows", 0, "query server max dml rows per statement, maximum number of rows allowed to return at a time for an update or delete with either 1) an equality where clauses on primary keys, or 2) a subselect statement. For update and delete statements in above two categories, vttablet will split the original query into multiple small queries based on this configuration value. ") @@ -150,6 +151,7 @@ func init() { flag.DurationVar(&healthCheckInterval, "health_check_interval", 20*time.Second, "Interval between health checks") flag.DurationVar(°radedThreshold, "degraded_threshold", 30*time.Second, "replication lag after which a replica is considered degraded (only used in status UI)") flag.DurationVar(&unhealthyThreshold, "unhealthy_threshold", 2*time.Hour, "replication lag after which a replica is considered unhealthy") + flag.DurationVar(&transitionGracePeriod, "serving_state_grace_period", 0, "how long to pause after broadcasting health to vtgate, before enforcing a new serving state") flag.BoolVar(&enableReplicationReporter, "enable_replication_reporter", false, "Use polling to track replication lag.") } @@ -195,6 +197,7 @@ func Init() { currentConfig.Healthcheck.IntervalSeconds.Set(healthCheckInterval) currentConfig.Healthcheck.DegradedThresholdSeconds.Set(degradedThreshold) currentConfig.Healthcheck.UnhealthyThresholdSeconds.Set(unhealthyThreshold) + currentConfig.GracePeriods.TransitionSeconds.Set(transitionGracePeriod) switch *streamlog.QueryLogFormat { case streamlog.QueryLogFormatText: @@ -223,13 +226,13 @@ type TabletConfig struct { Oltp OltpConfig `json:"oltp,omitempty"` HotRowProtection HotRowProtectionConfig `json:"hotRowProtection,omitempty"` - Healthcheck HealthcheckConfig `json:"healthcheck,omitempty"` + Healthcheck HealthcheckConfig `json:"healthcheck,omitempty"` + GracePeriods GracePeriodsConfig `json:"gracePeriods,omitempty"` ReplicationTracker ReplicationTrackerConfig `json:"replicationTracker,omitempty"` // Consolidator can be enable, disable, or notOnMaster. Default is enable. Consolidator string `json:"consolidator,omitempty"` - ShutdownGracePeriodSeconds Seconds `json:"shutdownGracePeriodSeconds,omitempty"` PassthroughDML bool `json:"passthroughDML,omitempty"` StreamBufferSize int `json:"streamBufferSize,omitempty"` QueryCacheSize int `json:"queryCacheSize,omitempty"` @@ -291,6 +294,13 @@ type HealthcheckConfig struct { UnhealthyThresholdSeconds Seconds `json:"unhealthyThresholdSeconds,omitempty"` } +// GracePeriodsConfig contains various grace periods. +// TODO(sougou): move lameduck here? +type GracePeriodsConfig struct { + TransactionShutdownSeconds Seconds `json:"transactionShutdownSeconds,omitempty"` + TransitionSeconds Seconds `json:"transitionSeconds,omitempty"` +} + // ReplicationTrackerConfig contains the config for the replication tracker. type ReplicationTrackerConfig struct { // Mode can be disable, polling or heartbeat. Default is disable. diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index 81bee701980..bcb5e684a30 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -63,6 +63,7 @@ func TestConfigParse(t *testing.T) { repl: password: '****' socket: a +gracePeriods: {} healthcheck: {} hotRowProtection: {} olapReadPool: {} @@ -106,6 +107,7 @@ func TestDefaultConfig(t *testing.T) { require.NoError(t, err) want := `cacheResultFields: true consolidator: enable +gracePeriods: {} healthcheck: degradedThresholdSeconds: 30 intervalSeconds: 20 @@ -309,4 +311,10 @@ func TestFlags(t *testing.T) { Init() want.Healthcheck.UnhealthyThresholdSeconds = 3 assert.Equal(t, want, currentConfig) + + transitionGracePeriod = 4 * time.Second + currentConfig.GracePeriods.TransitionSeconds = 0 + Init() + want.GracePeriods.TransitionSeconds = 4 + assert.Equal(t, want, currentConfig) } diff --git a/go/vt/vttablet/tabletserver/tx_engine.go b/go/vt/vttablet/tabletserver/tx_engine.go index 62c0545ad3b..9bd80a3c40f 100644 --- a/go/vt/vttablet/tabletserver/tx_engine.go +++ b/go/vt/vttablet/tabletserver/tx_engine.go @@ -107,7 +107,7 @@ func NewTxEngine(env tabletenv.Env) *TxEngine { config := env.Config() te := &TxEngine{ env: env, - shutdownGracePeriod: config.ShutdownGracePeriodSeconds.Get(), + shutdownGracePeriod: config.GracePeriods.TransactionShutdownSeconds.Get(), reservedConnStats: env.Exporter().NewTimings("ReservedConnections", "Reserved connections stats", "operation"), } limiter := txlimiter.New(env) diff --git a/go/vt/vttablet/tabletserver/tx_engine_test.go b/go/vt/vttablet/tabletserver/tx_engine_test.go index 2c3f6e8e974..c03bdb60602 100644 --- a/go/vt/vttablet/tabletserver/tx_engine_test.go +++ b/go/vt/vttablet/tabletserver/tx_engine_test.go @@ -47,7 +47,7 @@ func TestTxEngineClose(t *testing.T) { config.DB = newDBConfigs(db) config.TxPool.Size = 10 config.Oltp.TxTimeoutSeconds = 0.1 - config.ShutdownGracePeriodSeconds = 0 + config.GracePeriods.TransactionShutdownSeconds = 0 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) // Normal close. @@ -515,7 +515,7 @@ func setupTxEngine(db *fakesqldb.DB) *TxEngine { config.DB = newDBConfigs(db) config.TxPool.Size = 10 config.Oltp.TxTimeoutSeconds = 0.1 - config.ShutdownGracePeriodSeconds = 0 + config.GracePeriods.TransactionShutdownSeconds = 0 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) return te } From c3bb83e8a2eea5fe01ddf4cc8c06b541a36af6ea Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 12 Jul 2020 16:08:46 -0700 Subject: [PATCH 028/100] vttablet: SetServingType handles alsoAllow Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/endtoend/framework/client.go | 2 +- .../vttablet/tabletmanager/rpc_replication.go | 6 +-- go/vt/vttablet/tabletmanager/state_change.go | 16 +----- go/vt/vttablet/tabletserver/controller.go | 2 +- go/vt/vttablet/tabletserver/state_manager.go | 52 ++++++++++++++----- .../tabletserver/state_manager_test.go | 47 ++++++----------- go/vt/vttablet/tabletserver/tabletserver.go | 11 ++-- .../tabletserver/tabletserver_test.go | 10 ++-- go/vt/vttablet/tabletservermock/controller.go | 14 ++--- 9 files changed, 76 insertions(+), 84 deletions(-) diff --git a/go/vt/vttablet/endtoend/framework/client.go b/go/vt/vttablet/endtoend/framework/client.go index 7f8e1a0a205..b68f7d346ee 100644 --- a/go/vt/vttablet/endtoend/framework/client.go +++ b/go/vt/vttablet/endtoend/framework/client.go @@ -162,7 +162,7 @@ func (client *QueryClient) ReadTransaction(dtid string) (*querypb.TransactionMet // SetServingType is for testing transitions. // It currently supports only master->replica and back. func (client *QueryClient) SetServingType(tabletType topodatapb.TabletType) error { - _, err := client.server.SetServingType(tabletType, time.Time{}, true, nil) + err := client.server.SetServingType(tabletType, time.Time{}, true) return err } diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index 7285fae9d9b..e958ec4f049 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -381,12 +381,12 @@ func (tm *TabletManager) demoteMaster(ctx context.Context, revertPartialFailure // considered successful. If we are already not serving, this will be // idempotent. log.Infof("DemoteMaster disabling query service") - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), false, nil); err != nil { + if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), false); err != nil { return nil, vterrors.Wrap(err, "SetServingType(serving=false) failed") } defer func() { if finalErr != nil && revertPartialFailure && wasServing { - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true, nil); err != nil { + if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true); err != nil { log.Warningf("SetServingType(serving=true) failed during revert: %v", err) } } @@ -460,7 +460,7 @@ func (tm *TabletManager) UndoDemoteMaster(ctx context.Context) error { // Update serving graph tablet := tm.Tablet() log.Infof("UndoDemoteMaster re-enabling query service") - if _ /* state changed */, err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true, nil); err != nil { + if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true); err != nil { return vterrors.Wrap(err, "SetServingType(serving=true) failed") } diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go index 7ae11252bd5..ac4b0aa6c4a 100644 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ b/go/vt/vttablet/tabletmanager/state_change.go @@ -212,19 +212,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable tm.replManager.SetTabletType(newTablet.Type) if allowQuery { - // Query service should be running. - if oldTablet.Type == topodatapb.TabletType_REPLICA && - newTablet.Type == topodatapb.TabletType_MASTER { - // When promoting from replica to master, allow both master and replica - // queries to be served during gracePeriod. - if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, []topodatapb.TabletType{oldTablet.Type}); err == nil { - time.Sleep(gracePeriod) - } else { - log.Errorf("Can't start query service for MASTER+REPLICA mode: %v", err) - } - } - - if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true, nil); err != nil { + if err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true); err != nil { log.Errorf("Cannot start query service: %v", err) } } else { @@ -238,7 +226,7 @@ func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTable } log.Infof("Disabling query service on type change, reason: %v", disallowQueryReason) - if _, err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, false, nil); err != nil { + if err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, false); err != nil { log.Errorf("SetServingType(serving=false) failed: %v", err) } } diff --git a/go/vt/vttablet/tabletserver/controller.go b/go/vt/vttablet/tabletserver/controller.go index 4ce32a9fe00..3e8d32f6e8e 100644 --- a/go/vt/vttablet/tabletserver/controller.go +++ b/go/vt/vttablet/tabletserver/controller.go @@ -52,7 +52,7 @@ type Controller interface { // SetServingType transitions the query service to the required serving type. // Returns true if the state of QueryService or the tablet type changed. - SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool, alsoAllow []topodatapb.TabletType) (bool, error) + SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) error // EnterLameduck causes tabletserver to enter the lameduck state. EnterLameduck() diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 9abf42c49f7..ba4126255df 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -75,9 +75,8 @@ type stateManager struct { lameduck bool target querypb.Target retrying bool - // TODO(sougou): deprecate alsoAllow - alsoAllow []topodatapb.TabletType - terTimestamp time.Time + alsoAllow []topodatapb.TabletType + terTimestamp time.Time requests sync.WaitGroup @@ -105,8 +104,9 @@ type stateManager struct { // doesn't get spammed. checkMySQLThrottler *sync2.Semaphore - timebombDuration time.Duration - unhealthyThreshold time.Duration + timebombDuration time.Duration + unhealthyThreshold time.Duration + transitionGracePeriod time.Duration } type ( @@ -155,6 +155,7 @@ func (sm *stateManager) Init(env tabletenv.Env, target querypb.Target) { sm.timebombDuration = env.Config().OltpReadPool.TimeoutSeconds.Get() * 10 sm.hcticks = timer.NewTimer(env.Config().Healthcheck.IntervalSeconds.Get()) sm.unhealthyThreshold = env.Config().Healthcheck.UnhealthyThresholdSeconds.Get() + sm.transitionGracePeriod = env.Config().GracePeriods.TransitionSeconds.Get() } // SetServingType changes the state to the specified settings. @@ -164,7 +165,7 @@ func (sm *stateManager) Init(env tabletenv.Env, target querypb.Target) { // be honored. // If sm is already in the requested state, it returns stateChanged as // false. -func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { +func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState) error { defer sm.ExitLameduck() // Start is idempotent. @@ -176,24 +177,23 @@ func (sm *stateManager) SetServingType(tabletType topodatapb.TabletType, terTime } log.Infof("Starting transition to %v %v, timestamp: %v", tabletType, stateName(state), terTimestamp) - if sm.mustTransition(tabletType, terTimestamp, state, alsoAllow) { - return true, sm.execTransition(tabletType, state) + if sm.mustTransition(tabletType, terTimestamp, state) { + return sm.execTransition(tabletType, state) } - return false, nil + return nil } // mustTransition returns true if the requested state does not match the current // state. If so, it acquires the semaphore and returns true. If a transition is // already in progress, it waits. If the desired state is already reached, it // returns false without acquiring the semaphore. -func (sm *stateManager) mustTransition(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState, alsoAllow []topodatapb.TabletType) bool { +func (sm *stateManager) mustTransition(tabletType topodatapb.TabletType, terTimestamp time.Time, state servingState) bool { sm.transitioning.Acquire() sm.mu.Lock() defer sm.mu.Unlock() sm.wantTabletType = tabletType sm.wantState = state - sm.alsoAllow = alsoAllow sm.terTimestamp = terTimestamp if sm.target.TabletType == tabletType && sm.state == state { sm.transitioning.Release() @@ -299,7 +299,7 @@ func (sm *stateManager) StopService() { log.Info("Stopping TabletServer") // Stop replica tracking because StopService is used by all tests. sm.hcticks.Stop() - sm.SetServingType(sm.Target().TabletType, time.Time{}, StateNotConnected, nil) + sm.SetServingType(sm.Target().TabletType, time.Time{}, StateNotConnected) } // StartRequest validates the current state and target and registers @@ -503,7 +503,8 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving if tabletType == topodatapb.TabletType_UNKNOWN { tabletType = sm.wantTabletType } - log.Infof("TabletServer transition: %v -> %v, %s -> %s", sm.target.TabletType, tabletType, stateName(sm.state), stateName(state)) + log.Infof("TabletServer transition: %v(%v) -> %v(%v)", sm.target.TabletType, stateName(sm.state), tabletType, stateName(state)) + sm.handleGracePeriod(tabletType) sm.target.TabletType = tabletType if sm.state == StateNotConnected { // If we're transitioning out of StateNotConnected, we have @@ -515,6 +516,31 @@ func (sm *stateManager) setState(tabletType topodatapb.TabletType, state serving go sm.hcticks.Trigger() } +func (sm *stateManager) handleGracePeriod(tabletType topodatapb.TabletType) { + if tabletType != topodatapb.TabletType_MASTER { + // We allow serving of previous type only for a master transition. + sm.alsoAllow = nil + return + } + + if tabletType == topodatapb.TabletType_MASTER && + sm.target.TabletType != topodatapb.TabletType_MASTER && + sm.transitionGracePeriod != 0 { + + sm.alsoAllow = []topodatapb.TabletType{sm.target.TabletType} + // This is not a perfect solution because multiple back and forth + // transitions will launch multiple of these goroutines. But the + // system will eventually converge. + go func() { + time.Sleep(sm.transitionGracePeriod) + + sm.mu.Lock() + defer sm.mu.Unlock() + sm.alsoAllow = nil + }() + } +} + // Broadcast fetches the replication status and broadcasts // the state to all subscribed. func (sm *stateManager) Broadcast() { diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index d290b8d9f32..b1ff190ab7d 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -61,9 +61,8 @@ func TestStateManagerStateByName(t *testing.T) { func TestStateManagerServeMaster(t *testing.T) { sm := newTestStateManager(t) sm.EnterLameduck() - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) assert.Equal(t, false, sm.lameduck) assert.Equal(t, testNow, sm.terTimestamp) @@ -89,9 +88,8 @@ func TestStateManagerServeMaster(t *testing.T) { func TestStateManagerServeNonMaster(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.tracker, testStateClosed) @@ -111,9 +109,8 @@ func TestStateManagerServeNonMaster(t *testing.T) { func TestStateManagerUnserveMaster(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateNotServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateNotServing) require.NoError(t, err) - assert.True(t, stateChanged) verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.te, testStateClosed) @@ -135,9 +132,8 @@ func TestStateManagerUnserveMaster(t *testing.T) { func TestStateManagerUnserveNonmaster(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotServing, nil) + err := sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotServing) require.NoError(t, err) - assert.True(t, stateChanged) verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.te, testStateClosed) @@ -160,9 +156,8 @@ func TestStateManagerUnserveNonmaster(t *testing.T) { func TestStateManagerClose(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotConnected, nil) + err := sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotConnected) require.NoError(t, err) - assert.True(t, stateChanged) verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.te, testStateClosed) @@ -182,9 +177,8 @@ func TestStateManagerClose(t *testing.T) { func TestStateManagerStopService(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) assert.Equal(t, topodatapb.TabletType_REPLICA, sm.target.TabletType) assert.Equal(t, StateServing, sm.state) @@ -209,9 +203,8 @@ func (te *testWatcher) Close() { go func() { defer te.wg.Done() - stateChanged, err := te.sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotServing, nil) + err := te.sm.SetServingType(topodatapb.TabletType_RDONLY, testNow, StateNotServing) assert.NoError(te.t, err) - assert.True(te.t, stateChanged) }() } @@ -222,9 +215,8 @@ func TestStateManagerSetServingTypeRace(t *testing.T) { sm: sm, } sm.watcher = te - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) // Ensure the next call waits and then succeeds. te.wg.Wait() @@ -236,13 +228,11 @@ func TestStateManagerSetServingTypeRace(t *testing.T) { func TestStateManagerSetServingTypeNoChange(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) - stateChanged, err = sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, nil) + err = sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing) require.NoError(t, err) - assert.False(t, stateChanged) verifySubcomponent(t, 1, sm.messager, testStateClosed) verifySubcomponent(t, 2, sm.tracker, testStateClosed) @@ -267,9 +257,8 @@ func TestStateManagerTransitionFailRetry(t *testing.T) { sm := newTestStateManager(t) sm.qe.(*testQueryEngine).failMySQL = true - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing) require.Error(t, err) - assert.True(t, stateChanged) // Calling retryTransition while retrying should be a no-op. sm.retryTransition("") @@ -298,16 +287,14 @@ func TestStateManagerTransitionFailRetry(t *testing.T) { func TestStateManagerNotConnectedType(t *testing.T) { sm := newTestStateManager(t) sm.EnterLameduck() - stateChanged, err := sm.SetServingType(topodatapb.TabletType_RESTORE, testNow, StateNotServing, nil) + err := sm.SetServingType(topodatapb.TabletType_RESTORE, testNow, StateNotServing) require.NoError(t, err) - assert.True(t, stateChanged) assert.Equal(t, topodatapb.TabletType_RESTORE, sm.target.TabletType) assert.Equal(t, StateNotConnected, sm.state) - stateChanged, err = sm.SetServingType(topodatapb.TabletType_BACKUP, testNow, StateNotServing, nil) + err = sm.SetServingType(topodatapb.TabletType_BACKUP, testNow, StateNotServing) require.NoError(t, err) - assert.True(t, stateChanged) assert.Equal(t, topodatapb.TabletType_BACKUP, sm.target.TabletType) assert.Equal(t, StateNotConnected, sm.state) @@ -319,9 +306,8 @@ func TestStateManagerCheckMySQL(t *testing.T) { sm := newTestStateManager(t) - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) sm.qe.(*testQueryEngine).failMySQL = true order.Set(0) @@ -430,7 +416,7 @@ func TestStateManagerWaitForRequests(t *testing.T) { sm.timebombDuration = 10 * time.Second sm.replHealthy = true - _, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing) require.NoError(t, err) err = sm.StartRequest(ctx, target, false) @@ -481,9 +467,8 @@ func TestStateManagerNotify(t *testing.T) { gotServing = serving ch <- struct{}{} } - stateChanged, err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing, nil) + err := sm.SetServingType(topodatapb.TabletType_MASTER, testNow, StateServing) require.NoError(t, err) - assert.True(t, stateChanged) assert.Equal(t, topodatapb.TabletType_MASTER, sm.target.TabletType) assert.Equal(t, StateServing, sm.state) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 5201a8fc9a2..35be5f38917 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -294,16 +294,14 @@ func (tsv *TabletServer) InitACL(tableACLConfigFile string, enforceTableACLConfi } // SetServingType changes the serving type of the tabletserver. It starts or -// stops internal services as deemed necessary. The tabletType determines the -// primary serving type, while alsoAllow specifies other tablet types that -// should also be honored for serving. +// stops internal services as deemed necessary. // Returns true if the state of QueryService or the tablet type changed. -func (tsv *TabletServer) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool, alsoAllow []topodatapb.TabletType) (stateChanged bool, err error) { +func (tsv *TabletServer) SetServingType(tabletType topodatapb.TabletType, terTimestamp time.Time, serving bool) error { state := StateNotServing if serving { state = StateServing } - return tsv.sm.SetServingType(tabletType, terTimestamp, state, alsoAllow) + return tsv.sm.SetServingType(tabletType, terTimestamp, state) } // StartService is a convenience function for InitDBConfig->SetServingType @@ -313,8 +311,7 @@ func (tsv *TabletServer) StartService(target querypb.Target, dbcfgs *dbconfigs.D return err } // StartService is only used for testing. So, we cheat by aggressively setting replication to healthy. - _, err := tsv.sm.SetServingType(target.TabletType, time.Time{}, StateServing, nil) - return err + return tsv.sm.SetServingType(target.TabletType, time.Time{}, StateServing) } // StopService shuts down the tabletserver to the uninitialized state. diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index a01b614f072..b9a2ac9aeb8 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -62,7 +62,7 @@ func TestBeginOnReplica(t *testing.T) { db.AddQueryPattern(".*", &sqltypes.Result{}) target := querypb.Target{TabletType: topodatapb.TabletType_REPLICA} - _, err := tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, nil) + err := tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true) require.NoError(t, err) options := querypb.ExecuteOptions{ @@ -103,7 +103,7 @@ func TestTabletServerMasterToReplica(t *testing.T) { require.NoError(t, err) ch := make(chan bool) go func() { - tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, []topodatapb.TabletType{topodatapb.TabletType_MASTER}) + tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true) ch <- true }() @@ -126,13 +126,13 @@ func TestTabletServerRedoLogIsKeptBetweenRestarts(t *testing.T) { _, tsv, db := newTestTxExecutor(t) defer tsv.StopService() defer db.Close() - tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, nil) + tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true) turnOnTxEngine := func() { - tsv.SetServingType(topodatapb.TabletType_MASTER, time.Time{}, true, nil) + tsv.SetServingType(topodatapb.TabletType_MASTER, time.Time{}, true) } turnOffTxEngine := func() { - tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true, nil) + tsv.SetServingType(topodatapb.TabletType_REPLICA, time.Time{}, true) } tpc := tsv.te.twoPC diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go index 0da4fba8315..633847d7f88 100644 --- a/go/vt/vttablet/tabletservermock/controller.go +++ b/go/vt/vttablet/tabletservermock/controller.go @@ -133,24 +133,20 @@ func (tqsc *Controller) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.DB } // SetServingType is part of the tabletserver.Controller interface -func (tqsc *Controller) SetServingType(tabletType topodatapb.TabletType, terTime time.Time, serving bool, alsoAllow []topodatapb.TabletType) (bool, error) { +func (tqsc *Controller) SetServingType(tabletType topodatapb.TabletType, terTime time.Time, serving bool) error { tqsc.mu.Lock() defer tqsc.mu.Unlock() - stateChanged := false if tqsc.SetServingTypeError == nil { - stateChanged = tqsc.queryServiceEnabled != serving || tqsc.CurrentTarget.TabletType != tabletType tqsc.CurrentTarget.TabletType = tabletType tqsc.queryServiceEnabled = serving } - if stateChanged { - tqsc.StateChanges <- &StateChange{ - Serving: serving, - TabletType: tabletType, - } + tqsc.StateChanges <- &StateChange{ + Serving: serving, + TabletType: tabletType, } tqsc.isInLameduck = false - return stateChanged, tqsc.SetServingTypeError + return tqsc.SetServingTypeError } // IsServing is part of the tabletserver.Controller interface From 8e237663c6975fbe6a72bf62c088a87147d7811d Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 12 Jul 2020 19:32:33 -0700 Subject: [PATCH 029/100] vttablet: tmState initial cut Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/rpc_replication.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index e958ec4f049..1e2ab8d40b8 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -623,7 +623,7 @@ func (tm *TabletManager) ReplicaWasRestarted(ctx context.Context, parent *topoda if tablet.Type != topodatapb.TabletType_MASTER { return nil } - tablet.Type = topodatapb.TabletType_MASTER + tablet.Type = topodatapb.TabletType_REPLICA tablet.MasterTermStartTime = nil tm.updateState(ctx, tablet, "ReplicaWasRestarted") return nil From 453497f7038a8218be4bd0e5446a0428962cb100 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sun, 12 Jul 2020 21:25:47 -0700 Subject: [PATCH 030/100] vttablet: use tmState Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/replmanager.go | 13 +- go/vt/vttablet/tabletmanager/restore.go | 20 +- go/vt/vttablet/tabletmanager/rpc_actions.go | 22 +- .../vttablet/tabletmanager/rpc_replication.go | 17 +- go/vt/vttablet/tabletmanager/rpc_server.go | 7 - go/vt/vttablet/tabletmanager/shard_sync.go | 11 +- go/vt/vttablet/tabletmanager/state_change.go | 314 --------------- .../tabletmanager/state_change_test.go | 20 +- go/vt/vttablet/tabletmanager/tm_init.go | 145 ++----- go/vt/vttablet/tabletmanager/tm_init_test.go | 4 +- go/vt/vttablet/tabletmanager/tm_state.go | 356 ++++++++++++++++++ 11 files changed, 439 insertions(+), 490 deletions(-) delete mode 100644 go/vt/vttablet/tabletmanager/state_change.go create mode 100644 go/vt/vttablet/tabletmanager/tm_state.go diff --git a/go/vt/vttablet/tabletmanager/replmanager.go b/go/vt/vttablet/tabletmanager/replmanager.go index 52b30988461..a48b33d66ea 100644 --- a/go/vt/vttablet/tabletmanager/replmanager.go +++ b/go/vt/vttablet/tabletmanager/replmanager.go @@ -80,8 +80,6 @@ func (rm *replManager) SetTabletType(tabletType topodatapb.TabletType) { if rm.ticks.Running() { return } - // Do an explicit check and then start the timer. - rm.check() rm.ticks.Start(rm.check) } @@ -92,13 +90,20 @@ func (rm *replManager) check() { return } } else { - // TODO(sougou): this was ported from previous behavior. - // Need to check if this is should be &&. + // If only one of the threads is stopped, it's probably + // intentional. So, we don't repair replication. if status.SQLThreadRunning || status.IOThreadRunning { return } } + // We need to obtain the action lock if we're going to fix + // replication + if err := rm.tm.lock(rm.ctx); err != nil { + return + } + defer rm.tm.unlock() + if rm.firstFailure { log.Infof("Replication is stopped, reconnecting to master.") } diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index ca7cc1fc23e..0b5a2695432 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -76,10 +76,14 @@ func (tm *TabletManager) RestoreData(ctx context.Context, logger logutil.Logger, } func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool) error { - var originalType topodatapb.TabletType + // If we're called during init, tmState may not be open yet. + tm.tmState.Open(ctx) + tablet := tm.Tablet() - originalType, tablet.Type = tablet.Type, topodatapb.TabletType_RESTORE - tm.updateState(ctx, tablet, "restore from backup") + originalType := tablet.Type + if err := tm.tmState.ChangeTabletType(ctx, topodatapb.TabletType_RESTORE); err != nil { + return err + } // Try to restore. Depending on the reason for failure, we may be ok. // If we're not ok, return an error and the tm will log.Fatalf, @@ -167,8 +171,9 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L // alter replication here. default: // If anything failed, we should reset the original tablet type - tablet.Type = originalType - tm.updateState(ctx, tablet, "failed for restore from backup") + if err := tm.tmState.ChangeTabletType(ctx, originalType); err != nil { + log.Errorf("Could not change back to original tablet type %v: %v", originalType, err) + } return vterrors.Wrap(err, "Can't restore backup") } @@ -182,10 +187,7 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L } // Change type back to original type if we're ok to serve. - tablet.Type = originalType - tm.updateState(ctx, tablet, "after restore from backup") - - return nil + return tm.tmState.ChangeTabletType(ctx, originalType) } // restoreToTimeFromBinlog restores to the snapshot time of the keyspace diff --git a/go/vt/vttablet/tabletmanager/rpc_actions.go b/go/vt/vttablet/tabletmanager/rpc_actions.go index 2be85e5808a..0992d63b4cf 100644 --- a/go/vt/vttablet/tabletmanager/rpc_actions.go +++ b/go/vt/vttablet/tabletmanager/rpc_actions.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/vterrors" "golang.org/x/net/context" @@ -75,25 +74,10 @@ func (tm *TabletManager) changeTypeLocked(ctx context.Context, tabletType topoda return fmt.Errorf("Tablet: %v, is already drained", tm.tabletAlias) } - tablet := tm.Tablet() - tablet.Type = tabletType - // If we have been told we're master, set master term start time to Now - // and save it topo immediately. - if tabletType == topodatapb.TabletType_MASTER { - tablet.MasterTermStartTime = logutil.TimeToProto(time.Now()) - - // change our type in the topology, and set masterTermStartTime on tablet record if applicable - _, err := topotools.ChangeType(ctx, tm.TopoServer, tm.tabletAlias, tabletType, tablet.MasterTermStartTime) - if err != nil { - return err - } - } else { - tablet.MasterTermStartTime = nil + if err := tm.tmState.ChangeTabletType(ctx, tabletType); err != nil { + return err } - // updateState will invoke broadcastHealth if needed. - tm.updateState(ctx, tablet, "ChangeType") - // Let's see if we need to fix semi-sync acking. if err := tm.fixSemiSyncAndReplication(tm.Tablet().Type); err != nil { return vterrors.Wrap(err, "fixSemiSyncAndReplication failed, may not ack correctly") @@ -132,7 +116,7 @@ func (tm *TabletManager) RefreshState(ctx context.Context) error { } defer tm.unlock() - return tm.refreshTablet(ctx, "RefreshState") + return tm.tmState.RefreshFromTopo(ctx) } // RunHealthCheck will manually run the health check on the tablet. diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index 1e2ab8d40b8..aefc42b266e 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -381,12 +381,12 @@ func (tm *TabletManager) demoteMaster(ctx context.Context, revertPartialFailure // considered successful. If we are already not serving, this will be // idempotent. log.Infof("DemoteMaster disabling query service") - if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), false); err != nil { + if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.tmState.MasterTermStartTime(), false); err != nil { return nil, vterrors.Wrap(err, "SetServingType(serving=false) failed") } defer func() { if finalErr != nil && revertPartialFailure && wasServing { - if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true); err != nil { + if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.tmState.MasterTermStartTime(), true); err != nil { log.Warningf("SetServingType(serving=true) failed during revert: %v", err) } } @@ -460,7 +460,7 @@ func (tm *TabletManager) UndoDemoteMaster(ctx context.Context) error { // Update serving graph tablet := tm.Tablet() log.Infof("UndoDemoteMaster re-enabling query service") - if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.masterTermStartTime(), true); err != nil { + if err := tm.QueryServiceControl.SetServingType(tablet.Type, tm.tmState.MasterTermStartTime(), true); err != nil { return vterrors.Wrap(err, "SetServingType(serving=true) failed") } @@ -521,9 +521,9 @@ func (tm *TabletManager) setMasterLocked(ctx context.Context, parentAlias *topod // unintentionally change the type of RDONLY tablets tablet := tm.Tablet() if tablet.Type == topodatapb.TabletType_MASTER { - tablet.Type = topodatapb.TabletType_REPLICA - tablet.MasterTermStartTime = nil - tm.updateState(ctx, tablet, "setMasterLocked") + if err := tm.tmState.ChangeTabletType(ctx, topodatapb.TabletType_REPLICA); err != nil { + return err + } } // See if we were replicating at all, and should be replicating. @@ -623,10 +623,7 @@ func (tm *TabletManager) ReplicaWasRestarted(ctx context.Context, parent *topoda if tablet.Type != topodatapb.TabletType_MASTER { return nil } - tablet.Type = topodatapb.TabletType_REPLICA - tablet.MasterTermStartTime = nil - tm.updateState(ctx, tablet, "ReplicaWasRestarted") - return nil + return tm.tmState.ChangeTabletType(ctx, topodatapb.TabletType_REPLICA) } // StopReplicationAndGetStatus stops MySQL replication, and returns the diff --git a/go/vt/vttablet/tabletmanager/rpc_server.go b/go/vt/vttablet/tabletmanager/rpc_server.go index 25e300c1b73..d451fc778c9 100644 --- a/go/vt/vttablet/tabletmanager/rpc_server.go +++ b/go/vt/vttablet/tabletmanager/rpc_server.go @@ -58,13 +58,6 @@ func (tm *TabletManager) unlock() { tm.actionMutex.Unlock() } -// checkLock checks we have locked the actionMutex. -func (tm *TabletManager) checkLock() { - if !tm.actionMutexLocked { - panic("programming error: this action should have taken the actionMutex") - } -} - // HandleRPCPanic is part of the RPCTM interface. func (tm *TabletManager) HandleRPCPanic(ctx context.Context, name string, args, reply interface{}, verbose bool, err *error) { // panic handling diff --git a/go/vt/vttablet/tabletmanager/shard_sync.go b/go/vt/vttablet/tabletmanager/shard_sync.go index 5748c10b929..6b47e0c977d 100644 --- a/go/vt/vttablet/tabletmanager/shard_sync.go +++ b/go/vt/vttablet/tabletmanager/shard_sync.go @@ -96,7 +96,7 @@ func (tm *TabletManager) shardSyncLoop(ctx context.Context, notifyChan <-chan st switch tablet.Type { case topodatapb.TabletType_MASTER: // If we think we're master, check if we need to update the shard record. - masterAlias, err := syncShardMaster(ctx, tm.TopoServer, tablet, tm.masterTermStartTime()) + masterAlias, err := syncShardMaster(ctx, tm.TopoServer, tablet, tm.tmState.MasterTermStartTime()) if err != nil { log.Errorf("Failed to sync shard record: %v", err) // Start retry timer and go back to sleep. @@ -273,12 +273,3 @@ func (tm *TabletManager) notifyShardSync() { default: } } - -func (tm *TabletManager) masterTermStartTime() time.Time { - tm.pubMu.Lock() - defer tm.pubMu.Unlock() - if tm.tablet == nil { - return time.Time{} - } - return logutil.ProtoToTime(tm.tablet.MasterTermStartTime) -} diff --git a/go/vt/vttablet/tabletmanager/state_change.go b/go/vt/vttablet/tabletmanager/state_change.go deleted file mode 100644 index ac4b0aa6c4a..00000000000 --- a/go/vt/vttablet/tabletmanager/state_change.go +++ /dev/null @@ -1,314 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tabletmanager - -// This file handles the tm state changes. - -import ( - "flag" - "fmt" - "strings" - "time" - - "github.com/golang/protobuf/proto" - "golang.org/x/net/context" - "vitess.io/vitess/go/event" - "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/vt/key" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/mysqlctl" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools" - "vitess.io/vitess/go/vt/vttablet/tabletmanager/events" - "vitess.io/vitess/go/vt/vttablet/tabletserver/rules" -) - -var ( - // constants for this module - historyLength = 16 - - // gracePeriod is the amount of time we pause after broadcasting to vtgate - // that we're going to stop serving a particular target type (e.g. when going - // spare, or when being promoted to master). During this period, we expect - // vtgate to gracefully redirect traffic elsewhere, before we begin actually - // rejecting queries for that target type. - gracePeriod time.Duration - - publishRetryInterval = flag.Duration("publish_retry_interval", 30*time.Second, "how long vttablet waits to retry publishing the tablet record") -) - -// Query rules from blacklist -const blacklistQueryRules string = "BlacklistQueryRules" - -// loadBlacklistRules loads and builds the blacklist query rules -func (tm *TabletManager) loadBlacklistRules(ctx context.Context, tablet *topodatapb.Tablet, blacklistedTables []string) (err error) { - blacklistRules := rules.New() - if len(blacklistedTables) > 0 { - // tables, first resolve wildcards - tables, err := mysqlctl.ResolveTables(ctx, tm.MysqlDaemon, topoproto.TabletDbName(tablet), blacklistedTables) - if err != nil { - return err - } - - // Verify that at least one table matches the wildcards, so - // that we don't add a rule to blacklist all tables - if len(tables) > 0 { - log.Infof("Blacklisting tables %v", strings.Join(tables, ", ")) - qr := rules.NewQueryRule("enforce blacklisted tables", "blacklisted_table", rules.QRFailRetry) - for _, t := range tables { - qr.AddTableCond(t) - } - blacklistRules.Add(qr) - } - } - - loadRuleErr := tm.QueryServiceControl.SetQueryRules(blacklistQueryRules, blacklistRules) - if loadRuleErr != nil { - log.Warningf("Fail to load query rule set %s: %s", blacklistQueryRules, loadRuleErr) - } - return nil -} - -// lameduck changes the QueryServiceControl state to lameduck, -// brodcasts the new health, then sleep for grace period, to give time -// to clients to get the new status. -func (tm *TabletManager) lameduck(reason string) { - log.Infof("TabletManager is entering lameduck, reason: %v", reason) - tm.QueryServiceControl.EnterLameduck() - time.Sleep(gracePeriod) - log.Infof("TabletManager is leaving lameduck") -} - -// refreshTablet needs to be run after an action may have changed the current -// state of the tablet. -func (tm *TabletManager) refreshTablet(ctx context.Context, reason string) error { - tm.checkLock() - log.Infof("Executing post-action state refresh: %v", reason) - - span, ctx := trace.NewSpan(ctx, "TabletManager.refreshTablet") - span.Annotate("reason", reason) - defer span.Finish() - - // TODO(sougou): change this to specifically look for global topo changes. - tablet := tm.Tablet() - tm.changeCallback(ctx, tablet, tablet) - log.Infof("Done with post-action state refresh") - return nil -} - -// updateState will use the provided tablet record as the new tablet state, -// the current tablet as a base, run changeCallback, and dispatch the event. -func (tm *TabletManager) updateState(ctx context.Context, newTablet *topodatapb.Tablet, reason string) { - oldTablet := tm.Tablet() - if oldTablet == nil { - oldTablet = &topodatapb.Tablet{} - } - log.Infof("Running tablet callback because: %v", reason) - tm.changeCallback(ctx, oldTablet, newTablet) - tm.setTablet(newTablet) - tm.publishState(ctx) - event.Dispatch(&events.StateChange{ - OldTablet: *oldTablet, - NewTablet: *newTablet, - Reason: reason, - }) -} - -// changeCallback is run after every action that might -// have changed something in the tablet record or in the topology. -func (tm *TabletManager) changeCallback(ctx context.Context, oldTablet, newTablet *topodatapb.Tablet) { - tm.checkLock() - - span, ctx := trace.NewSpan(ctx, "TabletManager.changeCallback") - defer span.Finish() - - allowQuery := topo.IsRunningQueryService(newTablet.Type) - // TODO(sougou): find a better way to compute this. - terTime := logutil.ProtoToTime(newTablet.MasterTermStartTime) - - // Read the shard to get SourceShards / TabletControlMap if - // we're going to use it. - var shardInfo *topo.ShardInfo - var err error - // this is just for logging - var disallowQueryReason string - // this is actually used to set state - var disallowQueryService string - var blacklistedTables []string - updateBlacklistedTables := true - if allowQuery { - shardInfo, err = tm.TopoServer.GetShard(ctx, newTablet.Keyspace, newTablet.Shard) - if err != nil { - log.Errorf("Cannot read shard for this tablet %v, might have inaccurate SourceShards and TabletControls: %v", newTablet.Alias, err) - updateBlacklistedTables = false - } else { - if newTablet.Type == topodatapb.TabletType_MASTER { - if len(shardInfo.SourceShards) > 0 { - allowQuery = false - disallowQueryReason = "master tablet with filtered replication on" - disallowQueryService = disallowQueryReason - } - } - srvKeyspace, err := tm.TopoServer.GetSrvKeyspace(ctx, newTablet.Alias.Cell, newTablet.Keyspace) - if err != nil { - log.Errorf("failed to get SrvKeyspace %v with: %v", newTablet.Keyspace, err) - } else { - - for _, partition := range srvKeyspace.GetPartitions() { - if partition.GetServedType() != newTablet.Type { - continue - } - - for _, tabletControl := range partition.GetShardTabletControls() { - if key.KeyRangeEqual(tabletControl.GetKeyRange(), newTablet.GetKeyRange()) { - if tabletControl.QueryServiceDisabled { - allowQuery = false - disallowQueryReason = "TabletControl.DisableQueryService set" - disallowQueryService = disallowQueryReason - } - break - } - } - } - } - if tc := shardInfo.GetTabletControl(newTablet.Type); tc != nil { - if topo.InCellList(newTablet.Alias.Cell, tc.Cells) { - - blacklistedTables = tc.BlacklistedTables - } - } - } - } else { - disallowQueryReason = fmt.Sprintf("not a serving tablet type(%v)", newTablet.Type) - disallowQueryService = disallowQueryReason - } - tm.setServicesDesiredState(disallowQueryService) - if updateBlacklistedTables { - if err := tm.loadBlacklistRules(ctx, newTablet, blacklistedTables); err != nil { - // FIXME(alainjobart) how to handle this error? - log.Errorf("Cannot update blacklisted tables rule: %v", err) - } else { - tm.setBlacklistedTables(blacklistedTables) - } - } - - tm.replManager.SetTabletType(newTablet.Type) - - if allowQuery { - if err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, true); err != nil { - log.Errorf("Cannot start query service: %v", err) - } - } else { - // Query service should be stopped. - if topo.IsSubjectToLameduck(oldTablet.Type) && - newTablet.Type == topodatapb.TabletType_SPARE && - gracePeriod > 0 { - // When a non-MASTER serving type is going SPARE, - // put query service in lameduck during gracePeriod. - tm.lameduck(disallowQueryReason) - } - - log.Infof("Disabling query service on type change, reason: %v", disallowQueryReason) - if err := tm.QueryServiceControl.SetServingType(newTablet.Type, terTime, false); err != nil { - log.Errorf("SetServingType(serving=false) failed: %v", err) - } - } - - if tm.UpdateStream != nil { - if topo.IsRunningUpdateStream(newTablet.Type) { - tm.UpdateStream.Enable() - } else { - tm.UpdateStream.Disable() - } - } - - // Update the stats to our current type. - s := topoproto.TabletTypeLString(newTablet.Type) - statsTabletType.Set(s) - statsTabletTypeCount.Add(s, 1) - - // See if we need to start or stop vreplication. - if tm.VREngine != nil { - if newTablet.Type == topodatapb.TabletType_MASTER { - tm.VREngine.Open(tm.BatchCtx) - } else { - tm.VREngine.Close() - } - } -} - -func (tm *TabletManager) publishState(ctx context.Context) { - tm.pubMu.Lock() - defer tm.pubMu.Unlock() - log.Infof("Publishing state: %v", tm.tablet) - // If retry is in progress, there's nothing to do. - if tm.isPublishing { - return - } - // Common code path: publish immediately. - ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer cancel() - _, err := tm.TopoServer.UpdateTabletFields(ctx, tm.tabletAlias, func(tablet *topodatapb.Tablet) error { - if err := topotools.CheckOwnership(tablet, tm.tablet); err != nil { - log.Error(err) - return topo.NewError(topo.NoUpdateNeeded, "") - } - *tablet = *proto.Clone(tm.tablet).(*topodatapb.Tablet) - return nil - }) - if err != nil { - log.Errorf("Unable to publish state to topo, will keep retrying: %v", err) - tm.isPublishing = true - // Keep retrying until success. - go tm.retryPublish() - } -} - -func (tm *TabletManager) retryPublish() { - tm.pubMu.Lock() - defer func() { - tm.isPublishing = false - tm.pubMu.Unlock() - }() - - for { - // Retry immediately the first time because the previous failure might have been - // due to an expired context. - ctx, cancel := context.WithTimeout(tm.BatchCtx, *topo.RemoteOperationTimeout) - _, err := tm.TopoServer.UpdateTabletFields(ctx, tm.tabletAlias, func(tablet *topodatapb.Tablet) error { - if err := topotools.CheckOwnership(tablet, tm.tablet); err != nil { - log.Error(err) - return topo.NewError(topo.NoUpdateNeeded, "") - } - *tablet = *proto.Clone(tm.tablet).(*topodatapb.Tablet) - return nil - }) - cancel() - if err != nil { - log.Errorf("Unable to publish state to topo, will keep retrying: %v", err) - tm.pubMu.Unlock() - time.Sleep(*publishRetryInterval) - tm.pubMu.Lock() - continue - } - log.Infof("Published state: %v", tm.tablet) - return - } -} diff --git a/go/vt/vttablet/tabletmanager/state_change_test.go b/go/vt/vttablet/tabletmanager/state_change_test.go index c762173a8b6..5dfb59bd715 100644 --- a/go/vt/vttablet/tabletmanager/state_change_test.go +++ b/go/vt/vttablet/tabletmanager/state_change_test.go @@ -50,16 +50,20 @@ func TestPublishState(t *testing.T) { tab1 := tm.Tablet() tab1.Keyspace = "tab1" - tm.setTablet(tab1) - tm.publishState(ctx) + tm.tmState.mu.Lock() + tm.tmState.tablet = tab1 + tm.tmState.publishStateLocked(ctx) + tm.tmState.mu.Unlock() ttablet, err = tm.TopoServer.GetTablet(ctx, tm.tabletAlias) require.NoError(t, err) assert.Equal(t, tab1, ttablet.Tablet) tab2 := tm.Tablet() tab2.Keyspace = "tab2" - tm.setTablet(tab2) - tm.retryPublish() + tm.tmState.mu.Lock() + tm.tmState.tablet = tab2 + tm.tmState.mu.Unlock() + tm.tmState.retryPublish() ttablet, err = tm.TopoServer.GetTablet(ctx, tm.tabletAlias) require.NoError(t, err) assert.Equal(t, tab2, ttablet.Tablet) @@ -67,14 +71,16 @@ func TestPublishState(t *testing.T) { // If hostname doesn't match, it should not update. tab3 := tm.Tablet() tab3.Hostname = "tab3" - tm.setTablet(tab3) - tm.publishState(ctx) + tm.tmState.mu.Lock() + tm.tmState.tablet = tab3 + tm.tmState.publishStateLocked(ctx) + tm.tmState.mu.Unlock() ttablet, err = tm.TopoServer.GetTablet(ctx, tm.tabletAlias) require.NoError(t, err) assert.Equal(t, tab2, ttablet.Tablet) // Same for retryPublish. - tm.retryPublish() + tm.tmState.retryPublish() ttablet, err = tm.TopoServer.GetTablet(ctx, tm.tabletAlias) require.NoError(t, err) assert.Equal(t, tab2, ttablet.Tablet) diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 4cb73cb4b73..55bb09b3899 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -47,8 +47,6 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/vt/dbconnpool" - "github.com/golang/protobuf/proto" - "vitess.io/vitess/go/history" "vitess.io/vitess/go/netutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/binlog" @@ -69,6 +67,9 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) +// Query rules from blacklist +const blacklistQueryRules string = "BlacklistQueryRules" + var ( // The following flags initialize the tablet record. tabletHostname = flag.String("tablet_hostname", "", "if not empty, this hostname will be assumed instead of trying to resolve it") @@ -127,6 +128,9 @@ type TabletManager struct { UpdateStream binlog.UpdateStreamControl VREngine *vreplication.Engine + // tmState manages the TabletManager state. + tmState *tmState + // replManager manages replication. replManager *replManager @@ -137,10 +141,6 @@ type TabletManager struct { // when we transition back from something like MASTER. baseTabletType topodatapb.TabletType - // History of the health checks, public so status - // pages can display it - History *history.History - // actionMutex is there to run only one action at a time. If // both tm.actionMutex and tm.mutex needs to be taken, // take actionMutex first. @@ -161,10 +161,6 @@ type TabletManager struct { // only hold the mutex to update the fields, nothing else. mutex sync.Mutex - // _shardInfo and _srvKeyspace are cached and refreshed on RefreshState. - _shardInfo *topo.ShardInfo - _srvKeyspace *topodatapb.SrvKeyspace - // _shardSyncChan is a channel for informing the shard sync goroutine that // it should wake up and recheck the tablet state, to make sure it and the // shard record are in sync. @@ -179,27 +175,11 @@ type TabletManager struct { // _shardSyncCancel is the function to stop the background shard sync goroutine. _shardSyncCancel context.CancelFunc - // _disallowQueryService is set to the reason we should be - // disallowing queries from being served. It is set from changeCallback, - // and used by healthcheck. If empty, we should allow queries. - // It is set if the current type is not serving, if a TabletControl - // tells us not to serve, or if filtered replication is running. - _disallowQueryService string - - // _blacklistedTables has the list of tables we are currently - // blacklisting. - _blacklistedTables []string - // _lockTablesConnection is used to get and release the table read locks to pause replication _lockTablesConnection *dbconnpool.DBConnection _lockTablesTimer *time.Timer // _isBackupRunning tells us whether there is a backup that is currently running _isBackupRunning bool - - pubMu sync.Mutex - // tablet has the Tablet record we last read from the topology server. - tablet *topodatapb.Tablet - isPublishing bool } var healthCheckInterval time.Duration @@ -207,7 +187,6 @@ var healthCheckInterval time.Duration // InitConfig is a temp function to keep things working during the refactor. func InitConfig(config *tabletenv.TabletConfig) { healthCheckInterval = config.Healthcheck.IntervalSeconds.Get() - gracePeriod = config.GracePeriods.TransitionSeconds.Get() } // BuildTabletFromInput builds a tablet record from input parameters. @@ -262,11 +241,11 @@ func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32) ( // Start starts the TabletManager. func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { - tm.setTablet(tablet) tm.DBConfigs.DBName = topoproto.TabletDbName(tablet) tm.replManager = newReplManager(tm.BatchCtx, tm, healthCheckInterval) - tm.History = history.New(historyLength) tm.tabletAlias = tablet.Alias + tm.tmState = newTMState(tm, tablet) + demoteType, err := topoproto.ParseTabletType(*demoteMasterType) if err != nil { return err @@ -278,10 +257,11 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { ctx, cancel := context.WithTimeout(tm.BatchCtx, *initTimeout) defer cancel() - if err := tm.createKeyspaceShard(ctx); err != nil { + si, err := tm.createKeyspaceShard(ctx) + if err != nil { return err } - if err := tm.checkMastership(ctx); err != nil { + if err := tm.checkMastership(ctx, si); err != nil { return err } if err := tm.checkMysql(ctx); err != nil { @@ -340,9 +320,7 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { return err } defer tm.unlock() - tablet = tm.Tablet() - tm.changeCallback(tm.BatchCtx, tablet, tablet) - + tm.tmState.Open(tm.BatchCtx) return nil } @@ -372,6 +350,8 @@ func (tm *TabletManager) Close() { if _, err := tm.TopoServer.UpdateTabletFields(updateCtx, tm.tabletAlias, f); err != nil { log.Warningf("Failed to update tablet record, may contain stale identifiers: %v", err) } + + tm.tmState.Close() } // Stop shuts down the tm. Normally this is not necessary, since we use @@ -392,9 +372,10 @@ func (tm *TabletManager) Stop() { } tm.MysqlDaemon.Close() + tm.tmState.Close() } -func (tm *TabletManager) createKeyspaceShard(ctx context.Context) error { +func (tm *TabletManager) createKeyspaceShard(ctx context.Context) (*topo.ShardInfo, error) { // mutex is needed because we set _shardInfo and _srvKeyspace tm.mutex.Lock() defer tm.mutex.Unlock() @@ -403,23 +384,25 @@ func (tm *TabletManager) createKeyspaceShard(ctx context.Context) error { log.Infof("Reading/creating keyspace and shard records for %v/%v", tablet.Keyspace, tablet.Shard) // Read the shard, create it if necessary. + var shardInfo *topo.ShardInfo if err := tm.withRetry(ctx, "creating keyspace and shard", func() error { var err error - tm._shardInfo, err = tm.TopoServer.GetOrCreateShard(ctx, tablet.Keyspace, tablet.Shard) + shardInfo, err = tm.TopoServer.GetOrCreateShard(ctx, tablet.Keyspace, tablet.Shard) return err }); err != nil { - return vterrors.Wrap(err, "createKeyspaceShard: cannot GetOrCreateShard shard") + return nil, vterrors.Wrap(err, "createKeyspaceShard: cannot GetOrCreateShard shard") } + tm.tmState.RefreshFromTopoInfo(ctx, shardInfo, nil) // Rebuild keyspace if this the first tablet in this keyspace/cell srvKeyspace, err := tm.TopoServer.GetSrvKeyspace(ctx, tm.tabletAlias.Cell, tablet.Keyspace) switch { case err == nil: - tm._srvKeyspace = srvKeyspace + tm.tmState.RefreshFromTopoInfo(ctx, nil, srvKeyspace) case topo.IsErrType(err, topo.NoNode): go tm.rebuildKeyspace(tablet.Keyspace, rebuildKeyspaceRetryInterval) default: - return vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to read SrvKeyspace") + return nil, vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to read SrvKeyspace") } // Rebuild vschema graph if this is the first tablet in this keyspace/cell. @@ -429,27 +412,25 @@ func (tm *TabletManager) createKeyspaceShard(ctx context.Context) error { // Check if vschema was rebuilt after the initial creation of the keyspace. if _, keyspaceExists := srvVSchema.GetKeyspaces()[tablet.Keyspace]; !keyspaceExists { if err := tm.TopoServer.RebuildSrvVSchema(ctx, []string{tm.tabletAlias.Cell}); err != nil { - return vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to RebuildSrvVSchema") + return nil, vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to RebuildSrvVSchema") } } case topo.IsErrType(err, topo.NoNode): // There is no SrvSchema in this cell at all, so we definitely need to rebuild. if err := tm.TopoServer.RebuildSrvVSchema(ctx, []string{tm.tabletAlias.Cell}); err != nil { - return vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to RebuildSrvVSchema") + return nil, vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to RebuildSrvVSchema") } default: - return vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to read SrvVSchema") + return nil, vterrors.Wrap(err, "initeKeyspaceShardTopo: failed to read SrvVSchema") } - return nil + return shardInfo, nil } func (tm *TabletManager) rebuildKeyspace(keyspace string, retryInterval time.Duration) { var srvKeyspace *topodatapb.SrvKeyspace defer func() { log.Infof("Keyspace rebuilt: %v", keyspace) - tm.mutex.Lock() - defer tm.mutex.Unlock() - tm._srvKeyspace = srvKeyspace + tm.tmState.RefreshFromTopoInfo(tm.BatchCtx, nil, srvKeyspace) }() // RebuildKeyspace will fail until at least one tablet is up for every shard. @@ -478,11 +459,7 @@ func (tm *TabletManager) rebuildKeyspace(keyspace string, retryInterval time.Dur } } -func (tm *TabletManager) checkMastership(ctx context.Context) error { - tm.mutex.Lock() - si := tm._shardInfo - tm.mutex.Unlock() - +func (tm *TabletManager) checkMastership(ctx context.Context, si *topo.ShardInfo) error { if si.MasterAlias != nil && topoproto.TabletAliasEqual(si.MasterAlias, tm.tabletAlias) { // We're marked as master in the shard record, which could mean the master // tablet process was just restarted. However, we need to check if a new @@ -493,7 +470,7 @@ func (tm *TabletManager) checkMastership(ctx context.Context) error { case topo.IsErrType(err, topo.NoNode): // There's no existing tablet record, so we can assume // no one has left us a message to step down. - tm.updateTablet(func(tablet *topodatapb.Tablet) { + tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.Type = topodatapb.TabletType_MASTER // Update the master term start time (current value is 0) because we // assume that we are actually the MASTER and in case of a tiebreak, @@ -504,7 +481,7 @@ func (tm *TabletManager) checkMastership(ctx context.Context) error { if oldTablet.Type == topodatapb.TabletType_MASTER { // We're marked as master in the shard record, // and our existing tablet record agrees. - tm.updateTablet(func(tablet *topodatapb.Tablet) { + tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.Type = topodatapb.TabletType_MASTER tablet.MasterTermStartTime = oldTablet.MasterTermStartTime }) @@ -524,7 +501,7 @@ func (tm *TabletManager) checkMastership(ctx context.Context) error { oldMasterTermStartTime := oldTablet.GetMasterTermStartTime() currentShardTime := si.GetMasterTermStartTime() if oldMasterTermStartTime.After(currentShardTime) { - tm.updateTablet(func(tablet *topodatapb.Tablet) { + tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.Type = topodatapb.TabletType_MASTER tablet.MasterTermStartTime = oldTablet.MasterTermStartTime }) @@ -539,13 +516,13 @@ func (tm *TabletManager) checkMastership(ctx context.Context) error { func (tm *TabletManager) checkMysql(ctx context.Context) error { if appConfig, _ := tm.DBConfigs.AppWithDB().MysqlParams(); appConfig.Host != "" { - tm.updateTablet(func(tablet *topodatapb.Tablet) { + tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.MysqlHostname = appConfig.Host tablet.MysqlPort = int32(appConfig.Port) }) } else { // Assume unix socket was specified and try to get the port from mysqld - tm.updateTablet(func(tablet *topodatapb.Tablet) { + tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.MysqlHostname = tablet.Hostname }) mysqlPort, err := tm.MysqlDaemon.GetMysqlPort() @@ -553,7 +530,7 @@ func (tm *TabletManager) checkMysql(ctx context.Context) error { log.Warningf("Cannot get current mysql port, will keep retrying every %v: %v", mysqlPortRetryInterval, err) go tm.findMysqlPort(mysqlPortRetryInterval) } else { - tm.updateTablet(func(tablet *topodatapb.Tablet) { + tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.MysqlPort = mysqlPort }) } @@ -568,17 +545,8 @@ func (tm *TabletManager) findMysqlPort(retryInterval time.Duration) { if err != nil { continue } - // We need to get the action lock to make sure no one - // else is updating the tablet. - if err := tm.lock(tm.BatchCtx); err != nil { - continue - } - defer tm.unlock() log.Infof("Identified mysql port: %v", mport) - tm.pubMu.Lock() - tm.tablet.MysqlPort = mport - tm.pubMu.Unlock() - tm.publishState(tm.BatchCtx) + tm.tmState.SetMysqlPort(mport) return } } @@ -701,57 +669,20 @@ func (tm *TabletManager) withRetry(ctx context.Context, description string, work } } -func (tm *TabletManager) setTablet(tablet *topodatapb.Tablet) { - tm.pubMu.Lock() - tm.tablet = proto.Clone(tablet).(*topodatapb.Tablet) - tm.pubMu.Unlock() - - // Notify the shard sync loop that the tablet state changed. - tm.notifyShardSync() -} - -func (tm *TabletManager) updateTablet(update func(tablet *topodatapb.Tablet)) { - tm.pubMu.Lock() - update(tm.tablet) - tm.pubMu.Unlock() - - // Notify the shard sync loop that the tablet state changed. - tm.notifyShardSync() -} - // Tablet reads the stored Tablet from the tm. func (tm *TabletManager) Tablet() *topodatapb.Tablet { - tm.pubMu.Lock() - tablet := proto.Clone(tm.tablet).(*topodatapb.Tablet) - tm.pubMu.Unlock() - return tablet + return tm.tmState.Tablet() } // BlacklistedTables returns the list of currently blacklisted tables. func (tm *TabletManager) BlacklistedTables() []string { - tm.mutex.Lock() - defer tm.mutex.Unlock() - return tm._blacklistedTables + return tm.tmState.BlacklistedTables() } // DisallowQueryService returns the reason the query service should be // disabled, if any. func (tm *TabletManager) DisallowQueryService() string { - tm.mutex.Lock() - defer tm.mutex.Unlock() - return tm._disallowQueryService -} - -func (tm *TabletManager) setServicesDesiredState(disallowQueryService string) { - tm.mutex.Lock() - tm._disallowQueryService = disallowQueryService - tm.mutex.Unlock() -} - -func (tm *TabletManager) setBlacklistedTables(value []string) { - tm.mutex.Lock() - tm._blacklistedTables = value - tm.mutex.Unlock() + return tm.tmState.DisallowQueryService() } // hookExtraEnv returns the map to pass to local hooks diff --git a/go/vt/vttablet/tabletmanager/tm_init_test.go b/go/vt/vttablet/tabletmanager/tm_init_test.go index d1bf8275580..f1e3aa54d22 100644 --- a/go/vt/vttablet/tabletmanager/tm_init_test.go +++ b/go/vt/vttablet/tabletmanager/tm_init_test.go @@ -343,9 +343,7 @@ func TestStartFindMysqlPort(t *testing.T) { require.NoError(t, err) assert.Equal(t, int32(0), ti.MysqlPort) - tm.pubMu.Lock() fmd.MysqlPort.Set(3306) - tm.pubMu.Unlock() for i := 0; i < 10; i++ { ti, err := ts.GetTablet(ctx, tm.tabletAlias) require.NoError(t, err) @@ -395,7 +393,7 @@ func TestStartDoesNotUpdateReplicationDataForTabletInWrongShard(t *testing.T) { ctx := context.Background() ts := memorytopo.NewServer("cell1", "cell2") tm := newTestTM(t, ts, 1, "ks", "0") - defer tm.Stop() + tm.Stop() tabletAliases, err := ts.FindAllTabletAliasesInShard(ctx, "ks", "0") require.NoError(t, err) diff --git a/go/vt/vttablet/tabletmanager/tm_state.go b/go/vt/vttablet/tabletmanager/tm_state.go new file mode 100644 index 00000000000..e19f454d966 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/tm_state.go @@ -0,0 +1,356 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletmanager + +import ( + "flag" + "fmt" + "strings" + "sync" + "time" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "vitess.io/vitess/go/trace" + "vitess.io/vitess/go/vt/key" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools" + "vitess.io/vitess/go/vt/vttablet/tabletserver/rules" +) + +var publishRetryInterval = flag.Duration("publish_retry_interval", 30*time.Second, "how long vttablet waits to retry publishing the tablet record") + +// tmState manages the state of the TabletManager. +type tmState struct { + tm *TabletManager + ctx context.Context + cancel context.CancelFunc + + // Read-only variables. + tabletAlias *topodatapb.TabletAlias + keyspace string + shard string + keyrange *topodatapb.KeyRange + + mu sync.Mutex + isOpen bool + isResharding bool + tabletControls map[topodatapb.TabletType]bool + blacklistedTables map[topodatapb.TabletType][]string + tablet *topodatapb.Tablet + isPublishing bool +} + +func newTMState(tm *TabletManager, tablet *topodatapb.Tablet) *tmState { + return &tmState{ + tm: tm, + tabletAlias: tm.tabletAlias, + keyspace: tablet.Keyspace, + shard: tablet.Shard, + keyrange: tablet.KeyRange, + tablet: tablet, + } +} + +func (ts *tmState) Open(ctx context.Context) { + ts.mu.Lock() + defer ts.mu.Unlock() + if ts.isOpen { + return + } + + ts.ctx, ts.cancel = context.WithCancel(ctx) + ts.isOpen = true + ts.updateLocked(ts.ctx) + ts.publishStateLocked(ctx) +} + +func (ts *tmState) Close() { + ts.mu.Lock() + defer ts.mu.Unlock() + + ts.isOpen = false + ts.cancel() +} + +func (ts *tmState) RefreshFromTopo(ctx context.Context) error { + span, ctx := trace.NewSpan(ctx, "tmState.refreshFromTopo") + defer span.Finish() + + shardInfo, err := ts.tm.TopoServer.GetShard(ctx, ts.keyspace, ts.shard) + if err != nil { + return err + } + + srvKeyspace, err := ts.tm.TopoServer.GetSrvKeyspace(ctx, ts.tabletAlias.Cell, ts.keyspace) + if err != nil { + return err + } + ts.RefreshFromTopoInfo(ctx, shardInfo, srvKeyspace) + return nil +} + +func (ts *tmState) RefreshFromTopoInfo(ctx context.Context, shardInfo *topo.ShardInfo, srvKeyspace *topodatapb.SrvKeyspace) { + ts.mu.Lock() + defer ts.mu.Unlock() + + if shardInfo != nil { + ts.isResharding = len(shardInfo.SourceShards) > 0 + + ts.blacklistedTables = make(map[topodatapb.TabletType][]string) + for _, tc := range shardInfo.TabletControls { + if topo.InCellList(ts.tabletAlias.Cell, tc.Cells) { + ts.blacklistedTables[tc.TabletType] = tc.BlacklistedTables + } + } + } + + if srvKeyspace != nil { + ts.tabletControls = make(map[topodatapb.TabletType]bool) + for _, partition := range srvKeyspace.GetPartitions() { + for _, tabletControl := range partition.GetShardTabletControls() { + if key.KeyRangeEqual(tabletControl.GetKeyRange(), ts.keyrange) { + if tabletControl.QueryServiceDisabled { + ts.tabletControls[partition.GetServedType()] = true + } + break + } + } + } + } + + ts.updateLocked(ctx) +} + +func (ts *tmState) ChangeTabletType(ctx context.Context, tabletType topodatapb.TabletType) error { + ts.mu.Lock() + defer ts.mu.Unlock() + + if tabletType == topodatapb.TabletType_MASTER { + masterTermStartTime := logutil.TimeToProto(time.Now()) + + // Update the tablet record first. + _, err := topotools.ChangeType(ctx, ts.tm.TopoServer, ts.tabletAlias, tabletType, masterTermStartTime) + if err != nil { + return err + } + + ts.tablet.Type = tabletType + ts.tablet.MasterTermStartTime = masterTermStartTime + } else { + ts.tablet.Type = tabletType + ts.tablet.MasterTermStartTime = nil + } + + s := topoproto.TabletTypeLString(tabletType) + statsTabletType.Set(s) + statsTabletTypeCount.Add(s, 1) + + ts.updateLocked(ctx) + ts.publishStateLocked(ctx) + ts.tm.notifyShardSync() + return nil +} + +func (ts *tmState) SetMysqlPort(mport int32) { + ts.mu.Lock() + defer ts.mu.Unlock() + + ts.tablet.MysqlPort = mport + ts.publishStateLocked(ts.ctx) +} + +// UpdateTablet must be called during initialization only. +func (ts *tmState) UpdateTablet(update func(tablet *topodatapb.Tablet)) { + ts.mu.Lock() + defer ts.mu.Unlock() + update(ts.tablet) +} + +func (ts *tmState) Tablet() *topodatapb.Tablet { + ts.mu.Lock() + defer ts.mu.Unlock() + return proto.Clone(ts.tablet).(*topodatapb.Tablet) +} + +func (ts *tmState) MasterTermStartTime() time.Time { + ts.mu.Lock() + defer ts.mu.Unlock() + if ts.tablet == nil { + return time.Time{} + } + return logutil.ProtoToTime(ts.tablet.MasterTermStartTime) +} + +func (ts *tmState) DisallowQueryService() string { + ts.mu.Lock() + defer ts.mu.Unlock() + return ts.canServe(ts.tablet.Type) +} + +func (ts *tmState) BlacklistedTables() []string { + ts.mu.Lock() + defer ts.mu.Unlock() + return ts.blacklistedTables[ts.tablet.Type] +} + +func (ts *tmState) updateLocked(ctx context.Context) { + span, ctx := trace.NewSpan(ctx, "tmState.update") + defer span.Finish() + + if !ts.isOpen { + return + } + + ts.tm.replManager.SetTabletType(ts.tablet.Type) + + if err := ts.applyBlacklist(ctx); err != nil { + log.Errorf("Cannot update blacklisted tables rule: %v", err) + } + + terTime := logutil.ProtoToTime(ts.tablet.MasterTermStartTime) + reason := ts.canServe(ts.tablet.Type) + if reason == "" { + if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, true); err != nil { + log.Errorf("Cannot start query service: %v", err) + } + } else { + log.Infof("Disabling query service: %v", reason) + if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, false); err != nil { + log.Errorf("SetServingType(serving=false) failed: %v", err) + } + } + + if ts.tm.UpdateStream != nil { + if topo.IsRunningUpdateStream(ts.tablet.Type) { + ts.tm.UpdateStream.Enable() + } else { + ts.tm.UpdateStream.Disable() + } + } + + // See if we need to start or stop vreplication. + if ts.tm.VREngine != nil { + if ts.tablet.Type == topodatapb.TabletType_MASTER { + ts.tm.VREngine.Open(ts.tm.BatchCtx) + } else { + ts.tm.VREngine.Close() + } + } +} + +func (ts *tmState) canServe(tabletType topodatapb.TabletType) string { + if !topo.IsRunningQueryService(tabletType) { + return fmt.Sprintf("not a serving tablet type(%v)", tabletType) + } + if tabletType == topodatapb.TabletType_MASTER && ts.isResharding { + return "master tablet with filtered replication on" + } + if ts.tabletControls[tabletType] { + return "TabletControl.DisableQueryService set" + } + return "" +} + +func (ts *tmState) applyBlacklist(ctx context.Context) (err error) { + blacklistRules := rules.New() + blacklistedTables := ts.blacklistedTables[ts.tablet.Type] + if len(blacklistedTables) > 0 { + tables, err := mysqlctl.ResolveTables(ctx, ts.tm.MysqlDaemon, topoproto.TabletDbName(ts.tablet), blacklistedTables) + if err != nil { + return err + } + + // Verify that at least one table matches the wildcards, so + // that we don't add a rule to blacklist all tables + if len(tables) > 0 { + log.Infof("Blacklisting tables %v", strings.Join(tables, ", ")) + qr := rules.NewQueryRule("enforce blacklisted tables", "blacklisted_table", rules.QRFailRetry) + for _, t := range tables { + qr.AddTableCond(t) + } + blacklistRules.Add(qr) + } + } + + loadRuleErr := ts.tm.QueryServiceControl.SetQueryRules(blacklistQueryRules, blacklistRules) + if loadRuleErr != nil { + log.Warningf("Fail to load query rule set %s: %s", blacklistQueryRules, loadRuleErr) + } + return nil +} + +func (ts *tmState) publishStateLocked(ctx context.Context) { + log.Infof("Publishing state: %v", ts.tablet) + // If retry is in progress, there's nothing to do. + if ts.isPublishing { + return + } + // Fast path: publish immediately. + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer cancel() + _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tabletAlias, func(tablet *topodatapb.Tablet) error { + if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil { + log.Error(err) + return topo.NewError(topo.NoUpdateNeeded, "") + } + *tablet = *proto.Clone(ts.tablet).(*topodatapb.Tablet) + return nil + }) + if err != nil { + log.Errorf("Unable to publish state to topo, will keep retrying: %v", err) + ts.isPublishing = true + // Keep retrying until success. + go ts.retryPublish() + } +} + +func (ts *tmState) retryPublish() { + ts.mu.Lock() + defer ts.mu.Unlock() + + defer func() { ts.isPublishing = false }() + + for { + // Retry immediately the first time because the previous failure might have been + // due to an expired context. + ctx, cancel := context.WithTimeout(ts.ctx, *topo.RemoteOperationTimeout) + _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tabletAlias, func(tablet *topodatapb.Tablet) error { + if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil { + log.Error(err) + return topo.NewError(topo.NoUpdateNeeded, "") + } + *tablet = *proto.Clone(ts.tablet).(*topodatapb.Tablet) + return nil + }) + cancel() + if err != nil { + log.Errorf("Unable to publish state to topo, will keep retrying: %v", err) + ts.mu.Unlock() + time.Sleep(*publishRetryInterval) + ts.mu.Lock() + continue + } + log.Infof("Published state: %v", ts.tablet) + return + } +} From 991dde7a26b0799b3c154906d50e913f6421654e Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 15 Jul 2020 17:02:19 -0700 Subject: [PATCH 031/100] vttablet: create a separate displayState Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/tm_init.go | 11 +- go/vt/vttablet/tabletmanager/tm_state.go | 139 ++++++++++++------ go/vt/wrangler/testlib/reparent_utils_test.go | 2 - 3 files changed, 97 insertions(+), 55 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 55bb09b3899..02112108d8d 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -141,9 +141,10 @@ type TabletManager struct { // when we transition back from something like MASTER. baseTabletType topodatapb.TabletType - // actionMutex is there to run only one action at a time. If - // both tm.actionMutex and tm.mutex needs to be taken, - // take actionMutex first. + // actionMutex is there to run only one action at a time. + // This mutex can be held for long periods of time (hours), + // like in the case of a restore. This mutex must be obtained + // first before other mutexes. actionMutex sync.Mutex // actionMutexLocked is set to true after we acquire actionMutex, @@ -316,10 +317,6 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet) error { return nil } - if err := tm.lock(tm.BatchCtx); err != nil { - return err - } - defer tm.unlock() tm.tmState.Open(tm.BatchCtx) return nil } diff --git a/go/vt/vttablet/tabletmanager/tm_state.go b/go/vt/vttablet/tabletmanager/tm_state.go index e19f454d966..88c73e1c4c5 100644 --- a/go/vt/vttablet/tabletmanager/tm_state.go +++ b/go/vt/vttablet/tabletmanager/tm_state.go @@ -45,12 +45,15 @@ type tmState struct { ctx context.Context cancel context.CancelFunc - // Read-only variables. - tabletAlias *topodatapb.TabletAlias - keyspace string - shard string - keyrange *topodatapb.KeyRange - + // mu must be held while accessing the following members and + // while changing the state of the system to match these values. + // This can be held for many seconds while tmState connects to + // external components to change their state. + // Obtaining tm.actionMutex before calling a tmState function is + // not required. + // Because mu can be held for long, we publish the current state + // of these variables into displayState, which can be accessed + // more freely even while tmState is busy transitioning. mu sync.Mutex isOpen bool isResharding bool @@ -58,16 +61,19 @@ type tmState struct { blacklistedTables map[topodatapb.TabletType][]string tablet *topodatapb.Tablet isPublishing bool + + // displayState contains the current snapshot of the internal state + // and has its own mutex. + displayState displayState } func newTMState(tm *TabletManager, tablet *topodatapb.Tablet) *tmState { return &tmState{ - tm: tm, - tabletAlias: tm.tabletAlias, - keyspace: tablet.Keyspace, - shard: tablet.Shard, - keyrange: tablet.KeyRange, - tablet: tablet, + tm: tm, + displayState: displayState{ + tablet: *tablet, + }, + tablet: tablet, } } @@ -96,12 +102,12 @@ func (ts *tmState) RefreshFromTopo(ctx context.Context) error { span, ctx := trace.NewSpan(ctx, "tmState.refreshFromTopo") defer span.Finish() - shardInfo, err := ts.tm.TopoServer.GetShard(ctx, ts.keyspace, ts.shard) + shardInfo, err := ts.tm.TopoServer.GetShard(ctx, ts.Keyspace(), ts.Shard()) if err != nil { return err } - srvKeyspace, err := ts.tm.TopoServer.GetSrvKeyspace(ctx, ts.tabletAlias.Cell, ts.keyspace) + srvKeyspace, err := ts.tm.TopoServer.GetSrvKeyspace(ctx, ts.tm.tabletAlias.Cell, ts.Keyspace()) if err != nil { return err } @@ -118,7 +124,7 @@ func (ts *tmState) RefreshFromTopoInfo(ctx context.Context, shardInfo *topo.Shar ts.blacklistedTables = make(map[topodatapb.TabletType][]string) for _, tc := range shardInfo.TabletControls { - if topo.InCellList(ts.tabletAlias.Cell, tc.Cells) { + if topo.InCellList(ts.tm.tabletAlias.Cell, tc.Cells) { ts.blacklistedTables[tc.TabletType] = tc.BlacklistedTables } } @@ -128,7 +134,7 @@ func (ts *tmState) RefreshFromTopoInfo(ctx context.Context, shardInfo *topo.Shar ts.tabletControls = make(map[topodatapb.TabletType]bool) for _, partition := range srvKeyspace.GetPartitions() { for _, tabletControl := range partition.GetShardTabletControls() { - if key.KeyRangeEqual(tabletControl.GetKeyRange(), ts.keyrange) { + if key.KeyRangeEqual(tabletControl.GetKeyRange(), ts.KeyRange()) { if tabletControl.QueryServiceDisabled { ts.tabletControls[partition.GetServedType()] = true } @@ -149,7 +155,7 @@ func (ts *tmState) ChangeTabletType(ctx context.Context, tabletType topodatapb.T masterTermStartTime := logutil.TimeToProto(time.Now()) // Update the tablet record first. - _, err := topotools.ChangeType(ctx, ts.tm.TopoServer, ts.tabletAlias, tabletType, masterTermStartTime) + _, err := topotools.ChangeType(ctx, ts.tm.TopoServer, ts.tm.tabletAlias, tabletType, masterTermStartTime) if err != nil { return err } @@ -186,33 +192,6 @@ func (ts *tmState) UpdateTablet(update func(tablet *topodatapb.Tablet)) { update(ts.tablet) } -func (ts *tmState) Tablet() *topodatapb.Tablet { - ts.mu.Lock() - defer ts.mu.Unlock() - return proto.Clone(ts.tablet).(*topodatapb.Tablet) -} - -func (ts *tmState) MasterTermStartTime() time.Time { - ts.mu.Lock() - defer ts.mu.Unlock() - if ts.tablet == nil { - return time.Time{} - } - return logutil.ProtoToTime(ts.tablet.MasterTermStartTime) -} - -func (ts *tmState) DisallowQueryService() string { - ts.mu.Lock() - defer ts.mu.Unlock() - return ts.canServe(ts.tablet.Type) -} - -func (ts *tmState) BlacklistedTables() []string { - ts.mu.Lock() - defer ts.mu.Unlock() - return ts.blacklistedTables[ts.tablet.Type] -} - func (ts *tmState) updateLocked(ctx context.Context) { span, ctx := trace.NewSpan(ctx, "tmState.update") defer span.Finish() @@ -229,6 +208,8 @@ func (ts *tmState) updateLocked(ctx context.Context) { terTime := logutil.ProtoToTime(ts.tablet.MasterTermStartTime) reason := ts.canServe(ts.tablet.Type) + ts.publishForDisplay(reason) + if reason == "" { if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, true); err != nil { log.Errorf("Cannot start query service: %v", err) @@ -308,7 +289,7 @@ func (ts *tmState) publishStateLocked(ctx context.Context) { // Fast path: publish immediately. ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() - _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tabletAlias, func(tablet *topodatapb.Tablet) error { + _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tm.tabletAlias, func(tablet *topodatapb.Tablet) error { if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil { log.Error(err) return topo.NewError(topo.NoUpdateNeeded, "") @@ -334,7 +315,7 @@ func (ts *tmState) retryPublish() { // Retry immediately the first time because the previous failure might have been // due to an expired context. ctx, cancel := context.WithTimeout(ts.ctx, *topo.RemoteOperationTimeout) - _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tabletAlias, func(tablet *topodatapb.Tablet) error { + _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tm.tabletAlias, func(tablet *topodatapb.Tablet) error { if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil { log.Error(err) return topo.NewError(topo.NoUpdateNeeded, "") @@ -354,3 +335,69 @@ func (ts *tmState) retryPublish() { return } } + +// displayState is the externalized version of tmState +// that can be used for observability. The internal version +// of tmState may not be accessible due to longer mutex holds. +// tmState uses publishForDisplay to keep these values uptodate. +type displayState struct { + mu sync.Mutex + tablet topodatapb.Tablet + reason string + blackListedTables []string +} + +// Note that the methods for displayState are all in tmState. +func (ts *tmState) publishForDisplay(reason string) { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + + ts.displayState.tablet = *ts.tablet + ts.displayState.reason = reason + ts.displayState.blackListedTables = ts.blacklistedTables[ts.tablet.Type] +} + +func (ts *tmState) Tablet() *topodatapb.Tablet { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + return proto.Clone(&ts.displayState.tablet).(*topodatapb.Tablet) +} + +func (ts *tmState) MasterTermStartTime() time.Time { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + if ts.tablet == nil { + return time.Time{} + } + return logutil.ProtoToTime(ts.tablet.MasterTermStartTime) +} + +func (ts *tmState) DisallowQueryService() string { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + return ts.displayState.reason +} + +func (ts *tmState) BlacklistedTables() []string { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + return ts.displayState.blackListedTables +} + +func (ts *tmState) Keyspace() string { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + return ts.displayState.tablet.Keyspace +} + +func (ts *tmState) Shard() string { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + return ts.displayState.tablet.Shard +} + +func (ts *tmState) KeyRange() *topodatapb.KeyRange { + ts.displayState.mu.Lock() + defer ts.displayState.mu.Unlock() + return ts.displayState.tablet.KeyRange +} diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 230030f0265..04a5dce8d99 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -130,8 +130,6 @@ func TestReparentTablet(t *testing.T) { replica.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", - "FAKE SET MASTER", - "FAKE SET MASTER", } replica.StartActionLoop(t, wr) defer replica.StopActionLoop(t) From 124b2457bdf3e75cbccedb89c527893f599e91c1 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 15 Jul 2020 18:21:32 -0700 Subject: [PATCH 032/100] vttablet: check replication on open This fixes the reparent tests. Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletmanager/replmanager.go | 21 ++++++++++++------- go/vt/vttablet/tabletmanager/restore.go | 3 --- go/vt/vttablet/tabletmanager/tm_init.go | 3 +++ go/vt/vttablet/tabletmanager/tm_state.go | 14 ++++++------- go/vt/wrangler/testlib/reparent_utils_test.go | 7 +++++++ 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/replmanager.go b/go/vt/vttablet/tabletmanager/replmanager.go index a48b33d66ea..d6b434a1345 100644 --- a/go/vt/vttablet/tabletmanager/replmanager.go +++ b/go/vt/vttablet/tabletmanager/replmanager.go @@ -80,10 +80,24 @@ func (rm *replManager) SetTabletType(tabletType topodatapb.TabletType) { if rm.ticks.Running() { return } + // Run an immediate check to fix replication if it was broken. + // A higher caller may already have te action lock. So, we use + // a code path that avoids it. + rm.checkActionLocked() rm.ticks.Start(rm.check) } func (rm *replManager) check() { + // We need to obtain the action lock if we're going to fix + // replication + if err := rm.tm.lock(rm.ctx); err != nil { + return + } + defer rm.tm.unlock() + rm.checkActionLocked() +} + +func (rm *replManager) checkActionLocked() { status, err := rm.tm.MysqlDaemon.ReplicationStatus() if err != nil { if err != mysql.ErrNotReplica { @@ -97,13 +111,6 @@ func (rm *replManager) check() { } } - // We need to obtain the action lock if we're going to fix - // replication - if err := rm.tm.lock(rm.ctx); err != nil { - return - } - defer rm.tm.unlock() - if rm.firstFailure { log.Infof("Replication is stopped, reconnecting to master.") } diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index 0b5a2695432..6246360741d 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -76,9 +76,6 @@ func (tm *TabletManager) RestoreData(ctx context.Context, logger logutil.Logger, } func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool) error { - // If we're called during init, tmState may not be open yet. - tm.tmState.Open(ctx) - tablet := tm.Tablet() originalType := tablet.Type if err := tm.tmState.ChangeTabletType(ctx, topodatapb.TabletType_RESTORE); err != nil { diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 02112108d8d..df3dc905acf 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -599,6 +599,9 @@ func (tm *TabletManager) handleRestore(ctx context.Context) (bool, error) { // - restoreFromBackup is not set: we initHealthCheck right away if *restoreFromBackup { go func() { + // Open the state manager after restore is done. + defer tm.tmState.Open(ctx) + // restoreFromBackup will just be a regular action // (same as if it was triggered remotely) if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), *waitForBackupInterval, false /* deleteBeforeRestore */); err != nil { diff --git a/go/vt/vttablet/tabletmanager/tm_state.go b/go/vt/vttablet/tabletmanager/tm_state.go index 88c73e1c4c5..39abd73edbe 100644 --- a/go/vt/vttablet/tabletmanager/tm_state.go +++ b/go/vt/vttablet/tabletmanager/tm_state.go @@ -200,15 +200,15 @@ func (ts *tmState) updateLocked(ctx context.Context) { return } - ts.tm.replManager.SetTabletType(ts.tablet.Type) + terTime := logutil.ProtoToTime(ts.tablet.MasterTermStartTime) + reason := ts.canServe(ts.tablet.Type) + ts.publishForDisplay(reason) if err := ts.applyBlacklist(ctx); err != nil { log.Errorf("Cannot update blacklisted tables rule: %v", err) } - terTime := logutil.ProtoToTime(ts.tablet.MasterTermStartTime) - reason := ts.canServe(ts.tablet.Type) - ts.publishForDisplay(reason) + ts.tm.replManager.SetTabletType(ts.tablet.Type) if reason == "" { if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, true); err != nil { @@ -243,12 +243,12 @@ func (ts *tmState) canServe(tabletType topodatapb.TabletType) string { if !topo.IsRunningQueryService(tabletType) { return fmt.Sprintf("not a serving tablet type(%v)", tabletType) } - if tabletType == topodatapb.TabletType_MASTER && ts.isResharding { - return "master tablet with filtered replication on" - } if ts.tabletControls[tabletType] { return "TabletControl.DisableQueryService set" } + if tabletType == topodatapb.TabletType_MASTER && ts.isResharding { + return "master tablet with filtered replication on" + } return "" } diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 04a5dce8d99..31ccc53e729 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -127,9 +127,16 @@ func TestReparentTablet(t *testing.T) { defer master.StopActionLoop(t) // replica loop + // We have to set the settings as replicating. Otherwise, + // the replication manager intervenes and tries to fix replication, + // which ends up making this test unpredictable. + replica.FakeMysqlDaemon.Replicating = true + replica.FakeMysqlDaemon.IOThreadRunning = true replica.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", "FAKE SET MASTER", + "START SLAVE", } replica.StartActionLoop(t, wr) defer replica.StopActionLoop(t) From 017f2a90c409930e413718cbef7031aec819b6b5 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Wed, 15 Jul 2020 22:57:19 -0700 Subject: [PATCH 033/100] vttablet: remove spurious sleep in binlog_watcher It was delaying the tests. Signed-off-by: Sugu Sougoumarane --- go/vt/vttablet/tabletserver/binlog_watcher.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/vt/vttablet/tabletserver/binlog_watcher.go b/go/vt/vttablet/tabletserver/binlog_watcher.go index 59b8e0c90c4..08b29b03370 100644 --- a/go/vt/vttablet/tabletserver/binlog_watcher.go +++ b/go/vt/vttablet/tabletserver/binlog_watcher.go @@ -92,12 +92,11 @@ func (blw *BinlogWatcher) process(ctx context.Context) { err := blw.vs.Stream(ctx, "current", nil, filter, func(events []*binlogdatapb.VEvent) error { return nil }) + log.Infof("ReplicatinWatcher VStream ended: %v, retrying in 5 seconds", err) select { case <-ctx.Done(): return case <-time.After(5 * time.Second): } - log.Infof("ReplicatinWatcher VStream ended: %v, retrying in 5 seconds", err) - time.Sleep(5 * time.Second) } } From 6af2b164bea4bebab875e0c3779800b3b0f30b66 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 17 Jul 2020 18:29:08 -0700 Subject: [PATCH 034/100] vttablet: fix after rebase Signed-off-by: Sugu Sougoumarane --- .../testlib/init_shard_master_test.go | 318 ------------------ .../testlib/planned_reparent_shard_test.go | 2 +- 2 files changed, 1 insertion(+), 319 deletions(-) delete mode 100644 go/vt/wrangler/testlib/init_shard_master_test.go diff --git a/go/vt/wrangler/testlib/init_shard_master_test.go b/go/vt/wrangler/testlib/init_shard_master_test.go deleted file mode 100644 index ccab2d8bb42..00000000000 --- a/go/vt/wrangler/testlib/init_shard_master_test.go +++ /dev/null @@ -1,318 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testlib - -import ( - "fmt" - "strings" - "testing" - "time" - - "golang.org/x/net/context" - - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/mysql/fakesqldb" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/memorytopo" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/vttablet/tmclient" - "vitess.io/vitess/go/vt/wrangler" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -// TestInitMasterShard is the good scenario test, where everything -// works as planned -func TestInitMasterShard(t *testing.T) { - ctx := context.Background() - db := fakesqldb.New(t) - defer db.Close() - ts := memorytopo.NewServer("cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) - vp := NewVtctlPipe(t, ts) - defer vp.Close() - - db.AddQuery("CREATE DATABASE IF NOT EXISTS `vt_test_keyspace`", &sqltypes.Result{}) - - // Create a master, a couple good replicas - master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) - goodReplica1 := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db) - goodReplica2 := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, db) - - // Master: set a plausible ReplicationPosition to return, - // and expect to add entry in _vt.reparent_journal - master.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ - GTIDSet: mysql.MariadbGTIDSet{ - 5: mysql.MariadbGTID{ - Domain: 5, - Server: 456, - Sequence: 890, - }, - }, - } - master.FakeMysqlDaemon.ReadOnly = true - master.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET ALL REPLICATION", - "CREATE DATABASE IF NOT EXISTS _vt", - "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", - "CREATE DATABASE IF NOT EXISTS _vt", - "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", - "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", - } - master.StartActionLoop(t, wr) - defer master.StopActionLoop(t) - - // wait for shard record to be updated with master - startTime := time.Now() - for { - if time.Since(startTime) > 10*time.Second /* timeout */ { - si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) - if err != nil { - t.Fatalf("GetShard(%v, %v) failed: %v", master.Tablet.Keyspace, master.Tablet.Shard, err) - } - if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { - t.Fatalf("ShardInfo should have MasterAlias %v but has %v", topoproto.TabletAliasString(master.Tablet.Alias), topoproto.TabletAliasString(si.MasterAlias)) - } - } - si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) - if err != nil { - t.Fatalf("GetShard(%v, %v) failed: %v", master.Tablet.Keyspace, master.Tablet.Shard, err) - } - if topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { - break - } else { - time.Sleep(100 * time.Millisecond /* interval at which to re-check the shard record */) - } - } - - // Replica1: expect to be reset and re-parented - goodReplica1.FakeMysqlDaemon.ReadOnly = true - goodReplica1.FakeMysqlDaemon.SetReplicationPositionPos = master.FakeMysqlDaemon.CurrentMasterPosition - goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) - goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET ALL REPLICATION", - "FAKE SET SLAVE POSITION", - "FAKE SET MASTER", - "START SLAVE", - } - goodReplica1.StartActionLoop(t, wr) - defer goodReplica1.StopActionLoop(t) - - // Replica2: expect to be re-parented - goodReplica2.FakeMysqlDaemon.ReadOnly = true - goodReplica2.FakeMysqlDaemon.SetReplicationPositionPos = master.FakeMysqlDaemon.CurrentMasterPosition - goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) - goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET ALL REPLICATION", - "FAKE SET SLAVE POSITION", - "FAKE SET MASTER", - "START SLAVE", - } - goodReplica2.StartActionLoop(t, wr) - defer goodReplica2.StopActionLoop(t) - - // run InitShardMaster - // using deprecated flag until it is removed completely. at that time this should be replaced with -wait_replicas_timeout - if err := vp.Run([]string{"InitShardMaster", "-wait_slave_timeout", "10s", master.Tablet.Keyspace + "/" + master.Tablet.Shard, topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { - t.Fatalf("InitShardMaster failed: %v", err) - } - - // check what was run - if master.FakeMysqlDaemon.ReadOnly { - t.Errorf("master was not turned read-write") - } - si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) - if err != nil { - t.Fatalf("GetShard failed: %v", err) - } - if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { - t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias) - } - if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("goodReplica2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - checkSemiSyncEnabled(t, true, true, master) - checkSemiSyncEnabled(t, false, true, goodReplica1, goodReplica2) -} - -// TestInitMasterShardChecks makes sure the safety checks work -func TestInitMasterShardChecks(t *testing.T) { - ctx := context.Background() - db := fakesqldb.New(t) - defer db.Close() - ts := memorytopo.NewServer("cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) - - master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) - - // InitShardMaster with an unknown tablet - if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, &topodatapb.TabletAlias{ - Cell: master.Tablet.Alias.Cell, - Uid: master.Tablet.Alias.Uid + 1, - }, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not in the shard") { - t.Errorf("InitShardMaster with unknown alias returned wrong error: %v", err) - } - - // InitShardMaster with two masters in the shard, no force flag - // (master2 needs to run InitTablet with -force, as it is the second - // master in the same shard) - master2 := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_MASTER, db, ForceInitTablet()) - - if err := wr.InitShardMaster(ctx, master2.Tablet.Keyspace, master2.Tablet.Shard, master2.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the shard master") { - t.Errorf("InitShardMaster with two masters returned wrong error: %v", err) - } - - // InitShardMaster where the new master fails (use force flag - // as we have 2 masters). We force the failure by making the - // SQL commands executed on the master unexpected by the test fixture - master.StartActionLoop(t, wr) - defer master.StopActionLoop(t) - master2.StartActionLoop(t, wr) - defer master2.StopActionLoop(t) - if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "unexpected extra query") { - t.Errorf("InitShardMaster with new master failing in new master InitMaster returned wrong error: %v", err) - } -} - -// TestInitMasterShardOneReplicaFails makes sure that if one replica fails to -// proceed, the action completes anyway -func TestInitMasterShardOneReplicaFails(t *testing.T) { - ctx := context.Background() - db := fakesqldb.New(t) - defer db.Close() - ts := memorytopo.NewServer("cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) - - // Create a master, a couple replicas - master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, db) - goodReplica := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db) - badReplica := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, db) - - // Master: set a plausible ReplicationPosition to return, - // and expect to add entry in _vt.reparent_journal - master.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ - GTIDSet: mysql.MariadbGTIDSet{ - 5: mysql.MariadbGTID{ - Domain: 5, - Server: 456, - Sequence: 890, - }, - }, - } - master.FakeMysqlDaemon.ReadOnly = true - master.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET ALL REPLICATION", - "CREATE DATABASE IF NOT EXISTS _vt", - "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", - "CREATE DATABASE IF NOT EXISTS _vt", - "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", - "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", - } - master.StartActionLoop(t, wr) - defer master.StopActionLoop(t) - - // wait for shard record to be updated with master - startTime := time.Now() - for { - if time.Since(startTime) > 10*time.Second /* timeout */ { - si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) - if err != nil { - t.Fatalf("GetShard(%v, %v) failed: %v", master.Tablet.Keyspace, master.Tablet.Shard, err) - } - if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { - t.Fatalf("ShardInfo should have MasterAlias %v but has %v", topoproto.TabletAliasString(master.Tablet.Alias), topoproto.TabletAliasString(si.MasterAlias)) - } - } - si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) - if err != nil { - t.Fatalf("GetShard(%v, %v) failed: %v", master.Tablet.Keyspace, master.Tablet.Shard, err) - } - if topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { - break - } else { - time.Sleep(100 * time.Millisecond /* interval at which to re-check the shard record */) - } - } - - // goodReplica: expect to be re-parented - goodReplica.FakeMysqlDaemon.ReadOnly = true - goodReplica.FakeMysqlDaemon.SetReplicationPositionPos = master.FakeMysqlDaemon.CurrentMasterPosition - goodReplica.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) - goodReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET ALL REPLICATION", - "FAKE SET SLAVE POSITION", - "FAKE SET MASTER", - "START SLAVE", - } - goodReplica.StartActionLoop(t, wr) - defer goodReplica.StopActionLoop(t) - - // badReplica: insert an error by failing the master hostname input - // on purpose - badReplica.FakeMysqlDaemon.ReadOnly = true - badReplica.FakeMysqlDaemon.SetReplicationPositionPos = master.FakeMysqlDaemon.CurrentMasterPosition - badReplica.FakeMysqlDaemon.SetMasterInput = fmt.Sprintf("%v:%v", "", master.Tablet.MysqlPort) - badReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET ALL REPLICATION", - "FAKE SET SLAVE POSITION", - } - badReplica.StartActionLoop(t, wr) - defer badReplica.StopActionLoop(t) - - // also change the master alias in the Shard object, to make sure it - // is set back. - _, err := ts.UpdateShardFields(ctx, master.Tablet.Keyspace, master.Tablet.Shard, func(si *topo.ShardInfo) error { - // note it's OK to retry this and increment multiple times, - // we just want it to be different - si.MasterAlias.Uid++ - return nil - }) - if err != nil { - t.Fatalf("UpdateShardFields failed: %v", err) - } - - // run InitShardMaster without force, it fails because master is - // changing. - if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, false /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is not the shard master") { - t.Errorf("InitShardMaster with mismatched new master returned wrong error: %v", err) - } - - // run InitShardMaster - if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, true /*force*/, 10*time.Second); err == nil || !strings.Contains(err.Error(), "wrong input for SetMasterCommands") { - t.Errorf("InitShardMaster with one failed replica returned wrong error: %v", err) - } - - // check what was run: master should still be good - if master.FakeMysqlDaemon.ReadOnly { - t.Errorf("master was not turned read-write") - } - si, err := ts.GetShard(ctx, master.Tablet.Keyspace, master.Tablet.Shard) - if err != nil { - t.Fatalf("GetShard failed: %v", err) - } - if !topoproto.TabletAliasEqual(si.MasterAlias, master.Tablet.Alias) { - t.Errorf("unexpected shard master alias, got %v expected %v", si.MasterAlias, master.Tablet.Alias) - } -} diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 9ddaef4f38a..deab660c594 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -112,7 +112,7 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { // run PlannedReparentShard // using deprecated flag until it is removed completely. at that time this should be replaced with -wait_replicas_timeout - err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard}) + err := vp.Run([]string{"PlannedReparentShard", "-wait_replicas_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard}) require.NoError(t, err) // check what was run From 9f2ea0a8bf05fb73725c136003094cba78e308a8 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Sat, 18 Jul 2020 11:41:52 -0700 Subject: [PATCH 035/100] vttablet: stateManager details on status page Signed-off-by: Sugu Sougoumarane --- go/cmd/vttablet/status.go | 1 - go/vt/vttablet/tabletserver/state_manager.go | 70 ++++++++++++++++++-- go/vt/vttablet/tabletserver/status.go | 32 +++++++-- 3 files changed, 92 insertions(+), 11 deletions(-) diff --git a/go/cmd/vttablet/status.go b/go/cmd/vttablet/status.go index 323e2a2405e..22e45bb429f 100644 --- a/go/cmd/vttablet/status.go +++ b/go/cmd/vttablet/status.go @@ -30,7 +30,6 @@ var ( tabletTemplate = `