diff --git a/client/clients/gc/client.go b/client/clients/gc/client.go index e1b4ea05116..e9551680c00 100644 --- a/client/clients/gc/client.go +++ b/client/clients/gc/client.go @@ -18,6 +18,8 @@ import ( "context" "math" "time" + + "github.com/pingcap/errors" ) // Client is the interface for GC client. @@ -32,6 +34,45 @@ type Client interface { GetGCStatesClient(keyspaceID uint32) GCStatesClient } +// GCStatesAPIOptions represents all options for GC states API. +// +//nolint:revive +type GCStatesAPIOptions struct { + ExcludeGCBarriers bool + ExcludeGlobalGCBarriers bool +} + +// DefaultGCStatesAPIOptions returns the default options for GC states API. +func DefaultGCStatesAPIOptions() GCStatesAPIOptions { + return GCStatesAPIOptions{ + ExcludeGCBarriers: true, + ExcludeGlobalGCBarriers: true, + } +} + +// GCStatesAPIOption is the type of option for GC states API. +// +//nolint:revive +type GCStatesAPIOption func(*GCStatesAPIOptions) + +// ExcludeGCBarriers controls whether GetGCState and GetAllKeyspacesGCStates should exclude GC barriers. +// Enabling this can reduce the cost of reading GC states, and is recommended for most use cases. +// When GC barriers are needed, explicitly set false to this option. +func ExcludeGCBarriers(whetherToExclude bool) GCStatesAPIOption { + return func(opts *GCStatesAPIOptions) { + opts.ExcludeGCBarriers = whetherToExclude + } +} + +// ExcludeGlobalGCBarriers controls whether GetAllKeyspacesGCStates should exclude global GC barriers. +// Enabling this can reduce the cost of reading GC states, and is recommended for most use cases. +// When global GC barriers are needed, explicitly set false to this option. +func ExcludeGlobalGCBarriers(whetherToExclude bool) GCStatesAPIOption { + return func(opts *GCStatesAPIOptions) { + opts.ExcludeGlobalGCBarriers = whetherToExclude + } +} + // GCStatesClient is the interface for users to access GC states. // KeyspaceID is already bound to this type when created. // @@ -72,7 +113,7 @@ type GCStatesClient interface { // // When this method is called on a keyspace without keyspace-level GC enabled, it will be equivalent to calling it on // the NullKeyspace. - GetGCState(ctx context.Context) (GCState, error) + GetGCState(ctx context.Context, opts ...GCStatesAPIOption) (GCState, error) // SetGlobalGCBarrier sets a global GC barrier, which blocks GC like how GC barriers do, but is effective for all // keyspaces. This API is designed for some special needs to block GC of all keyspaces. // @@ -105,11 +146,11 @@ type GCStatesClient interface { SetGlobalGCBarrier(ctx context.Context, barrierID string, barrierTS uint64, ttl time.Duration) (*GlobalGCBarrierInfo, error) // DeleteGlobalGCBarrier deletes a global GC barrier. DeleteGlobalGCBarrier(ctx context.Context, barrierID string) (*GlobalGCBarrierInfo, error) - // Get the GC states from all keyspaces. + // GetAllKeyspacesGCStates gets GC states from all keyspaces. // The return value includes both GC states and global GC barriers information. // If a keyspace's state is not ENABLED(like DISABLE/ARCHIVED/TOMBSTONE), that keyspace is skipped. // If a keyspace is not configured with keyspace level GC, its GCState data is missing. - GetAllKeyspacesGCStates(ctx context.Context) (ClusterGCStates, error) + GetAllKeyspacesGCStates(ctx context.Context, opts ...GCStatesAPIOption) (ClusterGCStates, error) } // InternalController is the interface for controlling GC execution. @@ -245,16 +286,89 @@ func (b *GlobalGCBarrierInfo) isExpiredImpl(now time.Time) bool { //nolint:revive type GCState struct { // The ID of the keyspace this GC state belongs to. - KeyspaceID uint32 - TxnSafePoint uint64 - GCSafePoint uint64 - GCBarriers []*GCBarrierInfo + KeyspaceID uint32 + TxnSafePoint uint64 + GCSafePoint uint64 + hasGCBarriers bool + gcBarriers []*GCBarrierInfo +} + +// NewGCStateWithoutGCBarriers creates a GCState instance without GC barriers info. +func NewGCStateWithoutGCBarriers(keyspaceID uint32, txnSafePoint uint64, gcSafePoint uint64) GCState { + return GCState{ + KeyspaceID: keyspaceID, + TxnSafePoint: txnSafePoint, + GCSafePoint: gcSafePoint, + hasGCBarriers: false, + gcBarriers: nil, + } +} + +// NewGCStateWithGCBarriers creates a GCState instance with GC barriers info. +func NewGCStateWithGCBarriers(keyspaceID uint32, txnSafePoint uint64, gcSafePoint uint64, gcBarriers []*GCBarrierInfo) GCState { + return GCState{ + KeyspaceID: keyspaceID, + TxnSafePoint: txnSafePoint, + GCSafePoint: gcSafePoint, + hasGCBarriers: true, + gcBarriers: gcBarriers, + } +} + +// HasGCBarriers returns whether the GCState instance carries GC barriers info. Note that valid GC barriers info +// can be empty. +func (s GCState) HasGCBarriers() bool { + return s.hasGCBarriers +} + +// GetGCBarriers retrieves GC barriers from the GCState instance, or returns an error if it doesn't carry any GC barrier +// info. +func (s GCState) GetGCBarriers() ([]*GCBarrierInfo, error) { + if !s.HasGCBarriers() { + return nil, errors.New("trying to get GC barriers from GCState that doesn't provide GC barriers info. " + + "to retrieve GC barriers, pass false to excludeGCBarriers parameter to GC APIs") + } + return s.gcBarriers, nil } // ClusterGCStates represents the information of the GC state for all keyspaces. type ClusterGCStates struct { // Maps from keyspace id to GC state of that keyspace. - GCStates map[uint32]GCState - // All existing global GC barriers. - GlobalGCBarriers []*GlobalGCBarrierInfo + GCStates map[uint32]GCState + hasGlobalGCBarriers bool + globalGCBarriers []*GlobalGCBarrierInfo +} + +// NewClusterGCStatesWithoutGlobalGCBarriers creates a ClusterGCStates instance without global GC barriers info. +func NewClusterGCStatesWithoutGlobalGCBarriers(gcStates map[uint32]GCState) ClusterGCStates { + return ClusterGCStates{ + GCStates: gcStates, + hasGlobalGCBarriers: false, + globalGCBarriers: nil, + } +} + +// NewClusterGCStatesWithGlobalGCBarriers creates a ClusterGCStates instance with global GC barriers info. +func NewClusterGCStatesWithGlobalGCBarriers(gcStates map[uint32]GCState, globalGCBarriers []*GlobalGCBarrierInfo) ClusterGCStates { + return ClusterGCStates{ + GCStates: gcStates, + hasGlobalGCBarriers: true, + globalGCBarriers: globalGCBarriers, + } +} + +// HasGlobalGCBarriers returns whether the ClusterGCStates instance carries global GC barriers info. Note that valid +// global GC barriers info can be empty. +func (s ClusterGCStates) HasGlobalGCBarriers() bool { + return s.hasGlobalGCBarriers +} + +// GetGlobalGCBarriers retrieves global GC barriers from the ClusterGCStates instance, or returns an error if it doesn't +// carry any global GC barrier info. +func (s ClusterGCStates) GetGlobalGCBarriers() ([]*GlobalGCBarrierInfo, error) { + if !s.HasGlobalGCBarriers() { + return nil, errors.New("trying to get global GC barriers from ClusterGCStates that doesn't provide global GC barriers info. " + + "to retrieve global GC barriers, pass false to excludeGlobalGCBarriers parameter to GC APIs") + } + return s.globalGCBarriers, nil } diff --git a/client/clients/gc/client_test.go b/client/clients/gc/client_test.go index e96b7c83999..0361d2fa2c3 100644 --- a/client/clients/gc/client_test.go +++ b/client/clients/gc/client_test.go @@ -58,3 +58,42 @@ func TestGCBarrierInfoExpiration(t *testing.T) { re.False(b1.isExpiredImpl(now)) re.False(b1.isExpiredImpl(now.Add(time.Hour * 24 * 365 * 10))) } + +func TestGCStateAccessors(t *testing.T) { + re := require.New(t) + + state := NewGCStateWithoutGCBarriers(1, 2, 3) + re.False(state.HasGCBarriers()) + barriers, err := state.GetGCBarriers() + re.Error(err) + re.Nil(barriers) + + state = NewGCStateWithGCBarriers(1, 2, 3, []*GCBarrierInfo{ + NewGCBarrierInfo("b1", 4, time.Second, time.Now()), + }) + re.True(state.HasGCBarriers()) + barriers, err = state.GetGCBarriers() + re.NoError(err) + re.Len(barriers, 1) + re.Equal("b1", barriers[0].BarrierID) +} + +func TestClusterGCStatesAccessors(t *testing.T) { + re := require.New(t) + + state := NewGCStateWithoutGCBarriers(1, 2, 3) + clusterState := NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]GCState{1: state}) + re.False(clusterState.HasGlobalGCBarriers()) + barriers, err := clusterState.GetGlobalGCBarriers() + re.Error(err) + re.Nil(barriers) + + clusterState = NewClusterGCStatesWithGlobalGCBarriers(map[uint32]GCState{1: state}, []*GlobalGCBarrierInfo{ + NewGlobalGCBarrierInfo("b1", 4, time.Second, time.Now()), + }) + re.True(clusterState.HasGlobalGCBarriers()) + barriers, err = clusterState.GetGlobalGCBarriers() + re.NoError(err) + re.Len(barriers, 1) + re.Equal("b1", barriers[0].BarrierID) +} diff --git a/client/gc_client.go b/client/gc_client.go index 9d813fd0180..916cef3ffe3 100644 --- a/client/gc_client.go +++ b/client/gc_client.go @@ -295,7 +295,7 @@ func (c gcStatesClient) DeleteGCBarrier(ctx context.Context, barrierID string) ( } // GetGCState gets the current GC state. -func (c gcStatesClient) GetGCState(ctx context.Context) (gc.GCState, error) { +func (c gcStatesClient) GetGCState(ctx context.Context, opts ...gc.GCStatesAPIOption) (gc.GCState, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetGCState", opentracing.ChildOf(span.Context())) defer span.Finish() @@ -303,11 +303,17 @@ func (c gcStatesClient) GetGCState(ctx context.Context) (gc.GCState, error) { start := time.Now() defer func() { metrics.CmdDurationGetGCState.Observe(time.Since(start).Seconds()) }() + options := gc.DefaultGCStatesAPIOptions() + for _, opt := range opts { + opt(&options) + } + ctx, cancel := context.WithTimeout(ctx, c.client.inner.option.Timeout) defer cancel() req := &pdpb.GetGCStateRequest{ - Header: c.client.requestHeader(), - KeyspaceScope: wrapKeyspaceScope(c.keyspaceID), + Header: c.client.requestHeader(), + KeyspaceScope: wrapKeyspaceScope(c.keyspaceID), + ExcludeGcBarriers: options.ExcludeGCBarriers, } protoClient, ctx := c.client.getClientAndContext(ctx) if protoClient == nil { @@ -319,24 +325,23 @@ func (c gcStatesClient) GetGCState(ctx context.Context) (gc.GCState, error) { } gcState := resp.GetGcState() - return pbToGCState(gcState, start), nil + return pbToGCState(gcState, start, options.ExcludeGCBarriers), nil } -func pbToGCState(pb *pdpb.GCState, reqStartTime time.Time) gc.GCState { +func pbToGCState(pb *pdpb.GCState, reqStartTime time.Time, excludeGCBarriers bool) gc.GCState { keyspaceID := constants.NullKeyspaceID if pb.KeyspaceScope != nil { keyspaceID = pb.KeyspaceScope.KeyspaceId } + if excludeGCBarriers { + return gc.NewGCStateWithoutGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint()) + } + gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) for _, b := range pb.GetGcBarriers() { gcBarriers = append(gcBarriers, pbToGCBarrierInfo(b, reqStartTime)) } - return gc.GCState{ - KeyspaceID: keyspaceID, - TxnSafePoint: pb.GetTxnSafePoint(), - GCSafePoint: pb.GetGcSafePoint(), - GCBarriers: gcBarriers, - } + return gc.NewGCStateWithGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint(), gcBarriers) } // SetGlobalGCBarrier sets (creates or updates) a global GC barrier. @@ -394,7 +399,7 @@ func (c gcStatesClient) DeleteGlobalGCBarrier(ctx context.Context, barrierID str } // GetAllKeyspacesGCStates gets the GC states from all keyspaces. -func (c gcStatesClient) GetAllKeyspacesGCStates(ctx context.Context) (gc.ClusterGCStates, error) { +func (c gcStatesClient) GetAllKeyspacesGCStates(ctx context.Context, opts ...gc.GCStatesAPIOption) (gc.ClusterGCStates, error) { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span = span.Tracer().StartSpan("pdclient.GetAllKeyspacesGCState", opentracing.ChildOf(span.Context())) defer span.Finish() @@ -402,10 +407,17 @@ func (c gcStatesClient) GetAllKeyspacesGCStates(ctx context.Context) (gc.Cluster start := time.Now() defer func() { metrics.CmdDurationGetAllKeyspacesGCStates.Observe(time.Since(start).Seconds()) }() + options := gc.DefaultGCStatesAPIOptions() + for _, opt := range opts { + opt(&options) + } + ctx, cancel := context.WithTimeout(ctx, c.client.inner.option.Timeout) defer cancel() req := &pdpb.GetAllKeyspacesGCStatesRequest{ - Header: c.client.requestHeader(), + Header: c.client.requestHeader(), + ExcludeGcBarriers: options.ExcludeGCBarriers, + ExcludeGlobalGcBarriers: options.ExcludeGlobalGCBarriers, } protoClient, ctx := c.client.getClientAndContext(ctx) if protoClient == nil { @@ -417,8 +429,7 @@ func (c gcStatesClient) GetAllKeyspacesGCStates(ctx context.Context) (gc.Cluster return gc.ClusterGCStates{}, err } - var ret gc.ClusterGCStates - ret.GCStates = make(map[uint32]gc.GCState, len(resp.GetGcStates())) + gcStates := make(map[uint32]gc.GCState, len(resp.GetGcStates())) for _, state := range resp.GetGcStates() { var keyspaceID uint32 if state.KeyspaceScope == nil { @@ -426,10 +437,15 @@ func (c gcStatesClient) GetAllKeyspacesGCStates(ctx context.Context) (gc.Cluster } else { keyspaceID = state.KeyspaceScope.KeyspaceId } - ret.GCStates[keyspaceID] = pbToGCState(state, start) + gcStates[keyspaceID] = pbToGCState(state, start, options.ExcludeGCBarriers) } + if options.ExcludeGlobalGCBarriers { + return gc.NewClusterGCStatesWithoutGlobalGCBarriers(gcStates), nil + } + + globalGCBarriers := make([]*gc.GlobalGCBarrierInfo, 0, len(resp.GetGlobalGcBarriers())) for _, barrier := range resp.GetGlobalGcBarriers() { - ret.GlobalGCBarriers = append(ret.GlobalGCBarriers, pbToGlobalGCBarrierInfo(barrier, start)) + globalGCBarriers = append(globalGCBarriers, pbToGlobalGCBarrierInfo(barrier, start)) } - return ret, nil + return gc.NewClusterGCStatesWithGlobalGCBarriers(gcStates, globalGCBarriers), nil } diff --git a/client/go.mod b/client/go.mod index b8181aebbde..df7c208681a 100644 --- a/client/go.mod +++ b/client/go.mod @@ -10,7 +10,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 + github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/stretchr/testify v1.9.0 diff --git a/client/go.sum b/client/go.sum index 6a12f558c40..2916b077851 100644 --- a/client/go.sum +++ b/client/go.sum @@ -53,8 +53,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 h1:oteLtLuoWZN3uvfH836U0IIJ+s3UOk11q7GaQ0Tk+wc= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 h1:n6QWAac97mv2NJhn17iFPFnsE5fMgtPLNmsGZeqq78o= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/go.mod b/go.mod index a3a993b884e..ff8905d4680 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 + github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/metering_sdk v0.0.0-20260203082503-b9f282339654 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 diff --git a/go.sum b/go.sum index a8c6345a634..dcdd0d8a419 100644 --- a/go.sum +++ b/go.sum @@ -480,8 +480,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 h1:oteLtLuoWZN3uvfH836U0IIJ+s3UOk11q7GaQ0Tk+wc= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 h1:n6QWAac97mv2NJhn17iFPFnsE5fMgtPLNmsGZeqq78o= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/pkg/gc/gc_state_manager.go b/pkg/gc/gc_state_manager.go index 1aed4b98fab..228fbc74d1b 100644 --- a/pkg/gc/gc_state_manager.go +++ b/pkg/gc/gc_state_manager.go @@ -118,7 +118,8 @@ type GCStateManager struct { cfg config.PDServerConfig keyspaceManager *keyspace.Manager - allKeyspacesGCStatesSingleFlight *syncutil.OrderedSingleFlight[map[uint32]GCState] + allKeyspacesGCStatesSingleFlight *syncutil.OrderedSingleFlight[map[uint32]GCState] + allKeyspacesGCStatesExcludeGCBarriersSingleFlight *syncutil.OrderedSingleFlight[map[uint32]GCState] } // NewGCStateManager creates a GCStateManager of GC and services. @@ -128,6 +129,7 @@ func NewGCStateManager(store endpoint.GCStateProvider, cfg config.PDServerConfig cfg: cfg, keyspaceManager: keyspaceManager, allKeyspacesGCStatesSingleFlight: syncutil.NewOrderedSingleFlight[map[uint32]GCState](), + allKeyspacesGCStatesExcludeGCBarriersSingleFlight: syncutil.NewOrderedSingleFlight[map[uint32]GCState](), } } @@ -632,7 +634,7 @@ func (m *GCStateManager) deleteGCBarrierImpl(ctx context.Context, keyspaceID uin // it's running in a in-transaction context. // The parameter `keyspaceID` is expected to be either the NullKeyspaceID or the ID of a keyspace that has // keyspace-level GC enabled. Otherwise, the result would be undefined. -func (m *GCStateManager) getGCStateInTransaction(keyspaceID uint32, _ *endpoint.GCStateWriteBatch) (GCState, error) { +func (m *GCStateManager) getGCStateInTransaction(keyspaceID uint32, excludeGCBarriers bool, _ *endpoint.GCStateWriteBatch) (GCState, error) { result := GCState{ KeyspaceID: keyspaceID, } @@ -653,16 +655,18 @@ func (m *GCStateManager) getGCStateInTransaction(keyspaceID uint32, _ *endpoint. return GCState{}, err } - result.GCBarriers, err = m.gcMetaStorage.LoadAllGCBarriers(keyspaceID) - if err != nil { - return GCState{}, err - } + if !excludeGCBarriers { + result.GCBarriers, err = m.gcMetaStorage.LoadAllGCBarriers(keyspaceID) + if err != nil { + return GCState{}, err + } - // Remove GC barrier whose barrierID is "gc_worker", which is only exists for providing compatibility with the old - // versions. - result.GCBarriers = slices.DeleteFunc(result.GCBarriers, func(b *endpoint.GCBarrier) bool { - return b.BarrierID == keypath.GCWorkerServiceSafePointID - }) + // Remove GC barrier whose barrierID is "gc_worker", which is only exists for providing compatibility with the old + // versions. + result.GCBarriers = slices.DeleteFunc(result.GCBarriers, func(b *endpoint.GCBarrier) bool { + return b.BarrierID == keypath.GCWorkerServiceSafePointID + }) + } return result, nil } @@ -671,7 +675,7 @@ func (m *GCStateManager) getGCStateInTransaction(keyspaceID uint32, _ *endpoint. // // When this method is called on a keyspace without keyspace-level GC enabled, it will be equivalent to calling it on // the NullKeyspace. -func (m *GCStateManager) GetGCState(keyspaceID uint32) (GCState, error) { +func (m *GCStateManager) GetGCState(keyspaceID uint32, excludeGCBarriers bool) (GCState, error) { keyspaceID, _, err := m.redirectKeyspace(keyspaceID, true) if err != nil { return GCState{}, err @@ -682,7 +686,7 @@ func (m *GCStateManager) GetGCState(keyspaceID uint32) (GCState, error) { var result GCState err = m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error { var err1 error - result, err1 = m.getGCStateInTransaction(keyspaceID, wb) + result, err1 = m.getGCStateInTransaction(keyspaceID, excludeGCBarriers, wb) return err1 }) @@ -698,15 +702,23 @@ func (m *GCStateManager) GetGCState(keyspaceID uint32) (GCState, error) { // the caller should NEVER change the content of the returned result and error. It's guaranteed that the returned result // must be fetched AFTER the beginning of the current invocation, and it never reuses the result of invocations that // started earlier than the current one. -func (m *GCStateManager) GetAllKeyspacesGCStates(ctx context.Context) (map[uint32]GCState, error) { - return m.allKeyspacesGCStatesSingleFlight.Do(ctx, func(execCtx context.Context) (map[uint32]GCState, error) { - result, err := m.getAllKeyspacesGCStatesImpl(execCtx) +func (m *GCStateManager) GetAllKeyspacesGCStates(ctx context.Context, excludeGCBarriers bool) (map[uint32]GCState, error) { + if !excludeGCBarriers { + return m.allKeyspacesGCStatesSingleFlight.Do(ctx, func(execCtx context.Context) (map[uint32]GCState, error) { + result, err := m.getAllKeyspacesGCStatesImpl(execCtx, excludeGCBarriers) + failpoint.Inject("onGetAllKeyspacesGCStatesFinish", func() {}) + return result, err + }) + } + + return m.allKeyspacesGCStatesExcludeGCBarriersSingleFlight.Do(ctx, func(execCtx context.Context) (map[uint32]GCState, error) { + result, err := m.getAllKeyspacesGCStatesImpl(execCtx, excludeGCBarriers) failpoint.Inject("onGetAllKeyspacesGCStatesFinish", func() {}) return result, err }) } -func (m *GCStateManager) getAllKeyspacesGCStatesImpl(ctx context.Context) (map[uint32]GCState, error) { +func (m *GCStateManager) getAllKeyspacesGCStatesImpl(ctx context.Context, excludeGCBarriers bool) (map[uint32]GCState, error) { failpoint.InjectCall("onGetAllKeyspacesGCStatesStart") mutexLocked := false @@ -732,7 +744,7 @@ func (m *GCStateManager) getAllKeyspacesGCStatesImpl(ctx context.Context) (map[u results := make(map[uint32]GCState) lock() err := m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error { - nullKeyspaceState, err1 := m.getGCStateInTransaction(constant.NullKeyspaceID, wb) + nullKeyspaceState, err1 := m.getGCStateInTransaction(constant.NullKeyspaceID, excludeGCBarriers, wb) if err1 != nil { return err1 } @@ -774,7 +786,7 @@ func (m *GCStateManager) getAllKeyspacesGCStatesImpl(ctx context.Context) (map[u lock() err = m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error { - state, err1 := m.getGCStateInTransaction(keyspaceMeta.Id, wb) + state, err1 := m.getGCStateInTransaction(keyspaceMeta.Id, excludeGCBarriers, wb) if err1 != nil { return err1 } diff --git a/pkg/gc/gc_state_manager_test.go b/pkg/gc/gc_state_manager_test.go index a35e113546f..56ee3d5ca5b 100644 --- a/pkg/gc/gc_state_manager_test.go +++ b/pkg/gc/gc_state_manager_test.go @@ -230,7 +230,7 @@ func (s *gcStateManagerTestSuite) TearDownTest() { func (s *gcStateManagerTestSuite) checkTxnSafePoint(keyspaceID uint32, expectedTxnSafePoint uint64) { re := s.Require() - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, true) re.NoError(err) re.Equal(expectedTxnSafePoint, state.TxnSafePoint) } @@ -358,7 +358,7 @@ func (s *gcStateManagerTestSuite) TestAdvanceGCSafePointBasic() { re := s.Require() checkGCSafePoint := func(keyspaceID uint32, expectedGCSafePoint uint64) { - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, true) re.NoError(err) re.Equal(expectedGCSafePoint, state.GCSafePoint) } @@ -484,7 +484,7 @@ func (s *gcStateManagerTestSuite) TestCompatibleUpdateGCSafePointSequentiallyWit func (s *gcStateManagerTestSuite) TestCompatibleUpdateGCSafePointSequentiallyWithNewLoad() { for _, keyspaceID := range s.keyspacePresets.manageable { s.testCompatibleGCSafePointUpdateSequentiallyImpl(keyspaceID, func(keyspaceID uint32) (uint64, error) { - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, true) if err != nil { return 0, err } @@ -661,7 +661,7 @@ func (s *gcStateManagerTestSuite) TestCompatibleServiceGCSafePointRoundingTTL() re.NoError(err) re.True(updated) - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Len(state.GCBarriers, 1) re.Equal("svc1", state.GCBarriers[0].BarrierID) @@ -673,7 +673,7 @@ func (s *gcStateManagerTestSuite) TestCompatibleServiceGCSafePointRoundingTTL() re.NoError(err) re.True(updated) - state, err = s.manager.GetGCState(keyspaceID) + state, err = s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Len(state.GCBarriers, 1) re.Equal("svc1", state.GCBarriers[0].BarrierID) @@ -685,7 +685,7 @@ func (s *gcStateManagerTestSuite) TestCompatibleServiceGCSafePointRoundingTTL() func (s *gcStateManagerTestSuite) getGCBarrier(keyspaceID uint32, barrierID string) *endpoint.GCBarrier { re := s.Require() - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) idx := slices.IndexFunc(state.GCBarriers, func(b *endpoint.GCBarrier) bool { return b.BarrierID == barrierID @@ -698,7 +698,7 @@ func (s *gcStateManagerTestSuite) getGCBarrier(keyspaceID uint32, barrierID stri func (s *gcStateManagerTestSuite) getAllGCBarriers(keyspaceID uint32) []*endpoint.GCBarrier { re := s.Require() - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) return state.GCBarriers } @@ -1733,7 +1733,7 @@ func (s *gcStateManagerTestSuite) TestRedirectKeyspace() { // Check all public methods that accepts keyspaceID are all correctly redirected. testedFunc := []func(keyspaceID uint32) error{ func(keyspaceID uint32) error { - _, err1 := s.manager.GetGCState(keyspaceID) + _, err1 := s.manager.GetGCState(keyspaceID, false) return errors.AddStack(err1) }, func(keyspaceID uint32) error { @@ -1789,14 +1789,14 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { // Check the result of GetAllKeyspaceGCStates and GetGCState are matching. checkAllKeyspaceGCStates := func() { - allStates, err := s.manager.GetAllKeyspacesGCStates(context.Background()) + allStates, err := s.manager.GetAllKeyspacesGCStates(context.Background(), false) re.NoError(err) re.Len(allStates, len(s.keyspacePresets.all)) for keyspaceID, state := range allStates { if slices.Contains(s.keyspacePresets.manageable, keyspaceID) { re.Equal(keyspaceID, state.KeyspaceID) - s, err := s.manager.GetGCState(keyspaceID) + s, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Equal(s, state) } else { @@ -1808,7 +1808,7 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { } for _, keyspaceID := range s.keyspacePresets.manageable { - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Equal(keyspaceID, state.KeyspaceID) if keyspaceID == constant.NullKeyspaceID { @@ -1822,7 +1822,7 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { } for _, keyspaceID := range slices.Concat(s.keyspacePresets.unmanageable, s.keyspacePresets.nullSynonyms) { - state, err := s.manager.GetGCState(keyspaceID) + state, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Equal(constant.NullKeyspaceID, state.KeyspaceID) re.False(state.IsKeyspaceLevel) @@ -1832,7 +1832,7 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { } for _, keyspaceID := range s.keyspacePresets.notExisting { - _, err := s.manager.GetGCState(keyspaceID) + _, err := s.manager.GetGCState(keyspaceID, false) re.Error(err) re.ErrorIs(err, errs.ErrKeyspaceNotFound) } @@ -1859,7 +1859,7 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { _, err = s.manager.SetGCBarrier(2, "b3", 60, time.Duration(math.MaxInt64), now) re.NoError(err) - state, err := s.manager.GetGCState(constant.NullKeyspaceID) + state, err := s.manager.GetGCState(constant.NullKeyspaceID, false) re.NoError(err) re.Equal(constant.NullKeyspaceID, state.KeyspaceID) re.False(state.IsKeyspaceLevel) @@ -1870,7 +1870,7 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { endpoint.NewGCBarrier("b2", 25, ptime(now.Add(time.Hour*2))), }, state.GCBarriers) - state, err = s.manager.GetGCState(2) + state, err = s.manager.GetGCState(2, false) re.NoError(err) re.Equal(uint32(2), state.KeyspaceID) re.True(state.IsKeyspaceLevel) @@ -1881,9 +1881,49 @@ func (s *gcStateManagerTestSuite) TestGetGCState() { endpoint.NewGCBarrier("b3", 60, nil), }, state.GCBarriers) + // Check excluding GC barriers + state, err = s.manager.GetGCState(2, true) + re.NoError(err) + re.Equal(uint32(2), state.KeyspaceID) + re.True(state.IsKeyspaceLevel) + re.Equal(uint64(50), state.TxnSafePoint) + re.Equal(uint64(45), state.GCSafePoint) + re.Empty(state.GCBarriers) + checkAllKeyspaceGCStates() } +func (s *gcStateManagerTestSuite) TestGetAllKeyspacesGCStatesExcludingGCBarriers() { + re := s.Require() + + for _, keyspaceID := range s.keyspacePresets.manageable { + _, err := s.manager.SetGCBarrier(keyspaceID, fmt.Sprintf("b1-%d", keyspaceID), 25, time.Hour, time.Now()) + re.NoError(err) + } + + allStates, err := s.manager.GetAllKeyspacesGCStates(context.Background(), false) + re.NoError(err) + re.Len(allStates, len(s.keyspacePresets.all)) + + for _, keyspaceID := range s.keyspacePresets.manageable { + state, ok := allStates[keyspaceID] + re.True(ok) + re.Len(state.GCBarriers, 1) + re.Equal(fmt.Sprintf("b1-%d", keyspaceID), state.GCBarriers[0].BarrierID) + re.Equal(uint64(25), state.GCBarriers[0].BarrierTS) + } + + allStates, err = s.manager.GetAllKeyspacesGCStates(context.Background(), true) + re.NoError(err) + re.Len(allStates, len(s.keyspacePresets.all)) + + for _, keyspaceID := range s.keyspacePresets.manageable { + state, ok := allStates[keyspaceID] + re.True(ok) + re.Empty(state.GCBarriers) + } +} + func (s *gcStateManagerTestSuite) TestGetAllKeyspacesMaxTxnSafePoint() { re := s.Require() @@ -1986,22 +2026,22 @@ func (s *gcStateManagerTestSuite) testDowngradeCompatibility(keyspaceID uint32) re.Equal(uint64(25), s.getLegacyGCWorkerServiceSafePoint(keyspaceID).SafePoint) // Not visible by GetGCStates or GetAllKeyspacesGCStates. - gcState, err := s.manager.GetGCState(keyspaceID) + gcState, err := s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Empty(gcState.GCBarriers) - allGCStates, err := s.manager.GetAllKeyspacesGCStates(context.Background()) + allGCStates, err := s.manager.GetAllKeyspacesGCStates(context.Background(), false) re.NoError(err) re.Empty(allGCStates[keyspaceID].GCBarriers) // And it works correctly when there are other valid GC barriers. _, err = s.manager.SetGCBarrier(keyspaceID, "b1", 40, time.Hour, now) re.NoError(err) - gcState, err = s.manager.GetGCState(keyspaceID) + gcState, err = s.manager.GetGCState(keyspaceID, false) re.NoError(err) re.Len(gcState.GCBarriers, 1) re.Equal("b1", gcState.GCBarriers[0].BarrierID) re.Equal(uint64(40), gcState.GCBarriers[0].BarrierTS) - allGCStates, err = s.manager.GetAllKeyspacesGCStates(context.Background()) + allGCStates, err = s.manager.GetAllKeyspacesGCStates(context.Background(), false) re.NoError(err) re.Len(allGCStates[keyspaceID].GCBarriers, 1) re.Equal("b1", allGCStates[keyspaceID].GCBarriers[0].BarrierID) @@ -2036,7 +2076,7 @@ func (s *gcStateManagerTestSuite) TestGetAllKeyspacesGCStatesConcurrentCallShari ch := make(chan result, 10) callOnce := func() { - gcStates, err := s.manager.GetAllKeyspacesGCStates(context.Background()) + gcStates, err := s.manager.GetAllKeyspacesGCStates(context.Background(), false) ch <- result{gcStates: gcStates, err: err} } @@ -2091,6 +2131,128 @@ func (s *gcStateManagerTestSuite) TestGetAllKeyspacesGCStatesConcurrentCallShari re.Equal(int64(2), executionCount.Load()) } +func (s *gcStateManagerTestSuite) TestGetAllKeyspacesGCStatesDifferentParametersCallsDoNotShareResult() { + re := s.Require() + + const keyspaceID = uint32(2) + _, err := s.manager.SetGCBarrier(keyspaceID, "b1", 25, time.Hour, time.Now()) + re.NoError(err) + + type result struct { + caller string + excludeGCBarriers bool + gcStates map[uint32]GCState + err error + } + + runScenario := func(firstExcludeGCBarriers bool) { + var executionCount atomic.Int64 + finishFailpointEnabled := true + + fullExecBefore := s.manager.allKeyspacesGCStatesSingleFlight.ExecCount() + excludeExecBefore := s.manager.allKeyspacesGCStatesExcludeGCBarriersSingleFlight.ExecCount() + + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/gc/onGetAllKeyspacesGCStatesFinish", "pause")) + re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/gc/onGetAllKeyspacesGCStatesStart", func() { + executionCount.Add(1) + })) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/onGetAllKeyspacesGCStatesStart")) + if finishFailpointEnabled { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/onGetAllKeyspacesGCStatesFinish")) + } + }() + + ch := make(chan result, 3) + callOnce := func(caller string, excludeGCBarriers bool) { + gcStates, err := s.manager.GetAllKeyspacesGCStates(context.Background(), excludeGCBarriers) + ch <- result{ + caller: caller, + excludeGCBarriers: excludeGCBarriers, + gcStates: gcStates, + err: err, + } + } + + go callOnce("first", firstExcludeGCBarriers) + + select { + case res := <-ch: + re.FailNowf("failpoint not taking effect to block the first invocation to GetAllKeyspacesGCStates", "caller: %s, excludeGCBarriers: %v, result: %v, err: %v", res.caller, res.excludeGCBarriers, res.gcStates, res.err) + case <-time.After(200 * time.Millisecond): + } + re.Equal(int64(1), executionCount.Load()) + + // The second call uses the other excludeGCBarriers value, so with the + // correct implementation it must start a separate execution immediately. + go callOnce("second", !firstExcludeGCBarriers) + + // The third call uses the same parameter as the first one. If the two + // parameter variants were incorrectly routed to a single OrderedSingleFlight + // instance, this third call could be merged into the second call's pending + // batch and receive a result for the wrong parameter. + go callOnce("third", firstExcludeGCBarriers) + + deadline := time.Now().Add(time.Second) + for executionCount.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + re.Equal( + int64(2), + executionCount.Load(), + "calls with different excludeGCBarriers values should use different OrderedSingleFlight instances", + ) + + select { + case res := <-ch: + re.FailNowf("expected all invocations to stay blocked before finish failpoint is released", "caller: %s, excludeGCBarriers: %v, result: %v, err: %v", res.caller, res.excludeGCBarriers, res.gcStates, res.err) + case <-time.After(100 * time.Millisecond): + } + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/onGetAllKeyspacesGCStatesFinish")) + finishFailpointEnabled = false + + fullCallCount := 0 + excludeCallCount := 0 + for range 3 { + var res result + select { + case res = <-ch: + case <-time.After(time.Second): + re.FailNow("GetAllKeyspacesGCStates blocked while expected to return") + } + re.NoError(res.err) + if res.excludeGCBarriers { + excludeCallCount++ + re.Empty(res.gcStates[keyspaceID].GCBarriers) + } else { + fullCallCount++ + re.Len(res.gcStates[keyspaceID].GCBarriers, 1) + re.Equal("b1", res.gcStates[keyspaceID].GCBarriers[0].BarrierID) + re.Equal(uint64(25), res.gcStates[keyspaceID].GCBarriers[0].BarrierTS) + } + } + + expectedFullCalls := 1 + expectedExcludeCalls := 2 + expectedFullExecs := fullExecBefore + 1 + expectedExcludeExecs := excludeExecBefore + 2 + if !firstExcludeGCBarriers { + expectedFullCalls = 2 + expectedExcludeCalls = 1 + expectedFullExecs = fullExecBefore + 2 + expectedExcludeExecs = excludeExecBefore + 1 + } + re.Equal(expectedFullCalls, fullCallCount) + re.Equal(expectedExcludeCalls, excludeCallCount) + re.Equal(expectedFullExecs, s.manager.allKeyspacesGCStatesSingleFlight.ExecCount()) + re.Equal(expectedExcludeExecs, s.manager.allKeyspacesGCStatesExcludeGCBarriersSingleFlight.ExecCount()) + } + + runScenario(false) + runScenario(true) +} + func TestGetAllKeysapcesGCStatesOnTooManyKeyspaces(t *testing.T) { re := require.New(t) @@ -2107,7 +2269,7 @@ func TestGetAllKeysapcesGCStatesOnTooManyKeyspaces(t *testing.T) { clean() }() - gcStates, err := gcStateManager.GetAllKeyspacesGCStates(context.Background()) + gcStates, err := gcStateManager.GetAllKeyspacesGCStates(context.Background(), false) re.Len(gcStates, totalKeyspaces+2) // Including the null keyspace, the default keyspace or the system keyspace. re.NoError(err) @@ -2206,14 +2368,14 @@ func benchmarkGetAllKeyspacesGCStatesImpl(b *testing.B, keyspacesCount int, para b.ResetTimer() if parallelism == 0 { for range b.N { - _, err := gcStateManager.GetAllKeyspacesGCStates(context.Background()) + _, err := gcStateManager.GetAllKeyspacesGCStates(context.Background(), false) re.NoError(err) } } else { b.SetParallelism(parallelism) b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := gcStateManager.GetAllKeyspacesGCStates(context.Background()) + _, err := gcStateManager.GetAllKeyspacesGCStates(context.Background(), false) re.NoError(err) } }) diff --git a/server/api/service_gc_safepoint.go b/server/api/service_gc_safepoint.go index d978dabd3cd..cc8640be8a7 100644 --- a/server/api/service_gc_safepoint.go +++ b/server/api/service_gc_safepoint.go @@ -59,7 +59,7 @@ type ListServiceGCSafepoint struct { // @Router /gc/safepoint [get] func (h *serviceGCSafepointHandler) GetGCSafePoint(w http.ResponseWriter, _ *http.Request) { gcStateManager := h.svr.GetGCStateManager() - gcState, err := gcStateManager.GetGCState(constant.NullKeyspaceID) + gcState, err := gcStateManager.GetGCState(constant.NullKeyspaceID, false) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return diff --git a/server/apiv2/handlers/safe_point.go b/server/apiv2/handlers/safe_point.go index a40e123a6ac..3c225409126 100644 --- a/server/apiv2/handlers/safe_point.go +++ b/server/apiv2/handlers/safe_point.go @@ -61,7 +61,7 @@ func LoadGCSafePoint(c *gin.Context) { c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) return } - gcState, err := manager.GetGCState(uint32(keyspaceID)) + gcState, err := manager.GetGCState(uint32(keyspaceID), true) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return diff --git a/server/gc_service.go b/server/gc_service.go index 7c4149acc91..fe7b3f83e79 100644 --- a/server/gc_service.go +++ b/server/gc_service.go @@ -440,7 +440,7 @@ func (s *GrpcServer) GetAllGCSafePointV2(ctx context.Context, request *pdpb.GetA return &pdpb.GetAllGCSafePointV2Response{Header: grpcutil.NotBootstrappedHeader()}, nil } - gcStates, err := s.gcStateManager.GetAllKeyspacesGCStates(ctx) + gcStates, err := s.gcStateManager.GetAllKeyspacesGCStates(ctx, true) if err != nil { return &pdpb.GetAllGCSafePointV2Response{ Header: grpcutil.WrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -704,7 +704,7 @@ func (s *GrpcServer) GetGCState(ctx context.Context, request *pdpb.GetGCStateReq return &pdpb.GetGCStateResponse{Header: grpcutil.NotBootstrappedHeader()}, nil } - gcState, err := s.gcStateManager.GetGCState(getKeyspaceID(request.GetKeyspaceScope())) + gcState, err := s.gcStateManager.GetGCState(getKeyspaceID(request.GetKeyspaceScope()), request.GetExcludeGcBarriers()) if err != nil { return &pdpb.GetGCStateResponse{ Header: grpcutil.WrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -740,7 +740,7 @@ func (s *GrpcServer) GetAllKeyspacesGCStates(ctx context.Context, request *pdpb. return &pdpb.GetAllKeyspacesGCStatesResponse{Header: grpcutil.NotBootstrappedHeader()}, nil } - gcStates, err := s.gcStateManager.GetAllKeyspacesGCStates(ctx) + gcStates, err := s.gcStateManager.GetAllKeyspacesGCStates(ctx, request.GetExcludeGcBarriers()) if err != nil { return &pdpb.GetAllKeyspacesGCStatesResponse{ Header: grpcutil.WrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -752,15 +752,18 @@ func (s *GrpcServer) GetAllKeyspacesGCStates(ctx context.Context, request *pdpb. gcStatesPb = append(gcStatesPb, gcStateToProto(gcState, now)) } - globalBarriers, err := s.gcStateManager.LoadAllGlobalGCBarriers() - if err != nil { - return &pdpb.GetAllKeyspacesGCStatesResponse{ - Header: grpcutil.WrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), - }, nil - } - gcBarriersPb := make([]*pdpb.GlobalGCBarrierInfo, 0, len(globalBarriers)) - for _, barrier := range globalBarriers { - gcBarriersPb = append(gcBarriersPb, globalGCBarrierToProto(barrier, now)) + var gcBarriersPb []*pdpb.GlobalGCBarrierInfo + if !request.GetExcludeGlobalGcBarriers() { + globalBarriers, err := s.gcStateManager.LoadAllGlobalGCBarriers() + if err != nil { + return &pdpb.GetAllKeyspacesGCStatesResponse{ + Header: grpcutil.WrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), + }, nil + } + gcBarriersPb = make([]*pdpb.GlobalGCBarrierInfo, 0, len(globalBarriers)) + for _, barrier := range globalBarriers { + gcBarriersPb = append(gcBarriersPb, globalGCBarrierToProto(barrier, now)) + } } return &pdpb.GetAllKeyspacesGCStatesResponse{ diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 768b10c4a36..a7870afb00f 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -2070,10 +2070,12 @@ func (*clientStatefulTestSuite) waitForGCBarrierExpiring(re *require.Assertions, // checkGCBarrier checks whether the specified GC barrier has the specified barrier TS. This function assumes the // barrier TS is never 0, and passing 0 means asserting the GC barrier does not exist. func (s *clientStatefulTestSuite) checkGCBarrier(re *require.Assertions, keyspaceID uint32, barrierID string, expectedBarrierTS uint64) { - gcState, err := s.client.GetGCStatesClient(keyspaceID).GetGCState(context.Background()) + gcState, err := s.client.GetGCStatesClient(keyspaceID).GetGCState(context.Background(), gc.ExcludeGCBarriers(false)) + re.NoError(err) + gcBarriers, err := gcState.GetGCBarriers() re.NoError(err) found := false - for _, b := range gcState.GCBarriers { + for _, b := range gcBarriers { if b.BarrierID == barrierID { if found { re.Failf("duplicated barrier ID found in the GC states", "barrierID: %s, GC state: %+v", barrierID, gcState) @@ -2096,10 +2098,15 @@ func (s *clientStatefulTestSuite) checkGCBarrier(re *require.Assertions, keyspac // checkGlobalGCBarrier checks whether the specified global GC barrier has the specified barrier TS. // This function assumes the barrier TS is never 0, and passing 0 means asserting the GC barrier does not exist. func (s *clientStatefulTestSuite) checkGlobalGCBarrier(re *require.Assertions, barrierID string, expectedBarrierTS uint64) { - gcStates, err := s.client.GetGCStatesClient(constants.NullKeyspaceID).GetAllKeyspacesGCStates(context.Background()) + gcStates, err := s.client.GetGCStatesClient(constants.NullKeyspaceID).GetAllKeyspacesGCStates( + context.Background(), + gc.ExcludeGlobalGCBarriers(false), + ) + re.NoError(err) + globalGCBarriers, err := gcStates.GetGlobalGCBarriers() re.NoError(err) found := false - for _, b := range gcStates.GlobalGCBarriers { + for _, b := range globalGCBarriers { if b.BarrierID == barrierID { if found { re.Failf("duplicated barrier ID found in the global GC barriers", "barrierID: %s", barrierID) @@ -2172,7 +2179,7 @@ func (s *clientStatefulTestSuite) testUpdateServiceGCSafePointImpl(keyspaceID ui // Suppress the unuseful lint warning. //nolint:unparam loadServiceGCSafePointByServiceID := func(serviceID string) *endpoint.ServiceSafePoint { - gcStates, err := s.srv.GetGCStateManager().GetGCState(keyspaceID) + gcStates, err := s.srv.GetGCStateManager().GetGCState(keyspaceID, false) re.NoError(err) for _, b := range gcStates.GCBarriers { if b.BarrierID == serviceID { @@ -2454,6 +2461,12 @@ func (s *clientStatefulTestSuite) TestGCBarriers() { re.Equal("b1", b.BarrierID) re.Equal(uint64(10), b.BarrierTS) re.Equal(int64(math.MaxInt64), int64(b.TTL)) + // Test GetGCState's behavior of excluding GC barriers by default. + state, err := cli.GetGCState(ctx) + re.NoError(err) + re.False(state.HasGCBarriers()) + _, err = state.GetGCBarriers() + re.Error(err) s.checkGCBarrier(re, keyspaceID, "b1", 10) // Allows advancing to a value below the GC barrier. @@ -2706,21 +2719,31 @@ func (s *clientStatefulTestSuite) TestGetAllKeyspaceGCStates() { re.NoError(err) res, err := cli.GetAllKeyspacesGCStates(ctx) re.NoError(err) - re.Len(res.GlobalGCBarriers, 1) - re.Equal("b1", res.GlobalGCBarriers[0].BarrierID) - re.Equal(uint64(10), res.GlobalGCBarriers[0].BarrierTS) - re.Equal(time.Duration(math.MaxInt64), res.GlobalGCBarriers[0].TTL) + re.False(res.HasGlobalGCBarriers()) + _, err = res.GetGlobalGCBarriers() + re.Error(err) + + res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGlobalGCBarriers(false)) + re.NoError(err) + globalGCBarriers, err := res.GetGlobalGCBarriers() + re.NoError(err) + re.Len(globalGCBarriers, 1) + re.Equal("b1", globalGCBarriers[0].BarrierID) + re.Equal(uint64(10), globalGCBarriers[0].BarrierTS) + re.Equal(time.Duration(math.MaxInt64), globalGCBarriers[0].TTL) _, err = cli.SetGlobalGCBarrier(ctx, "b2", 12, 2*time.Second) re.NoError(err) - res, err = cli.GetAllKeyspacesGCStates(ctx) + res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGlobalGCBarriers(false)) + re.NoError(err) + globalGCBarriers, err = res.GetGlobalGCBarriers() re.NoError(err) - re.Len(res.GlobalGCBarriers, 2) - re.Equal("b2", res.GlobalGCBarriers[1].BarrierID) - re.Equal(uint64(12), res.GlobalGCBarriers[1].BarrierTS) + re.Len(globalGCBarriers, 2) + re.Equal("b2", globalGCBarriers[1].BarrierID) + re.Equal(uint64(12), globalGCBarriers[1].BarrierTS) // Returned TTL is rounded to seconds, so it can be exactly 1s here. - re.GreaterOrEqual(res.GlobalGCBarriers[1].TTL, time.Second) - re.LessOrEqual(res.GlobalGCBarriers[1].TTL, 2*time.Second) + re.GreaterOrEqual(globalGCBarriers[1].TTL, time.Second) + re.LessOrEqual(globalGCBarriers[1].TTL, 2*time.Second) cli1 := s.client.GetGCStatesClient(1) _, err = cli1.SetGCBarrier(ctx, "b3", 13, math.MaxInt64) @@ -2729,22 +2752,34 @@ func (s *clientStatefulTestSuite) TestGetAllKeyspaceGCStates() { re.NoError(err) state, ok := res.GCStates[1] re.True(ok) - re.Equal("b3", state.GCBarriers[0].BarrierID) - re.Equal(uint64(13), state.GCBarriers[0].BarrierTS) - re.Equal(time.Duration(math.MaxInt64), state.GCBarriers[0].TTL) + re.False(state.HasGCBarriers()) + _, err = state.GetGCBarriers() + re.Error(err) + + res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGCBarriers(false), gc.ExcludeGlobalGCBarriers(false)) + re.NoError(err) + state, ok = res.GCStates[1] + re.True(ok) + gcBarriers, err := state.GetGCBarriers() + re.NoError(err) + re.Equal("b3", gcBarriers[0].BarrierID) + re.Equal(uint64(13), gcBarriers[0].BarrierTS) + re.Equal(time.Duration(math.MaxInt64), gcBarriers[0].TTL) cli2 := s.client.GetGCStatesClient(2) _, err = cli2.SetGCBarrier(ctx, "b4", 14, 3*time.Second) re.NoError(err) - res, err = cli.GetAllKeyspacesGCStates(ctx) + res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGCBarriers(false), gc.ExcludeGlobalGCBarriers(false)) re.NoError(err) state, ok = res.GCStates[2] re.True(ok) - re.Equal("b4", state.GCBarriers[0].BarrierID) - re.Equal(uint64(14), state.GCBarriers[0].BarrierTS) + gcBarriers, err = state.GetGCBarriers() + re.NoError(err) + re.Equal("b4", gcBarriers[0].BarrierID) + re.Equal(uint64(14), gcBarriers[0].BarrierTS) // Returned TTL is rounded to seconds, so it can be exactly 2s here. - re.GreaterOrEqual(state.GCBarriers[0].TTL, 2*time.Second) - re.LessOrEqual(state.GCBarriers[0].TTL, 3*time.Second) + re.GreaterOrEqual(gcBarriers[0].TTL, 2*time.Second) + re.LessOrEqual(gcBarriers[0].TTL, 3*time.Second) } func TestDecodeHttpKeyRange(t *testing.T) { diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index f4a28d2de92..d7e9bbe9486 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 + github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 002a419b646..77db71e9ab9 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -473,8 +473,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 h1:oteLtLuoWZN3uvfH836U0IIJ+s3UOk11q7GaQ0Tk+wc= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 h1:n6QWAac97mv2NJhn17iFPFnsE5fMgtPLNmsGZeqq78o= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/tests/server/api/service_gc_safepoint_test.go b/tests/server/api/service_gc_safepoint_test.go index fcf839d07b0..bb0a5c1dc34 100644 --- a/tests/server/api/service_gc_safepoint_test.go +++ b/tests/server/api/service_gc_safepoint_test.go @@ -118,7 +118,7 @@ func (suite *serviceGCSafepointTestSuite) checkServiceGCSafepoint(cluster *tests err = testutil.CheckDelete(tests.TestDialClient, sspURL+"/a", testutil.StatusOK(re)) re.NoError(err) - state, err := gcStateManager.GetGCState(constant.NullKeyspaceID) + state, err := gcStateManager.GetGCState(constant.NullKeyspaceID, false) re.NoError(err) left := state.GCBarriers leftSsps := make([]*endpoint.ServiceSafePoint, 0, len(left)) diff --git a/tests/server/gc/gc_test.go b/tests/server/gc/gc_test.go index 3b261a205c8..2dafb743fb4 100644 --- a/tests/server/gc/gc_test.go +++ b/tests/server/gc/gc_test.go @@ -217,6 +217,13 @@ func TestGCOperations(t *testing.T) { re.Equal(uint64(15), resp.GetGcState().GetGcBarriers()[0].GetBarrierTs()) re.Greater(resp.GetGcState().GetGcBarriers()[0].GetTtlSeconds(), int64(3500)) re.Less(resp.GetGcState().GetGcBarriers()[0].GetTtlSeconds(), int64(3601)) + + req.ExcludeGcBarriers = true + resp, err = grpcPDClient.GetGCState(ctx, req) + re.NoError(err) + re.NotNil(resp.Header) + re.Nil(resp.Header.Error) + re.Empty(resp.GetGcState().GetGcBarriers()) } } @@ -267,6 +274,15 @@ func TestGCOperations(t *testing.T) { re.Less(gcState.GetGcBarriers()[0].GetTtlSeconds(), int64(3601)) } + req.ExcludeGcBarriers = true + resp, err = grpcPDClient.GetAllKeyspacesGCStates(ctx, req) + re.NoError(err) + re.NotNil(resp.Header) + re.Nil(resp.Header.Error) + for _, gcState := range resp.GetGcStates() { + re.Empty(gcState.GetGcBarriers()) + } + // Global GC Barrier API for _, keyspaceID := range []uint32{constant.NullKeyspaceID, ks1.Id} { // Cleanup before test @@ -322,6 +338,18 @@ func TestGCOperations(t *testing.T) { re.Equal(math.MaxInt64, int(resp.GetNewBarrierInfo().GetTtlSeconds())) } + { + req := &pdpb.GetAllKeyspacesGCStatesRequest{ + Header: header, + ExcludeGlobalGcBarriers: true, + } + resp, err := grpcPDClient.GetAllKeyspacesGCStates(ctx, req) + re.NoError(err) + re.NotNil(resp.Header) + re.Nil(resp.Header.Error) + re.Empty(resp.GetGlobalGcBarriers()) + } + { // Failed to set a global GC barrier (below txn safe point) req := &pdpb.SetGlobalGCBarrierRequest{ diff --git a/tools/go.mod b/tools/go.mod index 7571bba04da..5ce52bac8c0 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -23,7 +23,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 + github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.20.5 diff --git a/tools/go.sum b/tools/go.sum index df33be33529..a4a08b87464 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -478,8 +478,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359 h1:oteLtLuoWZN3uvfH836U0IIJ+s3UOk11q7GaQ0Tk+wc= -github.com/pingcap/kvproto v0.0.0-20260511034003-fc9e0458a359/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473 h1:n6QWAac97mv2NJhn17iFPFnsE5fMgtPLNmsGZeqq78o= +github.com/pingcap/kvproto v0.0.0-20260514102340-daa7c864b473/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=