Skip to content

Commit ea8c217

Browse files
serathiusserathius
committed
Test cache in robustness tests
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com> Allow progress notifies Disable errors Reduce clients and disable watch to make linearization error easier to track Implement simple sleep failpoint to simplify tracking linearization bug Remove Txn and lease operation to make finding linearization issues easier
1 parent 5f38748 commit ea8c217

6 files changed

Lines changed: 159 additions & 160 deletions

File tree

‎cache/cache.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -284,9 +284,7 @@ func (c *Cache) getWatchLoop() {
284284
if err := ctx.Err(); err != nil {
285285
return
286286
}
287-
if err := c.getWatch(); err != nil {
288-
fmt.Printf("getWatch failed, will retry after %v: %v\n", backoff, err)
289-
}
287+
c.getWatch()
290288
select {
291289
case <-ctx.Done():
292290
return

‎tests/robustness/client/client.go‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"go.uber.org/zap"
2525

2626
"go.etcd.io/etcd/api/v3/mvccpb"
27+
"go.etcd.io/etcd/cache/v3"
2728
clientv3 "go.etcd.io/etcd/client/v3"
2829
"go.etcd.io/etcd/tests/v3/robustness/identity"
2930
"go.etcd.io/etcd/tests/v3/robustness/model"
@@ -64,6 +65,18 @@ func NewRecordingClient(endpoints []string, ids identity.Provider, baseTime time
6465
if err != nil {
6566
return nil, err
6667
}
68+
c, err := cache.New(cc, "")
69+
if err != nil {
70+
return nil, err
71+
}
72+
cc.Watcher = &cacheWatcher{
73+
Cache: c,
74+
Watcher: cc.Watcher,
75+
}
76+
cc.KV = &cacheKV{
77+
Cache: c,
78+
KV: cc.KV,
79+
}
6780
return &RecordingClient{
6881
ID: ids.NewClientID(),
6982
client: cc,
@@ -72,6 +85,33 @@ func NewRecordingClient(endpoints []string, ids identity.Provider, baseTime time
7285
}, nil
7386
}
7487

88+
type cacheKV struct {
89+
clientv3.KV
90+
Cache *cache.Cache
91+
}
92+
93+
func (ckv *cacheKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
94+
return ckv.Cache.Get(ctx, key, opts...)
95+
}
96+
97+
type cacheWatcher struct {
98+
Cache *cache.Cache
99+
Watcher clientv3.Watcher
100+
}
101+
102+
func (cw *cacheWatcher) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {
103+
return cw.Cache.Watch(ctx, key, opts...)
104+
}
105+
106+
func (cw *cacheWatcher) RequestProgress(ctx context.Context) error {
107+
return cw.Watcher.RequestProgress(ctx)
108+
}
109+
110+
func (cw *cacheWatcher) Close() error {
111+
cw.Cache.Close()
112+
return cw.Watcher.Close()
113+
}
114+
75115
func (c *RecordingClient) Close() error {
76116
return c.client.Close()
77117
}
@@ -284,8 +324,9 @@ func (c *RecordingClient) Watch(ctx context.Context, key string, rev int64, with
284324
Key: key,
285325
Revision: rev,
286326
WithPrefix: withPrefix,
287-
WithProgressNotify: withProgressNotify,
288-
WithPrevKV: withPrevKV,
327+
// TODO: Restore when cache supports
328+
// WithProgressNotify: withProgressNotify,
329+
// WithPrevKV: withPrevKV,
289330
}
290331
return c.watch(ctx, request)
291332
}
@@ -318,6 +359,9 @@ func (c *RecordingClient) watch(ctx context.Context, request model.WatchRequest)
318359
go func() {
319360
defer close(respCh)
320361
for r := range c.client.Watch(ctx, request.Key, ops...) {
362+
if r.Err() != nil {
363+
fmt.Printf("Got error %v\n", r.Err())
364+
}
321365
responses = append(responses, ToWatchResponse(r, c.baseTime))
322366
c.watchMux.Lock()
323367
c.watchOperations[index].Responses = responses

‎tests/robustness/failpoint/failpoint.go‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,23 +36,24 @@ const (
3636
)
3737

3838
var allFailpoints = []Failpoint{
39-
KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
40-
DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
41-
BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
42-
BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
43-
CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
44-
CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
45-
RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
46-
RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
47-
BeforeApplyOneConfChangeSleep,
48-
MemberReplace,
49-
MemberDowngrade,
50-
MemberDowngradeUpgrade,
51-
DropPeerNetwork,
52-
RaftBeforeSaveSleep,
53-
RaftAfterSaveSleep,
54-
ApplyBeforeOpenSnapshot,
55-
SleepBeforeSendWatchResponse,
39+
Sleep,
40+
// KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
41+
// DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
42+
// BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
43+
// BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
44+
// CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
45+
// CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
46+
// RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
47+
// RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
48+
// BeforeApplyOneConfChangeSleep,
49+
// MemberReplace,
50+
// MemberDowngrade,
51+
// MemberDowngradeUpgrade,
52+
// DropPeerNetwork,
53+
// RaftBeforeSaveSleep,
54+
// RaftAfterSaveSleep,
55+
// ApplyBeforeOpenSnapshot,
56+
// SleepBeforeSendWatchResponse,
5657
}
5758

5859
func PickRandom(clus *e2e.EtcdProcessCluster, profile traffic.Profile) (Failpoint, error) {
@@ -155,3 +156,20 @@ type AvailabilityChecker interface {
155156
type TimeoutInterface interface {
156157
Timeout() time.Duration
157158
}
159+
160+
var Sleep Failpoint = sleepFailpoint{}
161+
162+
type sleepFailpoint struct{}
163+
164+
func (f sleepFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) {
165+
time.Sleep(time.Second)
166+
return nil, nil
167+
}
168+
169+
func (f sleepFailpoint) Name() string {
170+
return "Sleep"
171+
}
172+
173+
func (f sleepFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess, traffic.Profile) bool {
174+
return true
175+
}

‎tests/robustness/scenarios/scenarios.go‎

Lines changed: 68 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@ import (
2020
"testing"
2121
"time"
2222

23-
"github.com/stretchr/testify/require"
24-
25-
"go.etcd.io/etcd/api/v3/version"
2623
"go.etcd.io/etcd/client/pkg/v3/fileutil"
2724
"go.etcd.io/etcd/server/v3/etcdserver"
2825
"go.etcd.io/etcd/tests/v3/framework/e2e"
@@ -50,16 +47,6 @@ var trafficProfiles = []TrafficProfile{
5047
Traffic: traffic.EtcdPutDeleteLease,
5148
Profile: traffic.LowTraffic,
5249
},
53-
{
54-
Name: "KubernetesHighTraffic",
55-
Traffic: traffic.Kubernetes,
56-
Profile: traffic.HighTrafficProfile,
57-
},
58-
{
59-
Name: "KubernetesLowTraffic",
60-
Traffic: traffic.Kubernetes,
61-
Profile: traffic.LowTraffic,
62-
},
6350
}
6451

6552
type TestScenario struct {
@@ -160,75 +147,72 @@ func Exploratory(_ *testing.T) []TestScenario {
160147
}
161148

162149
func Regression(t *testing.T) []TestScenario {
163-
v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd)
164-
require.NoErrorf(t, err, "Failed checking etcd version binary, binary: %q", e2e.BinPath.Etcd)
165-
166-
scenarios := []TestScenario{}
167-
scenarios = append(scenarios, TestScenario{
168-
Name: "Issue14370",
169-
Failpoint: failpoint.RaftBeforeSavePanic,
170-
Profile: traffic.LowTraffic.WithoutWatchLoop(),
171-
Traffic: traffic.EtcdPutDeleteLease,
172-
Cluster: *e2e.NewConfig(
173-
e2e.WithClusterSize(1),
174-
e2e.WithGoFailEnabled(true),
175-
),
176-
})
177-
scenarios = append(scenarios, TestScenario{
178-
Name: "Issue14685",
179-
Failpoint: failpoint.DefragBeforeCopyPanic,
180-
Profile: traffic.LowTraffic.WithoutWatchLoop(),
181-
Traffic: traffic.EtcdPutDeleteLease,
182-
Cluster: *e2e.NewConfig(
183-
e2e.WithClusterSize(1),
184-
e2e.WithGoFailEnabled(true),
185-
),
186-
})
187-
scenarios = append(scenarios, TestScenario{
188-
Name: "Issue13766",
189-
Failpoint: failpoint.KillFailpoint,
190-
Profile: traffic.HighTrafficProfile.WithoutWatchLoop(),
191-
Traffic: traffic.EtcdPut,
192-
Cluster: *e2e.NewConfig(
193-
e2e.WithSnapshotCount(100),
194-
),
195-
})
196-
scenarios = append(scenarios, TestScenario{
197-
Name: "Issue15220",
198-
Watch: client.WatchConfig{
199-
RequestProgress: true,
200-
},
201-
Profile: traffic.LowTraffic,
202-
Traffic: traffic.EtcdPutDeleteLease,
203-
Failpoint: failpoint.KillFailpoint,
204-
Cluster: *e2e.NewConfig(
205-
e2e.WithClusterSize(1),
206-
),
207-
})
208-
scenarios = append(scenarios, TestScenario{
209-
Name: "Issue17529",
210-
Profile: traffic.HighTrafficProfile,
211-
Traffic: traffic.Kubernetes,
212-
Failpoint: failpoint.SleepBeforeSendWatchResponse,
213-
Cluster: *e2e.NewConfig(
214-
e2e.WithClusterSize(1),
215-
e2e.WithGoFailEnabled(true),
216-
options.WithSnapshotCount(100),
217-
),
218-
})
219-
220-
scenarios = append(scenarios, TestScenario{
221-
Name: "Issue17780",
222-
Profile: traffic.LowTraffic.WithoutCompaction().WithoutWatchLoop(),
223-
Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic,
224-
Traffic: traffic.Kubernetes,
225-
Cluster: *e2e.NewConfig(
226-
e2e.WithClusterSize(1),
227-
e2e.WithCompactionBatchLimit(300),
228-
e2e.WithSnapshotCount(1000),
229-
e2e.WithGoFailEnabled(true),
230-
),
231-
})
150+
// scenarios := []TestScenario{}
151+
// scenarios = append(scenarios, TestScenario{
152+
// Name: "Issue14370",
153+
// Failpoint: failpoint.RaftBeforeSavePanic,
154+
// Profile: traffic.LowTraffic.WithoutWatchLoop(),
155+
// Traffic: traffic.EtcdPutDeleteLease,
156+
// Cluster: *e2e.NewConfig(
157+
// e2e.WithClusterSize(1),
158+
// e2e.WithGoFailEnabled(true),
159+
// ),
160+
// })
161+
// scenarios = append(scenarios, TestScenario{
162+
// Name: "Issue14685",
163+
// Failpoint: failpoint.DefragBeforeCopyPanic,
164+
// Profile: traffic.LowTraffic.WithoutWatchLoop(),
165+
// Traffic: traffic.EtcdPutDeleteLease,
166+
// Cluster: *e2e.NewConfig(
167+
// e2e.WithClusterSize(1),
168+
// e2e.WithGoFailEnabled(true),
169+
// ),
170+
// })
171+
// scenarios = append(scenarios, TestScenario{
172+
// Name: "Issue13766",
173+
// Failpoint: failpoint.KillFailpoint,
174+
// Profile: traffic.HighTrafficProfile.WithoutWatchLoop(),
175+
// Traffic: traffic.EtcdPut,
176+
// Cluster: *e2e.NewConfig(
177+
// e2e.WithSnapshotCount(100),
178+
// ),
179+
// })
180+
// scenarios = append(scenarios, TestScenario{
181+
// Name: "Issue15220",
182+
// Watch: client.WatchConfig{
183+
// RequestProgress: true,
184+
// },
185+
// Profile: traffic.LowTraffic,
186+
// Traffic: traffic.EtcdPutDeleteLease,
187+
// Failpoint: failpoint.KillFailpoint,
188+
// Cluster: *e2e.NewConfig(
189+
// e2e.WithClusterSize(1),
190+
// ),
191+
// })
192+
// scenarios = append(scenarios, TestScenario{
193+
// Name: "Issue17529",
194+
// Profile: traffic.HighTrafficProfile,
195+
// Traffic: traffic.Kubernetes,
196+
// Failpoint: failpoint.SleepBeforeSendWatchResponse,
197+
// Cluster: *e2e.NewConfig(
198+
// e2e.WithClusterSize(1),
199+
// e2e.WithGoFailEnabled(true),
200+
// options.WithSnapshotCount(100),
201+
// ),
202+
// })
203+
204+
// scenarios = append(scenarios, TestScenario{
205+
// Name: "Issue17780",
206+
// Profile: traffic.LowTraffic.WithoutCompaction().WithoutWatchLoop(),
207+
// Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic,
208+
// Traffic: traffic.Kubernetes,
209+
// Cluster: *e2e.NewConfig(
210+
// e2e.WithClusterSize(1),
211+
// e2e.WithCompactionBatchLimit(300),
212+
// e2e.WithSnapshotCount(1000),
213+
// e2e.WithGoFailEnabled(true),
214+
// ),
215+
// })
232216

233217
// NOTE:
234218
//
@@ -242,51 +226,6 @@ func Regression(t *testing.T) []TestScenario {
242226
// burstable value. A higher QPS can generate more new keys than
243227
// expected, making it difficult to determine an optimal compaction
244228
// batch limit within a larger key space.
245-
scenarios = append(scenarios, TestScenario{
246-
Name: "Issue19179",
247-
Profile: traffic.Profile{
248-
MinimalQPS: 50,
249-
MaximalQPS: 100,
250-
BurstableQPS: 100,
251-
MemberClientCount: 6,
252-
ClusterClientCount: 2,
253-
MaxNonUniqueRequestConcurrency: 3,
254-
}.WithoutCompaction(),
255-
Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic,
256-
Traffic: traffic.KubernetesCreateDelete,
257-
Cluster: *e2e.NewConfig(
258-
e2e.WithClusterSize(1),
259-
e2e.WithCompactionBatchLimit(50),
260-
e2e.WithSnapshotCount(1000),
261-
e2e.WithGoFailEnabled(true),
262-
),
263-
})
264-
scenarios = append(scenarios, TestScenario{
265-
Name: "Issue18089",
266-
Profile: traffic.LowTraffic.WithCompactionPeriod(100 * time.Millisecond), // Use frequent compaction for high reproduce rate
267-
Failpoint: failpoint.SleepBeforeSendWatchResponse,
268-
Traffic: traffic.EtcdDelete,
269-
Cluster: *e2e.NewConfig(
270-
e2e.WithClusterSize(1),
271-
e2e.WithGoFailEnabled(true),
272-
),
273-
})
274-
if v.Compare(version.V3_5) >= 0 {
275-
opts := []e2e.EPClusterOption{
276-
e2e.WithSnapshotCount(100),
277-
e2e.WithPeerProxy(true),
278-
e2e.WithIsPeerTLS(true),
279-
}
280-
if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) {
281-
opts = append(opts, e2e.WithSnapshotCatchUpEntries(100))
282-
}
283-
scenarios = append(scenarios, TestScenario{
284-
Name: "Issue15271",
285-
Failpoint: failpoint.BlackholeUntilSnapshot,
286-
Profile: traffic.HighTrafficProfile,
287-
Traffic: traffic.EtcdPut,
288-
Cluster: *e2e.NewConfig(opts...),
289-
})
290-
}
291-
return scenarios
229+
// scenarios = append(scenarios, TestScenario{
230+
return nil
292231
}

0 commit comments

Comments
 (0)