diff --git a/pkg/kv/kvserver/split_delay_helper_test.go b/pkg/kv/kvserver/split_delay_helper_test.go index dedcc82e4639..cda9f203da3b 100644 --- a/pkg/kv/kvserver/split_delay_helper_test.go +++ b/pkg/kv/kvserver/split_delay_helper_test.go @@ -155,7 +155,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { } s := maybeDelaySplitToAvoidSnapshot(ctx, h) assert.EqualValues(t, "; delayed by 5.5s to resolve: replica r1/2 not caught up: "+ - state.String()+" match=0 next=0 paused (without success)", s) + state.String()+" match=0 next=0 sentCommit=0 matchCommit=0 paused (without success)", s) }) } @@ -193,6 +193,7 @@ func TestSplitDelayToAvoidSnapshot(t *testing.T) { } } s := maybeDelaySplitToAvoidSnapshot(ctx, h) - assert.EqualValues(t, "; delayed by 2.5s to resolve: replica r1/2 not caught up: StateProbe match=0 next=0", s) + assert.EqualValues(t, "; delayed by 2.5s to resolve: replica r1/2 not caught up: "+ + "StateProbe match=0 next=0 sentCommit=0 matchCommit=0", s) }) } diff --git a/pkg/raft/confchange/confchange.go b/pkg/raft/confchange/confchange.go index 8e204b4ffc3d..313290c8505c 100644 --- a/pkg/raft/confchange/confchange.go +++ b/pkg/raft/confchange/confchange.go @@ -271,10 +271,11 @@ func (c Changer) initProgress( // at all (and will thus likely need a snapshot), though the app may // have applied a snapshot out of band before adding the replica (thus // making the first index the better choice). - Match: 0, - Next: max(c.LastIndex, 1), // invariant: Match < Next - Inflights: tracker.NewInflights(c.MaxInflight, c.MaxInflightBytes), - IsLearner: isLearner, + Match: 0, + MatchCommit: 0, + Next: max(c.LastIndex, 1), // invariant: Match < Next + Inflights: tracker.NewInflights(c.MaxInflight, c.MaxInflightBytes), + IsLearner: isLearner, // When a node is first added, we should mark it as recently active. // Otherwise, CheckQuorum may cause us to step down if it is invoked // before the added node has had a chance to communicate with us. diff --git a/pkg/raft/confchange/testdata/joint_autoleave.txt b/pkg/raft/confchange/testdata/joint_autoleave.txt index be138df3cfaa..cea89288bdcb 100644 --- a/pkg/raft/confchange/testdata/joint_autoleave.txt +++ b/pkg/raft/confchange/testdata/joint_autoleave.txt @@ -5,16 +5,16 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 # Autoleave is reflected in the config. enter-joint autoleave=true v2 v3 ---- voters=(1 2 3)&&(1) autoleave -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 # Can't enter-joint twice, even if autoleave changes. enter-joint autoleave=false @@ -24,6 +24,6 @@ config is already joint leave-joint ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/joint_idempotency.txt b/pkg/raft/confchange/testdata/joint_idempotency.txt index a47f3a662c88..5851e66294f3 100644 --- a/pkg/raft/confchange/testdata/joint_idempotency.txt +++ b/pkg/raft/confchange/testdata/joint_idempotency.txt @@ -5,19 +5,19 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 enter-joint r1 r2 r9 v2 v3 v4 v2 v3 v4 l2 l2 r4 r4 l1 l1 ---- voters=(3)&&(1) learners=(2) learners_next=(1) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +3: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 leave-joint ---- voters=(3) learners=(1 2) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +3: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/joint_learners_next.txt b/pkg/raft/confchange/testdata/joint_learners_next.txt index 6faddfe7e973..a90d02127abd 100644 --- a/pkg/raft/confchange/testdata/joint_learners_next.txt +++ b/pkg/raft/confchange/testdata/joint_learners_next.txt @@ -8,17 +8,17 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 enter-joint v2 l1 ---- voters=(2)&&(1) learners_next=(1) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 leave-joint ---- voters=(2) learners=(1) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/joint_safety.txt b/pkg/raft/confchange/testdata/joint_safety.txt index 75d11b199e02..84ba199d82f4 100644 --- a/pkg/raft/confchange/testdata/joint_safety.txt +++ b/pkg/raft/confchange/testdata/joint_safety.txt @@ -15,7 +15,7 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=3 +1: StateProbe match=0 next=3 sentCommit=0 matchCommit=0 leave-joint ---- @@ -25,7 +25,7 @@ can't leave a non-joint config enter-joint ---- voters=(1)&&(1) -1: StateProbe match=0 next=3 +1: StateProbe match=0 next=3 sentCommit=0 matchCommit=0 enter-joint ---- @@ -34,7 +34,7 @@ config is already joint leave-joint ---- voters=(1) -1: StateProbe match=0 next=3 +1: StateProbe match=0 next=3 sentCommit=0 matchCommit=0 leave-joint ---- @@ -45,10 +45,10 @@ enter-joint r1 v2 v3 l4 ---- voters=(2 3)&&(1) learners=(4) -1: StateProbe match=0 next=3 -2: StateProbe match=0 next=9 -3: StateProbe match=0 next=9 -4: StateProbe match=0 next=9 learner +1: StateProbe match=0 next=3 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 +4: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 learner enter-joint ---- @@ -67,15 +67,15 @@ can't apply simple config change in joint config leave-joint ---- voters=(2 3) learners=(4) -2: StateProbe match=0 next=9 -3: StateProbe match=0 next=9 -4: StateProbe match=0 next=9 learner +2: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 +4: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 learner simple l9 ---- voters=(2 3) learners=(4 9) -2: StateProbe match=0 next=9 -3: StateProbe match=0 next=9 -4: StateProbe match=0 next=9 learner -9: StateProbe match=0 next=14 learner +2: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 +4: StateProbe match=0 next=9 sentCommit=0 matchCommit=0 learner +9: StateProbe match=0 next=14 sentCommit=0 matchCommit=0 learner diff --git a/pkg/raft/confchange/testdata/simple_idempotency.txt b/pkg/raft/confchange/testdata/simple_idempotency.txt index e31c43b7ffff..da7c95cabf73 100644 --- a/pkg/raft/confchange/testdata/simple_idempotency.txt +++ b/pkg/raft/confchange/testdata/simple_idempotency.txt @@ -2,68 +2,68 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple v2 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple l1 ---- voters=(2) learners=(1) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple l1 ---- voters=(2) learners=(1) -1: StateProbe match=0 next=1 learner -2: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple r1 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple r1 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple v3 ---- voters=(2 3) -2: StateProbe match=0 next=2 -3: StateProbe match=0 next=7 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=7 sentCommit=0 matchCommit=0 simple r3 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple r3 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 simple r4 ---- voters=(2) -2: StateProbe match=0 next=2 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/simple_promote_demote.txt b/pkg/raft/confchange/testdata/simple_promote_demote.txt index b4b770de7c56..ebe3afa32d96 100644 --- a/pkg/raft/confchange/testdata/simple_promote_demote.txt +++ b/pkg/raft/confchange/testdata/simple_promote_demote.txt @@ -4,22 +4,22 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple v2 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple v3 ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 # Can atomically demote and promote without a hitch. # This is pointless, but possible. @@ -27,18 +27,18 @@ simple l1 v1 ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 # Can demote a voter. simple l2 ---- voters=(1 3) learners=(2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 # Can atomically promote and demote the same voter. # This is pointless, but possible. @@ -46,15 +46,15 @@ simple v2 l2 ---- voters=(1 3) learners=(2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 learner -3: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 learner +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 # Can promote a voter. simple v2 ---- voters=(1 2 3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 -3: StateProbe match=0 next=2 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/simple_safety.txt b/pkg/raft/confchange/testdata/simple_safety.txt index 6566c5fccf7d..6d53f1335acc 100644 --- a/pkg/raft/confchange/testdata/simple_safety.txt +++ b/pkg/raft/confchange/testdata/simple_safety.txt @@ -7,15 +7,15 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple v2 l3 ---- voters=(1 2) learners=(3) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=2 -3: StateProbe match=0 next=2 learner +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 learner simple r1 v5 @@ -46,11 +46,11 @@ simple l2 l3 l4 l5 ---- voters=(1) learners=(2 3 4 5) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=2 learner -3: StateProbe match=0 next=2 learner -4: StateProbe match=0 next=8 learner -5: StateProbe match=0 next=8 learner +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 learner +3: StateProbe match=0 next=2 sentCommit=0 matchCommit=0 learner +4: StateProbe match=0 next=8 sentCommit=0 matchCommit=0 learner +5: StateProbe match=0 next=8 sentCommit=0 matchCommit=0 learner simple r1 @@ -61,4 +61,4 @@ simple r2 r3 r4 r5 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/update.txt b/pkg/raft/confchange/testdata/update.txt index ac47bf3e8cdf..3dcd88c0b6e9 100644 --- a/pkg/raft/confchange/testdata/update.txt +++ b/pkg/raft/confchange/testdata/update.txt @@ -6,18 +6,18 @@ simple v1 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple v2 u1 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 simple u1 u2 u3 u1 u2 u3 ---- voters=(1 2) -1: StateProbe match=0 next=1 -2: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 +2: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/confchange/testdata/zero.txt b/pkg/raft/confchange/testdata/zero.txt index 226ade088323..067e8223d197 100644 --- a/pkg/raft/confchange/testdata/zero.txt +++ b/pkg/raft/confchange/testdata/zero.txt @@ -3,4 +3,4 @@ simple v1 r0 v0 l0 ---- voters=(1) -1: StateProbe match=0 next=1 +1: StateProbe match=0 next=1 sentCommit=0 matchCommit=0 diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index 4fbabdd2c0e0..535072a5dafa 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -654,7 +654,7 @@ func (r *raft) maybeSendAppend(to pb.PeerID) bool { Match: pr.Match, }) pr.SentEntries(len(entries), uint64(payloadsSize(entries))) - pr.SentCommit(commit) + pr.MaybeUpdateSentCommit(commit) return true } @@ -710,9 +710,7 @@ func (r *raft) sendHeartbeat(to pb.PeerID) { Commit: commit, Match: pr.Match, }) - if commit != 0 { - pr.SentCommit(commit) - } + pr.MaybeUpdateSentCommit(commit) } // sendFortify sends a fortification RPC to the given peer. @@ -745,6 +743,10 @@ func (r *raft) sendFortify(to pb.PeerID) { func (r *raft) bcastAppend() { r.trk.Visit(func(id pb.PeerID, _ *tracker.Progress) { if id == r.id { + // NB: the leader doesn't send MsgAppResp to itself here. This means that + // the leader will not have a chance to update its own + // MatchCommit/SentCommit. That is fine because the leader doesn't use + // MatchCommit/SentCommit for itself. It only uses the followers' values. return } r.maybeSendAppend(id) @@ -858,10 +860,11 @@ func (r *raft) reset(term uint64) { r.electionTracker.ResetVotes() r.trk.Visit(func(id pb.PeerID, pr *tracker.Progress) { *pr = tracker.Progress{ - Match: 0, - Next: r.raftLog.lastIndex() + 1, - Inflights: tracker.NewInflights(r.maxInflight, r.maxInflightBytes), - IsLearner: pr.IsLearner, + Match: 0, + MatchCommit: 0, + Next: r.raftLog.lastIndex() + 1, + Inflights: tracker.NewInflights(r.maxInflight, r.maxInflightBytes), + IsLearner: pr.IsLearner, } if id == r.id { pr.Match = r.raftLog.lastIndex() diff --git a/pkg/raft/testdata/async_storage_writes.txt b/pkg/raft/testdata/async_storage_writes.txt index ffbcbd8bfac0..f75ae8efd131 100644 --- a/pkg/raft/testdata/async_storage_writes.txt +++ b/pkg/raft/testdata/async_storage_writes.txt @@ -259,9 +259,9 @@ process-ready 1 2 3 status 1 ---- -1: StateReplicate match=11 next=13 -2: StateReplicate match=11 next=13 inflight=1 -3: StateReplicate match=11 next=13 inflight=1 +1: StateReplicate match=11 next=13 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=13 sentCommit=11 matchCommit=11 inflight=1 +3: StateReplicate match=11 next=13 sentCommit=11 matchCommit=11 inflight=1 deliver-msgs 1 2 3 ---- @@ -373,9 +373,9 @@ AppendThread->3 MsgStorageAppendResp Term:0 Log:1/12 status 1 ---- -1: StateReplicate match=12 next=14 -2: StateReplicate match=12 next=14 inflight=1 -3: StateReplicate match=12 next=14 inflight=1 +1: StateReplicate match=12 next=14 sentCommit=11 matchCommit=11 +2: StateReplicate match=12 next=14 sentCommit=12 matchCommit=11 inflight=1 +3: StateReplicate match=12 next=14 sentCommit=12 matchCommit=11 inflight=1 propose 1 prop_3 ---- diff --git a/pkg/raft/testdata/campaign_learner_must_vote.txt b/pkg/raft/testdata/campaign_learner_must_vote.txt index cbbacdea367f..42a62f27813a 100644 --- a/pkg/raft/testdata/campaign_learner_must_vote.txt +++ b/pkg/raft/testdata/campaign_learner_must_vote.txt @@ -122,7 +122,7 @@ stabilize 2 3 3->2 MsgFortifyLeaderResp Term:2 Log:0/0 LeadEpoch:1 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) Commit:3 DEBUG 2 received MsgAppResp(rejected, hint: (index 3, term 1)) from 3 for index 4 - DEBUG 2 decreased progress of 3 to [StateProbe match=0 next=4] + DEBUG 2 decreased progress of 3 to [StateProbe match=0 next=4 sentCommit=3 matchCommit=3] > 2 handling Ready Ready MustSync=false: Messages: diff --git a/pkg/raft/testdata/confchange_v1_add_single.txt b/pkg/raft/testdata/confchange_v1_add_single.txt index d2c9bf6c200f..e8cb0c1b1d9b 100644 --- a/pkg/raft/testdata/confchange_v1_add_single.txt +++ b/pkg/raft/testdata/confchange_v1_add_single.txt @@ -70,9 +70,9 @@ stabilize > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 sentCommit=4 matchCommit=0 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -93,4 +93,4 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 sentCommit=4 matchCommit=4 paused pendingSnap=4] diff --git a/pkg/raft/testdata/confchange_v2_add_double_auto.txt b/pkg/raft/testdata/confchange_v2_add_double_auto.txt index 7fcebc2927b4..d71e3304a67b 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_auto.txt @@ -93,9 +93,9 @@ stabilize 1 2 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 sentCommit=4 matchCommit=0 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -116,7 +116,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 sentCommit=4 matchCommit=4 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -168,9 +168,9 @@ stabilize 1 3 > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 3 for index 3 - DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 5] sent snapshot[index: 5, term: 1] to 3 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=6 paused pendingSnap=5] + DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 [firstindex: 3, commit: 5] sent snapshot[index: 5, term: 1] to 3 [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=6 sentCommit=5 matchCommit=0 paused pendingSnap=5] > 1 handling Ready Ready MustSync=false: Messages: @@ -191,7 +191,7 @@ stabilize 1 3 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/5 Commit:5 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 next=6 paused pendingSnap=5] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 next=6 sentCommit=5 matchCommit=5 paused pendingSnap=5] # Nothing else happens. stabilize diff --git a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt index ddbbc0cd03b7..78b0ff346cdb 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt @@ -76,9 +76,9 @@ stabilize 1 2 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 sentCommit=4 matchCommit=0 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -99,7 +99,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 sentCommit=4 matchCommit=4 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: diff --git a/pkg/raft/testdata/confchange_v2_add_single_auto.txt b/pkg/raft/testdata/confchange_v2_add_single_auto.txt index 834fa4a9a091..1d4466694e20 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_auto.txt @@ -71,9 +71,9 @@ stabilize > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 sentCommit=4 matchCommit=0 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -94,4 +94,4 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 sentCommit=4 matchCommit=4 paused pendingSnap=4] diff --git a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt index f9bcebf4d0ed..bcf2649eae32 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt @@ -71,9 +71,9 @@ stabilize 1 2 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) DEBUG 1 received MsgAppResp(rejected, hint: (index 0, term 0)) from 2 for index 3 - DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] - DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] - DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 paused pendingSnap=4] + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1 sentCommit=0 matchCommit=0] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=5 sentCommit=4 matchCommit=0 paused pendingSnap=4] > 1 handling Ready Ready MustSync=false: Messages: @@ -94,7 +94,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 > 1 receiving messages 2->1 MsgAppResp Term:1 Log:0/4 Commit:4 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 sentCommit=4 matchCommit=4 paused pendingSnap=4] # Check that we're not allowed to change membership again while in the joint state. # This leads to an empty entry being proposed instead (index 5 in the stabilize block diff --git a/pkg/raft/testdata/heartbeat_resp_recovers_from_probing.txt b/pkg/raft/testdata/heartbeat_resp_recovers_from_probing.txt index 08c60eb7f771..2647704bfb0d 100644 --- a/pkg/raft/testdata/heartbeat_resp_recovers_from_probing.txt +++ b/pkg/raft/testdata/heartbeat_resp_recovers_from_probing.txt @@ -28,20 +28,20 @@ ok status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 +3: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 # On the first replica, report the second one as not reachable. report-unreachable 1 2 ---- -DEBUG 1 failed to send message to 2 because it is unreachable [StateProbe match=11 next=12] +DEBUG 1 failed to send message to 2 because it is unreachable [StateProbe match=11 next=12 sentCommit=11 matchCommit=11] status 1 ---- -1: StateReplicate match=11 next=12 -2: StateProbe match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateProbe match=11 next=12 sentCommit=11 matchCommit=11 +3: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 tick-heartbeat 1 ---- @@ -85,6 +85,6 @@ stabilize status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 +3: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 diff --git a/pkg/raft/testdata/lagging_commit.txt b/pkg/raft/testdata/lagging_commit.txt index 6d75a225a097..f7bf4e90cbc0 100644 --- a/pkg/raft/testdata/lagging_commit.txt +++ b/pkg/raft/testdata/lagging_commit.txt @@ -114,9 +114,9 @@ dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13 status 1 ---- -1: StateReplicate match=13 next=14 -2: StateReplicate match=13 next=14 -3: StateReplicate match=11 next=14 inflight=2 +1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11 +2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13 +3: StateReplicate match=11 next=14 sentCommit=13 matchCommit=11 inflight=2 # The leader still observes that the entries are in-flight to the follower 3, # since it hasn't heard from it. Nothing triggers updating the follower's diff --git a/pkg/raft/testdata/msg_app_commit_index.txt b/pkg/raft/testdata/msg_app_commit_index.txt index 2d36d88e2479..5fc3ba849a15 100644 --- a/pkg/raft/testdata/msg_app_commit_index.txt +++ b/pkg/raft/testdata/msg_app_commit_index.txt @@ -109,9 +109,9 @@ dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13 status 1 ---- -1: StateReplicate match=13 next=14 -2: StateReplicate match=13 next=14 -3: StateReplicate match=13 next=14 +1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11 +2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13 +3: StateReplicate match=13 next=14 sentCommit=13 matchCommit=11 # Wait for the next heartbeat response. tick-heartbeat 1 @@ -189,3 +189,14 @@ stabilize 1 2 3 > 1 receiving messages 2->1 MsgHeartbeatResp Term:1 Log:0/0 3->1 MsgHeartbeatResp Term:1 Log:0/0 + +# The leader's sentCommit and the matchCommit remain stale even after stabilize. +# The reason is that the leader send itself a MsgAppResp when it receives a +# MsgProp. However, the leader doesn't send itself a MsgAppResp when it +# broadcasts a MsgApp to the followers. That's fine because the leader doesn't +# use its sentCommit/matchCommit for anything. +status 1 +---- +1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11 +2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13 +3: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13 diff --git a/pkg/raft/testdata/msg_app_commit_index_leader_old_version.txt b/pkg/raft/testdata/msg_app_commit_index_leader_old_version.txt index 11067a7529fa..53d6de30a943 100644 --- a/pkg/raft/testdata/msg_app_commit_index_leader_old_version.txt +++ b/pkg/raft/testdata/msg_app_commit_index_leader_old_version.txt @@ -117,11 +117,13 @@ deliver-msgs drop=(3) dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:12 dropped: 1->3 MsgApp Term:1 Log:1/13 Commit:13 +# The matchCommit can be ignored in this test because if the leader actually +# doesn't have 23.3, it wouldn't even have the matchCommit field. status 1 ---- -1: StateReplicate match=13 next=14 -2: StateReplicate match=13 next=14 -3: StateReplicate match=13 next=14 +1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11 +2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13 +3: StateReplicate match=13 next=14 sentCommit=13 matchCommit=11 # Wait for the next heartbeat response. tick-heartbeat 1 diff --git a/pkg/raft/testdata/replicate_pause.txt b/pkg/raft/testdata/replicate_pause.txt index 2d7ef5c9da49..7ee9a71e24f8 100644 --- a/pkg/raft/testdata/replicate_pause.txt +++ b/pkg/raft/testdata/replicate_pause.txt @@ -46,9 +46,9 @@ ok # Expect that in-flight tracking to nodes 2 and 3 is saturated. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=11 next=15 paused inflight=3[full] -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=14 next=15 sentCommit=11 matchCommit=11 +2: StateReplicate match=11 next=15 sentCommit=11 matchCommit=11 paused inflight=3[full] +3: StateReplicate match=11 next=15 sentCommit=11 matchCommit=11 paused inflight=3[full] log-level none ---- @@ -66,9 +66,9 @@ ok # Expect that the entries are committed and stored on nodes 1 and 2. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=14 next=15 -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=14 next=15 sentCommit=11 matchCommit=11 +2: StateReplicate match=14 next=15 sentCommit=14 matchCommit=14 +3: StateReplicate match=11 next=15 sentCommit=14 matchCommit=11 paused inflight=3[full] # Drop append messages to node 3. deliver-msgs drop=3 @@ -97,9 +97,9 @@ ok # In-flight tracking to nodes 2 and 3 is saturated, but node 3 is behind. status 1 ---- -1: StateReplicate match=14 next=18 -2: StateReplicate match=14 next=18 paused inflight=3[full] -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=14 next=18 sentCommit=11 matchCommit=11 +2: StateReplicate match=14 next=18 sentCommit=14 matchCommit=14 paused inflight=3[full] +3: StateReplicate match=11 next=15 sentCommit=14 matchCommit=11 paused inflight=3[full] log-level none ---- @@ -117,9 +117,9 @@ ok # Expect that the entries are committed and stored only on nodes 1 and 2. status 1 ---- -1: StateReplicate match=17 next=18 -2: StateReplicate match=17 next=18 -3: StateReplicate match=11 next=15 paused inflight=3[full] +1: StateReplicate match=17 next=18 sentCommit=14 matchCommit=14 +2: StateReplicate match=17 next=18 sentCommit=17 matchCommit=17 +3: StateReplicate match=11 next=15 sentCommit=14 matchCommit=11 paused inflight=3[full] # Make a heartbeat roundtrip. tick-heartbeat 1 @@ -188,6 +188,6 @@ ok # Eventually all nodes catch up on the committed state. status 1 ---- -1: StateReplicate match=17 next=18 -2: StateReplicate match=17 next=18 -3: StateReplicate match=17 next=18 +1: StateReplicate match=17 next=18 sentCommit=14 matchCommit=14 +2: StateReplicate match=17 next=18 sentCommit=17 matchCommit=17 +3: StateReplicate match=17 next=18 sentCommit=17 matchCommit=17 diff --git a/pkg/raft/testdata/slow_follower_after_compaction.txt b/pkg/raft/testdata/slow_follower_after_compaction.txt index 2ce02adae5c6..83a94fbbb6b0 100644 --- a/pkg/raft/testdata/slow_follower_after_compaction.txt +++ b/pkg/raft/testdata/slow_follower_after_compaction.txt @@ -43,9 +43,9 @@ ok # All nodes up-to-date. status 1 ---- -1: StateReplicate match=14 next=15 -2: StateReplicate match=14 next=15 -3: StateReplicate match=14 next=15 +1: StateReplicate match=14 next=15 sentCommit=11 matchCommit=11 +2: StateReplicate match=14 next=15 sentCommit=14 matchCommit=14 +3: StateReplicate match=14 next=15 sentCommit=14 matchCommit=14 log-level none ---- @@ -79,9 +79,9 @@ ok # Nodes 1 and 2 up-to-date, 3 is behind and MsgApp flow is throttled. status 1 ---- -1: StateReplicate match=18 next=19 -2: StateReplicate match=18 next=19 -3: StateReplicate match=14 next=17 paused inflight=2[full] +1: StateReplicate match=18 next=19 sentCommit=14 matchCommit=14 +2: StateReplicate match=18 next=19 sentCommit=18 matchCommit=18 +3: StateReplicate match=14 next=17 sentCommit=16 matchCommit=14 paused inflight=2[full] # Break the MsgApp flow from the leader to node 3. deliver-msgs drop=3 @@ -118,6 +118,6 @@ ok # All nodes caught up. status 1 ---- -1: StateReplicate match=18 next=19 -2: StateReplicate match=18 next=19 -3: StateReplicate match=18 next=19 +1: StateReplicate match=18 next=19 sentCommit=14 matchCommit=14 +2: StateReplicate match=18 next=19 sentCommit=18 matchCommit=18 +3: StateReplicate match=18 next=19 sentCommit=18 matchCommit=18 diff --git a/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt b/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt index d56e0d9a63d3..9c96104e3dc6 100644 --- a/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt +++ b/pkg/raft/testdata/snapshot_succeed_via_app_resp.txt @@ -41,9 +41,9 @@ ok status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateProbe match=0 next=11 paused inactive +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 +3: StateProbe match=0 next=11 sentCommit=10 matchCommit=0 paused inactive # Add the node that will receive a snapshot (it has no state at all, does not # even have a config). @@ -85,8 +85,8 @@ stabilize 1 ---- > 1 receiving messages 3->1 MsgHeartbeatResp Term:1 Log:0/0 - DEBUG 1 [firstindex: 12, commit: 11] sent snapshot[index: 11, term: 1] to 3 [StateProbe match=0 next=11] - DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=12 paused pendingSnap=11] + DEBUG 1 [firstindex: 12, commit: 11] sent snapshot[index: 11, term: 1] to 3 [StateProbe match=0 next=11 sentCommit=10 matchCommit=0] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=12 sentCommit=11 matchCommit=0 paused pendingSnap=11] > 1 handling Ready Ready MustSync=false: Messages: @@ -95,9 +95,9 @@ stabilize 1 status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateSnapshot match=0 next=12 paused pendingSnap=11 +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 +3: StateSnapshot match=0 next=12 sentCommit=11 matchCommit=0 paused pendingSnap=11 # Follower applies the snapshot. Note how it reacts with a MsgAppResp upon completion. # The snapshot fully catches the follower up (i.e. there are no more log entries it @@ -125,13 +125,13 @@ stabilize 1 ---- > 1 receiving messages 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=12 paused pendingSnap=11] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=12 sentCommit=11 matchCommit=11 paused pendingSnap=11] status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateReplicate match=11 next=12 +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 +3: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 # Let things settle. stabilize diff --git a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt index c683de2c9769..c28ea9e1ffcb 100644 --- a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt +++ b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -48,9 +48,9 @@ raft-state status 1 ---- -1: StateReplicate match=11 next=12 -2: StateReplicate match=11 next=12 -3: StateProbe match=0 next=11 paused inactive +1: StateReplicate match=11 next=12 sentCommit=10 matchCommit=10 +2: StateReplicate match=11 next=12 sentCommit=11 matchCommit=11 +3: StateProbe match=0 next=11 sentCommit=10 matchCommit=0 paused inactive raft-log 3 ---- @@ -83,9 +83,9 @@ ok status 1 ---- -1: StateReplicate match=12 next=13 -2: StateReplicate match=12 next=13 -3: StateProbe match=0 next=11 paused inactive +1: StateReplicate match=12 next=13 sentCommit=11 matchCommit=11 +2: StateReplicate match=12 next=13 sentCommit=12 matchCommit=12 +3: StateProbe match=0 next=11 sentCommit=10 matchCommit=0 paused inactive # 3 now gets the first MsgApp the leader originally sent, trying to append entry # 11 but this is rejected because the follower's log started at index 5. @@ -123,9 +123,9 @@ stabilize 1 > 1 receiving messages 3->1 MsgAppResp Term:1 Log:1/10 Rejected (Hint: 5) Commit:5 DEBUG 1 received MsgAppResp(rejected, hint: (index 5, term 1)) from 3 for index 10 - DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=6] - DEBUG 1 [firstindex: 11, commit: 12] sent snapshot[index: 12, term: 1] to 3 [StateProbe match=0 next=6] - DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=13 paused pendingSnap=12] + DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=6 sentCommit=5 matchCommit=5] + DEBUG 1 [firstindex: 11, commit: 12] sent snapshot[index: 12, term: 1] to 3 [StateProbe match=0 next=6 sentCommit=5 matchCommit=5] + DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=13 sentCommit=12 matchCommit=5 paused pendingSnap=12] > 1 handling Ready Ready MustSync=false: Messages: @@ -155,7 +155,7 @@ stabilize 1 > 1 receiving messages 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 LeadEpoch:1 3->1 MsgAppResp Term:1 Log:0/11 Commit:11 - DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=13 paused pendingSnap=12] + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=11 next=13 sentCommit=12 matchCommit=11 paused pendingSnap=12] > 1 handling Ready Ready MustSync=false: Messages: @@ -165,6 +165,6 @@ stabilize 1 # This is despite its PendingSnapshot having been 12. status 1 ---- -1: StateReplicate match=12 next=13 -2: StateReplicate match=12 next=13 -3: StateReplicate match=11 next=13 inflight=1 +1: StateReplicate match=12 next=13 sentCommit=11 matchCommit=11 +2: StateReplicate match=12 next=13 sentCommit=12 matchCommit=12 +3: StateReplicate match=11 next=13 sentCommit=12 matchCommit=11 inflight=1 diff --git a/pkg/raft/tracker/progress.go b/pkg/raft/tracker/progress.go index f3c448f9a81e..07d9ae9a9cfc 100644 --- a/pkg/raft/tracker/progress.go +++ b/pkg/raft/tracker/progress.go @@ -48,17 +48,21 @@ type Progress struct { // In StateSnapshot, Next == PendingSnapshot + 1. Next uint64 - // sentCommit is the highest commit index in flight to the follower. + // SentCommit is the highest commit index in flight to the follower. // // Generally, it is monotonic, but con regress in some cases, e.g. when // converting to `StateProbe` or when receiving a rejection from a follower. // - // In StateSnapshot, sentCommit == PendingSnapshot == Next-1. - sentCommit uint64 + // In StateSnapshot, SentCommit == PendingSnapshot == Next-1. + SentCommit uint64 - // matchCommit is the commit index at which the follower is known to match the + // MatchCommit is the commit index at which the follower is known to match the // leader. It is durable on the follower. - matchCommit uint64 + // Best-effort invariant: MatchCommit <= SentCommit + // It's a best-effort invariant because it doesn't really affect correctness. + // The worst case if MatchCommit > SentCommit is that the leader will send + // and extra MsgApp to the follower. + MatchCommit uint64 // State defines how the leader should interact with the follower. // @@ -151,7 +155,7 @@ func (pr *Progress) BecomeProbe() { pr.ResetState(StateProbe) pr.Next = pr.Match + 1 } - pr.sentCommit = min(pr.sentCommit, pr.Next-1) + pr.SentCommit = min(pr.SentCommit, pr.Next-1) } // BecomeReplicate transitions into StateReplicate, resetting Next to Match+1. @@ -166,7 +170,7 @@ func (pr *Progress) BecomeSnapshot(snapshoti uint64) { pr.ResetState(StateSnapshot) pr.PendingSnapshot = snapshoti pr.Next = snapshoti + 1 - pr.sentCommit = snapshoti + pr.SentCommit = snapshoti } // SentEntries updates the progress on the given number of consecutive entries @@ -197,7 +201,7 @@ func (pr *Progress) CanBumpCommit(index uint64) bool { // Next-1 in normal operation, or higher in some rare cases. Allow sending a // commit index eagerly only if we haven't already sent one that bumps the // follower's commit all the way to Next-1. - return index > pr.sentCommit && pr.sentCommit < pr.Next-1 + return index > pr.SentCommit && pr.SentCommit < pr.Next-1 } // IsFollowerCommitStale returns true if the follower's commit index it less @@ -205,12 +209,14 @@ func (pr *Progress) CanBumpCommit(index uint64) bool { // If the follower's commit index+1 is pr.Next, it means that sending a larger // commit index won't change anything, therefore we don't send it. func (pr *Progress) IsFollowerCommitStale(index uint64) bool { - return index > pr.matchCommit && pr.matchCommit+1 < pr.Next + return index > pr.MatchCommit && pr.MatchCommit+1 < pr.Next } -// SentCommit updates the sentCommit. -func (pr *Progress) SentCommit(commit uint64) { - pr.sentCommit = commit +// MaybeUpdateSentCommit updates the SentCommit if it needs to be updated. +func (pr *Progress) MaybeUpdateSentCommit(commit uint64) { + if commit > pr.SentCommit { + pr.SentCommit = commit + } } // MaybeUpdate is called when an MsgAppResp arrives from the follower, with the @@ -226,10 +232,11 @@ func (pr *Progress) MaybeUpdate(n uint64) bool { } // MaybeUpdateMatchCommit updates the match commit from a follower if it's -// larger than the previous received commit. +// larger than the previous match commit. func (pr *Progress) MaybeUpdateMatchCommit(commit uint64) { - if commit > pr.matchCommit { - pr.matchCommit = commit + if commit > pr.MatchCommit { + pr.MatchCommit = commit + pr.SentCommit = max(pr.SentCommit, commit) // Best-effort invariant: SentCommit >= MatchCommit } } @@ -255,8 +262,8 @@ func (pr *Progress) MaybeDecrTo(rejected, matchHint uint64) bool { // // TODO(tbg): why not use matchHint if it's larger? pr.Next = pr.Match + 1 - // Regress the sentCommit since it unlikely has been applied. - pr.sentCommit = min(pr.sentCommit, pr.Next-1) + // Regress the SentCommit since it unlikely has been applied. + pr.SentCommit = min(pr.SentCommit, pr.Next-1) return true } @@ -268,8 +275,8 @@ func (pr *Progress) MaybeDecrTo(rejected, matchHint uint64) bool { } pr.Next = max(min(rejected, matchHint+1), pr.Match+1) - // Regress the sentCommit since it unlikely has been applied. - pr.sentCommit = min(pr.sentCommit, pr.Next-1) + // Regress the SentCommit since it unlikely has been applied. + pr.SentCommit = min(pr.SentCommit, pr.Next-1) pr.MsgAppProbesPaused = false return true } @@ -374,7 +381,8 @@ func (pr *Progress) ShouldSendMsgApp(last, commit uint64, advanceCommit bool) bo func (pr *Progress) String() string { var buf strings.Builder - fmt.Fprintf(&buf, "%s match=%d next=%d", pr.State, pr.Match, pr.Next) + fmt.Fprintf(&buf, "%s match=%d next=%d sentCommit=%d matchCommit=%d", pr.State, pr.Match, + pr.Next, pr.SentCommit, pr.MatchCommit) if pr.IsLearner { fmt.Fprint(&buf, " learner") } diff --git a/pkg/raft/tracker/progress_test.go b/pkg/raft/tracker/progress_test.go index 5465af5559d0..a3edbc839c78 100644 --- a/pkg/raft/tracker/progress_test.go +++ b/pkg/raft/tracker/progress_test.go @@ -27,8 +27,10 @@ func TestProgressString(t *testing.T) { ins := NewInflights(1, 0) ins.Add(123, 1) pr := &Progress{ - Match: 1, - Next: 2, + MatchCommit: 1, + SentCommit: 2, + Match: 3, + Next: 4, State: StateSnapshot, PendingSnapshot: 123, RecentActive: false, @@ -36,7 +38,8 @@ func TestProgressString(t *testing.T) { IsLearner: true, Inflights: ins, } - const exp = `StateSnapshot match=1 next=2 learner paused pendingSnap=123 inactive inflight=1[full]` + const exp = "StateSnapshot match=3 next=4 sentCommit=2 matchCommit=1 learner paused " + + "pendingSnap=123 inactive inflight=1[full]" assert.Equal(t, exp, pr.String()) }