WorkerSuite.TestErrRefresh

15:00:15 FAIL: flag_test.go:94: WorkerSuite.TestErrRefresh
15:00:15
15:00:15 [LOG] 0:00.000 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Initial configuration (index=0): []
15:00:15 [LOG] 0:00.000 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Node at 95b9ad7b-6afc-4072-0140-f38681b5b768 [Follower] entering Follower state (Leader: "")
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_0 [WARN] raft: Heartbeat timeout from "" reached, starting election
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Node at 95b9ad7b-6afc-4072-0140-f38681b5b768 [Candidate] entering Candidate state in term 2
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_0 [DEBUG] raft: Votes needed: 1
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_0 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Election won. Tally: 1
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Node at 95b9ad7b-6afc-4072-0140-f38681b5b768 [Leader] entering Leader state
15:00:15 [LOG] 0:00.101 DEBUG juju.worker.raft.raftclusterer_test_1 [INFO] raft: Initial configuration (index=0): []
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Initial configuration (index=0): []
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Updating configuration with AddStaging (1, a3c3ad3f-20af-6c5a-ef81-cfc990067b6a) to [{Suffrage:Voter ID:0 Address:95b9ad7b-6afc-4072-0140-f38681b5b768} {Suffrage:Voter ID:1 Address:a3c3ad3f-20af-6c5a-ef81-cfc990067b6a}]
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Added peer 1, starting replication
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Updating configuration with AddStaging (2, 21e66717-166b-b073-e442-920ee2e3e30a) to [{Suffrage:Voter ID:0 Address:95b9ad7b-6afc-4072-0140-f38681b5b768} {Suffrage:Voter ID:1 Address:a3c3ad3f-20af-6c5a-ef81-cfc990067b6a} {Suffrage:Voter ID:2 Address:21e66717-166b-b073-e442-920ee2e3e30a}]
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: Added peer 2, starting replication
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_1 [INFO] raft: Node at a3c3ad3f-20af-6c5a-ef81-cfc990067b6a [Follower] entering Follower state (Leader: "")
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Node at 21e66717-166b-b073-e442-920ee2e3e30a [Follower] entering Follower state (Leader: "")
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Failed to get previous log: 4 log not found (last: 0)
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [WARN] raft: AppendEntries to {Voter 2 21e66717-166b-b073-e442-920ee2e3e30a} rejected, sending older logs (next: 1)
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: pipelining replication to peer {Voter 2 21e66717-166b-b073-e442-920ee2e3e30a}
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_1 [WARN] raft: Failed to get previous log: 3 log not found (last: 0)
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [WARN] raft: AppendEntries to {Voter 1 a3c3ad3f-20af-6c5a-ef81-cfc990067b6a} rejected, sending older logs (next: 1)
15:00:15 [LOG] 0:00.102 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: pipelining replication to peer {Voter 1 a3c3ad3f-20af-6c5a-ef81-cfc990067b6a}
15:00:15 [LOG] 0:00.103 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: aborting pipeline replication to peer {Voter 1 a3c3ad3f-20af-6c5a-ef81-cfc990067b6a}
15:00:15 [LOG] 0:00.103 DEBUG juju.worker.raft.raftclusterer_test_0 [INFO] raft: aborting pipeline replication to peer {Voter 2 21e66717-166b-b073-e442-920ee2e3e30a}
15:00:15 [LOG] 0:00.247 DEBUG juju.worker.raft.raftclusterer_test_1 [WARN] raft: Heartbeat timeout from "95b9ad7b-6afc-4072-0140-f38681b5b768" reached, starting election
15:00:15 [LOG] 0:00.247 DEBUG juju.worker.raft.raftclusterer_test_1 [INFO] raft: Node at a3c3ad3f-20af-6c5a-ef81-cfc990067b6a [Candidate] entering Candidate state in term 3
15:00:15 [LOG] 0:00.247 DEBUG juju.worker.raft.raftclusterer_test_1 [DEBUG] raft: Votes needed: 2
15:00:15 [LOG] 0:00.247 DEBUG juju.worker.raft.raftclusterer_test_1 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1
15:00:15 [LOG] 0:00.247 DEBUG juju.worker.raft.raftflag raft state changed: Candidate
15:00:15 [LOG] 0:00.247 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Rejecting vote request from a3c3ad3f-20af-6c5a-ef81-cfc990067b6a since we have a leader: 95b9ad7b-6afc-4072-0140-f38681b5b768
15:00:15 [LOG] 0:00.282 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Heartbeat timeout from "95b9ad7b-6afc-4072-0140-f38681b5b768" reached, starting election
15:00:15 [LOG] 0:00.282 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Node at 21e66717-166b-b073-e442-920ee2e3e30a [Candidate] entering Candidate state in term 3
15:00:15 [LOG] 0:00.282 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Votes needed: 2
15:00:15 [LOG] 0:00.282 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1
15:00:15 [LOG] 0:00.282 DEBUG juju.worker.raft.raftclusterer_test_1 [INFO] raft: Duplicate RequestVote for same term: 3
15:00:15 [LOG] 0:00.282 DEBUG juju.worker.raft.raftflag raft state changed: Candidate
15:00:15 [LOG] 0:00.297 DEBUG juju.worker.raft.raftclusterer_test_1 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.333 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.389 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Election timeout reached, restarting election
15:00:15 [LOG] 0:00.389 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Node at 21e66717-166b-b073-e442-920ee2e3e30a [Candidate] entering Candidate state in term 4
15:00:15 [LOG] 0:00.389 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Votes needed: 2
15:00:15 [LOG] 0:00.389 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_1 [INFO] raft: Node at a3c3ad3f-20af-6c5a-ef81-cfc990067b6a [Follower] entering Follower state (Leader: "")
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 2
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Election won. Tally: 2
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Node at 21e66717-166b-b073-e442-920ee2e3e30a [Leader] entering Leader state
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Added peer 0, starting replication
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: Added peer 1, starting replication
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftflag raft state changed: Follower
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftflag raft state changed: Leader
15:00:15 [LOG] 0:00.390 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: pipelining replication to peer {Voter 1 a3c3ad3f-20af-6c5a-ef81-cfc990067b6a}
15:00:15 [LOG] 0:00.440 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.440 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.454 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:00.490 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Failed to contact 0 in 100.136753ms
15:00:15 [LOG] 0:00.527 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:00.567 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.590 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Failed to contact 0 in 199.990889ms
15:00:15 [LOG] 0:00.606 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:00.674 DEBUG juju.worker.raft.raftclusterer_test_2 [WARN] raft: Failed to contact 0 in 284.604045ms
15:00:15 [LOG] 0:00.691 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:00.710 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.758 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 368.397232ms
15:00:15 [LOG] 0:00.801 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:00.846 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:00.856 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 466.466112ms
15:00:15 [LOG] 0:00.946 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 556.042598ms
15:00:15 [LOG] 0:00.948 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:01.030 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:01.031 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 641.782231ms
15:00:15 [LOG] 0:01.125 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 735.60894ms
15:00:15 [LOG] 0:01.172 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: command timed out
15:00:15 [LOG] 0:01.219 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 828.906789ms
15:00:15 [LOG] 0:01.256 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: command timed out
15:00:15 [LOG] 0:01.314 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 924.672194ms
15:00:15 [LOG] 0:01.401 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.010987456s
15:00:15 [LOG] 0:01.498 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.108215138s
15:00:15 [LOG] 0:01.536 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: send timed out
15:00:15 [LOG] 0:01.560 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: send timed out
15:00:15 [LOG] 0:01.593 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.203112828s
15:00:15 [LOG] 0:01.680 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.290319312s
15:00:15 [LOG] 0:01.775 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.385376712s
15:00:15 [LOG] 0:01.865 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.475392352s
15:00:15 [LOG] 0:01.956 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.566336279s
15:00:15 [LOG] 0:01.995 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: send timed out
15:00:15 [LOG] 0:02.051 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.6610445s
15:00:15 [LOG] 0:02.150 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.760514153s
15:00:15 [LOG] 0:02.249 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.859388536s
15:00:15 [LOG] 0:02.267 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: send timed out
15:00:15 [LOG] 0:02.348 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 1.958522479s
15:00:15 [LOG] 0:02.435 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.045529364s
15:00:15 [LOG] 0:02.531 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.141606164s
15:00:15 [LOG] 0:02.630 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.240693731s
15:00:15 [LOG] 0:02.718 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.328773028s
15:00:15 [LOG] 0:02.762 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: send timed out
15:00:15 [LOG] 0:02.806 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.416785785s
15:00:15 [LOG] 0:02.905 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.514962954s
15:00:15 [LOG] 0:03.000 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.610079288s
15:00:15 [LOG] 0:03.089 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.699717112s
15:00:15 [LOG] 0:03.181 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.791087373s
15:00:15 [LOG] 0:03.278 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.888168524s
15:00:15 [LOG] 0:03.375 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 2.985737104s
15:00:15 [LOG] 0:03.474 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.084477633s
15:00:15 [LOG] 0:03.567 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.177177934s
15:00:15 [LOG] 0:03.619 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: send timed out
15:00:15 [LOG] 0:03.658 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.26823841s
15:00:15 [LOG] 0:03.758 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.367969221s
15:00:15 [LOG] 0:03.846 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.456536711s
15:00:15 [LOG] 0:03.946 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.556685811s
15:00:15 [LOG] 0:04.034 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.644829781s
15:00:15 [LOG] 0:04.128 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.738015953s
15:00:15 [LOG] 0:04.162 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: send timed out
15:00:15 [LOG] 0:04.220 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.830504796s
15:00:15 [LOG] 0:04.303 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 3.913880875s
15:00:15 [LOG] 0:04.393 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.003361523s
15:00:15 [LOG] 0:04.476 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.086387142s
15:00:15 [LOG] 0:04.570 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.179933436s
15:00:15 [LOG] 0:04.653 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.263739328s
15:00:15 [LOG] 0:04.752 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.362112126s
15:00:15 [LOG] 0:04.845 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.455481711s
15:00:15 [LOG] 0:04.941 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.551657562s
15:00:15 [LOG] 0:05.033 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.643647161s
15:00:15 [LOG] 0:05.121 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.731572385s
15:00:15 [LOG] 0:05.215 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.82542678s
15:00:15 [LOG] 0:05.312 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 4.922149941s
15:00:15 [LOG] 0:05.402 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.012237932s
15:00:15 [LOG] 0:05.501 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.111428718s
15:00:15 [LOG] 0:05.587 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.19752631s
15:00:15 [LOG] 0:05.686 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.29652464s
15:00:15 [LOG] 0:05.768 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.378336039s
15:00:15 [LOG] 0:05.862 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.472612258s
15:00:15 [LOG] 0:05.961 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.571385379s
15:00:15 [LOG] 0:06.056 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.666781346s
15:00:15 [LOG] 0:06.152 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.762217782s
15:00:15 [LOG] 0:06.236 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.846818476s
15:00:15 [LOG] 0:06.247 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to heartbeat to 95b9ad7b-6afc-4072-0140-f38681b5b768: send timed out
15:00:15 [LOG] 0:06.335 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 5.945590607s
15:00:15 [LOG] 0:06.431 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.041657153s
15:00:15 [LOG] 0:06.531 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.141816053s
15:00:15 [LOG] 0:06.625 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.235258677s
15:00:15 [LOG] 0:06.724 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.334182173s
15:00:15 [LOG] 0:06.816 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.426534128s
15:00:15 [LOG] 0:06.861 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: send timed out
15:00:15 [LOG] 0:06.911 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.52142721s
15:00:15 [LOG] 0:07.002 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.612227328s
15:00:15 [LOG] 0:07.098 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.708382048s
15:00:15 [LOG] 0:07.186 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.796537149s
15:00:15 [LOG] 0:07.283 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.893756808s
15:00:15 [LOG] 0:07.375 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 6.985863314s
15:00:15 [LOG] 0:07.460 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.070678291s
15:00:15 [LOG] 0:07.556 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.166280714s
15:00:15 [LOG] 0:07.640 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.249934257s
15:00:15 [LOG] 0:07.736 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.3467608s
15:00:15 [LOG] 0:07.826 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.436516163s
15:00:15 [LOG] 0:07.922 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.532325878s
15:00:15 [LOG] 0:08.010 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.620631634s
15:00:15 [LOG] 0:08.106 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.716569478s
15:00:15 [LOG] 0:08.206 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.816727876s
15:00:15 [LOG] 0:08.306 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 7.916122715s
15:00:15 [LOG] 0:08.398 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.008761744s
15:00:15 [LOG] 0:08.496 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.106120046s
15:00:15 [LOG] 0:08.593 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.203495272s
15:00:15 [LOG] 0:08.699 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.309144547s
15:00:15 [LOG] 0:08.789 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.399212384s
15:00:15 [LOG] 0:08.881 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.491542718s
15:00:15 [LOG] 0:08.977 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.587512856s
15:00:15 [LOG] 0:09.063 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.673373259s
15:00:15 [LOG] 0:09.156 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.766260403s
15:00:15 [LOG] 0:09.248 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.858548836s
15:00:15 [LOG] 0:09.345 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 8.955171814s
15:00:15 [LOG] 0:09.440 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.050236764s
15:00:15 [LOG] 0:09.526 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.136598887s
15:00:15 [LOG] 0:09.622 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.232750299s
15:00:15 [LOG] 0:09.715 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.325246287s
15:00:15 [LOG] 0:09.809 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.419527452s
15:00:15 [LOG] 0:09.903 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.512892089s
15:00:15 [LOG] 0:09.998 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.608287752s
15:00:15 [LOG] 0:10.093 DEBUG juju.worker.raft.raftclusterer_test_2 [DEBUG] raft: Failed to contact 0 in 9.703065657s
15:00:15 flag_test.go:131:
15:00:15 // When the raft node toggles between leader/follower,
15:00:15 // then the worker will exit with ErrRefresh.
15:00:15 err := workertest.CheckKilled(c, s.worker)
15:00:15 /<email address hidden>/workertest/check.go:58:
15:00:15 c.Errorf("timed out waiting for worker to stop")
15:00:15 ... Error: timed out waiting for worker to stop
15:00:15
15:00:15 flag_test.go:132:
15:00:15 c.Assert(err, gc.Equals, raftflag.ErrRefresh)
15:00:15 ... obtained *errors.Err = &errors.unformatter{message:"workertest: worker not stopping", cause:error(nil), previous:error(nil), file:"/<email address hidden>/workertest/check.go", line:59} ("workertest: worker not stopping")
15:00:15 ... expected *errors.Err = &errors.unformatter{message:"raft leadership changed, restart worker", cause:error(nil), previous:error(nil), file:"github.com/juju/juju/worker/raft/raftflag/flag.go", line:32} ("raft leadership changed, restart worker")
15:00:15 ... Difference:
15:00:15 ... message: "workertest: worker not stopping" != "raft leadership changed, restart worker"
15:00:15 ... file: "/<email address hidden>/workertest/check.go" != "github.com/juju/juju/worker/raft/raftflag/flag.go"
15:00:15 ... line: 59 != 32
15:00:15
15:00:15
15:00:15 [LOG] 0:10.107 DEBUG juju.worker.raft.raftclusterer_test_2 [INFO] raft: aborting pipeline replication to peer {Voter 1 a3c3ad3f-20af-6c5a-ef81-cfc990067b6a}
15:00:15 [LOG] 0:10.157 DEBUG juju.worker.raft.raftclusterer_test_2 [ERR] raft: Failed to AppendEntries to {Voter 0 95b9ad7b-6afc-4072-0140-f38681b5b768}: send timed out