From da718dd0f84007f5958a4d46bf7634b2f694e40b Mon Sep 17 00:00:00 2001 From: Zach Schoenberger Date: Tue, 17 Oct 2023 06:57:39 -0700 Subject: [PATCH] small fix based on testing --- openraft/src/core/sm/mod.rs | 13 +- .../src/raft_state/membership_state/mod.rs | 7 + tests/_log/ut.2023-10-17-13 | 2335 +++++++++++++++++ .../t33_snapshot_delete_conflict_logs.rs | 28 +- 4 files changed, 2371 insertions(+), 12 deletions(-) create mode 100644 tests/_log/ut.2023-10-17-13 diff --git a/openraft/src/core/sm/mod.rs b/openraft/src/core/sm/mod.rs index ab9662c66..9a78c7936 100644 --- a/openraft/src/core/sm/mod.rs +++ b/openraft/src/core/sm/mod.rs @@ -274,11 +274,16 @@ where } InstallSnapshotData::Chunk(chunk) => match self.streaming.as_mut() { Some(streaming) => { - if streaming.receive(chunk).await? { - streaming.manifest.is_complete() - } else { - false + if !streaming.receive(chunk).await? { + // we received a chunk that doesn't exist in the manifest + tracing::warn!( + snapshot_req_id = debug(&req_id), + "{} chunk does not exist in manifest", + func_name!() + ); } + + streaming.manifest.is_complete() } None => { tracing::error!("should never happen"); diff --git a/openraft/src/raft_state/membership_state/mod.rs b/openraft/src/raft_state/membership_state/mod.rs index fdeddd3a1..9bdf74ac2 100644 --- a/openraft/src/raft_state/membership_state/mod.rs +++ b/openraft/src/raft_state/membership_state/mod.rs @@ -15,6 +15,7 @@ mod change_handler; #[cfg(test)] mod membership_state_test; pub(crate) use change_handler::ChangeHandler; +use tracing::info; /// The state of membership configs a raft node needs to know. /// @@ -85,6 +86,7 @@ where pub(crate) fn commit(&mut self, committed_log_id: &Option>) { if committed_log_id >= self.effective().log_id() { debug_assert!(committed_log_id.index() >= self.effective().log_id().index()); + info!("committed: {:?} effective: {:?}", self.committed, self.effective); self.committed = self.effective.clone(); } } @@ -123,6 +125,7 @@ where } if c.log_id() > self.committed.log_id() { + info!("c.log_id: {:?} committed: {:?}", c.log_id(), self.committed.log_id()); self.committed = c } @@ -151,6 +154,10 @@ where // Openraft allows at most only one non-committed membership config. // If there is another new config, self.effective must have been committed. + info!( + "append: committed: {:?} effective: {:?}", + self.committed, self.effective + ); self.committed = self.effective.clone(); self.effective = m; } diff --git a/tests/_log/ut.2023-10-17-13 b/tests/_log/ut.2023-10-17-13 new file mode 100644 index 000000000..73eab5814 --- /dev/null +++ b/tests/_log/ut.2023-10-17-13 @@ -0,0 +1,2335 @@ +2023-10-17T13:39:16.516904Z INFO ThreadId(02) initialized global tracing: in _log/ut at DEBUG +2023-10-17T13:39:16.521703Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: new +2023-10-17T13:39:16.525350Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.533831Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- manually init node-0 with a higher vote, in order to override conflict log on learner later +2023-10-17T13:39:16.535845Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: save_vote vote=Vote { leader_id: LeaderId { term: 4, node_id: 0 }, committed: false } +2023-10-17T13:39:16.540310Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: new +2023-10-17T13:39:16.540331Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: enter +2023-10-17T13:39:16.540728Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: new +2023-10-17T13:39:16.540750Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.543077Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: new +2023-10-17T13:39:16.543705Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: new +2023-10-17T13:39:16.545054Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: enter +2023-10-17T13:39:16.545233Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: exit +2023-10-17T13:39:16.549012Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: RaftStorage::get_membership membership_in_sm=StoredMembership { log_id: None, membership: Membership { configs: [], nodes: {} } } membership_in_log=[StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }] +2023-10-17T13:39:16.550894Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: save_vote vote=Vote { leader_id: LeaderId { term: 4, node_id: 0 }, committed: false } +2023-10-17T13:39:16.551765Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.551821Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: exit +2023-10-17T13:39:16.551977Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: new +2023-10-17T13:39:16.552000Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.552071Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.552729Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.552894Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: new +2023-10-17T13:39:16.552920Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.553531Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: raft node is initializing +2023-10-17T13:39:16.554065Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: new +2023-10-17T13:39:16.554084Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: enter +2023-10-17T13:39:16.554326Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: startup: state: RaftState { vote: UTime { data: Vote { leader_id: LeaderId { term: 4, node_id: 0 }, committed: false }, utime: Some(Instant { t: 1043048200198000 }) }, committed: None, purged_next: 0, log_ids: LogIdList { key_log_ids: [LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }] }, membership_state: MembershipState { committed: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} }, effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } }, snapshot_meta: SnapshotMeta { last_log_id: None, last_membership: StoredMembership { log_id: None, membership: Membership { configs: [], nodes: {} } }, snapshot_id: "" }, server_state: Learner, accepted: Accepted { leader_id: LeaderId { term: 0, node_id: 0 }, log_id: None }, io_state: IOState { building_snapshot: false, vote: Vote { leader_id: LeaderId { term: 4, node_id: 0 }, committed: false }, flushed: LogIOId { leader_id: LeaderId { term: 0, node_id: 0 }, log_id: None }, applied: None, purged: None }, snapshot_streaming: None, purge_upto: None } +2023-10-17T13:39:16.554550Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: startup: is_leader: false +2023-10-17T13:39:16.554824Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: startup: is_voter: true +2023-10-17T13:39:16.554896Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: startup: id=0 target_state: Follower +2023-10-17T13:39:16.554917Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: exit +2023-10-17T13:39:16.554982Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::engine::engine_impl@startup#8000000002: close time.busy=832µs time.idle=83.6µs +2023-10-17T13:39:16.555137Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 server-state .state -> Leader latest: Metrics{id:0, Follower, term:0, vote:0-0:uncommitted, last_log:None, last_applied:None, leader:None, membership:{log_id:None, voters:[], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.555228Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.9986025s +2023-10-17T13:39:16.555470Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.556065Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@run_engine_commands#8008000000002: new +2023-10-17T13:39:16.556096Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@run_engine_commands#8008000000002: enter +2023-10-17T13:39:16.556573Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@run_engine_commands#8008000000002: queued commands: start... +2023-10-17T13:39:16.556633Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@run_engine_commands#8008000000002: queued commands: end... +2023-10-17T13:39:16.556725Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@run_engine_commands#8008000000002: exit +2023-10-17T13:39:16.556750Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@run_engine_commands#8008000000002: close time.busy=629µs time.idle=56.2µs +2023-10-17T13:39:16.557113Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: new +2023-10-17T13:39:16.557143Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: enter +2023-10-17T13:39:16.557463Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: openraft::core::raft_core@current_leader#8000000003: new +2023-10-17T13:39:16.557479Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: openraft::core::raft_core@current_leader#8000000003: enter +2023-10-17T13:39:16.557562Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: openraft::core::raft_core@current_leader#8000000003: get current_leader self_id=0 vote=4-0:uncommitted +2023-10-17T13:39:16.557663Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: openraft::core::raft_core@current_leader#8000000003: exit +2023-10-17T13:39:16.557682Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: openraft::core::raft_core@current_leader#8000000003: close time.busy=184µs time.idle=35.2µs +2023-10-17T13:39:16.558034Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: report_metrics: Metrics{id:0, Follower, term:4, vote:4-0:uncommitted, last_log:0, last_applied:None, leader:None, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.558091Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.558096Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: exit +2023-10-17T13:39:16.558110Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@report_metrics#10008000000002: close time.busy=954µs time.idle=42.9µs +2023-10-17T13:39:16.558440Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: new +2023-10-17T13:39:16.558474Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.558510Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 server-state .state -> Leader latest: Metrics{id:0, Follower, term:4, vote:4-0:uncommitted, last_log:0, last_applied:None, leader:None, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.558522Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.995230958s +2023-10-17T13:39:16.558540Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.558833Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: new +2023-10-17T13:39:16.558850Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: enter +2023-10-17T13:39:16.558872Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: new +2023-10-17T13:39:16.558893Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: enter +2023-10-17T13:39:16.558914Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: openraft::core::raft_core@current_leader#8000000005: new +2023-10-17T13:39:16.558933Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: openraft::core::raft_core@current_leader#8000000005: enter +2023-10-17T13:39:16.558953Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: openraft::core::raft_core@current_leader#8000000005: get current_leader self_id=0 vote=4-0:uncommitted +2023-10-17T13:39:16.558975Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: openraft::core::raft_core@current_leader#8000000005: exit +2023-10-17T13:39:16.558993Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: openraft::core::raft_core@current_leader#8000000005: close time.busy=41.5µs time.idle=37.2µs +2023-10-17T13:39:16.559043Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: report_metrics: Metrics{id:0, Follower, term:4, vote:4-0:uncommitted, last_log:0, last_applied:None, leader:None, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.559071Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: exit +2023-10-17T13:39:16.559087Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: openraft::core::raft_core@report_metrics#8000000004: close time.busy=178µs time.idle=37.5µs +2023-10-17T13:39:16.559108Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.559122Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: exit +2023-10-17T13:39:16.559145Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8008000000003: close time.busy=271µs time.idle=41.4µs +2023-10-17T13:39:16.559170Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 server-state .state -> Leader latest: Metrics{id:0, Follower, term:4, vote:4-0:uncommitted, last_log:0, last_applied:None, leader:None, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.559185Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.994568s +2023-10-17T13:39:16.559210Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.559271Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.559297Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.559313Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.626016Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: enter +2023-10-17T13:39:16.626390Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: Tick sent: 1 +2023-10-17T13:39:16.626492Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: exit +2023-10-17T13:39:16.626514Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.626592Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.626642Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.626879Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: new +2023-10-17T13:39:16.626896Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: enter +2023-10-17T13:39:16.627023Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: recv from rx_notify: Tick 1 +2023-10-17T13:39:16.627360Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: received tick: 1, now: Instant { t: 1043048276657375 } +2023-10-17T13:39:16.627767Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: new +2023-10-17T13:39:16.627787Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: enter +2023-10-17T13:39:16.628065Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: try to trigger election by tick, now: Instant { t: 1043048277418375 } +2023-10-17T13:39:16.628281Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: this is the only voter, do election at once +2023-10-17T13:39:16.628594Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: do trigger election +2023-10-17T13:39:16.628750Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: new +2023-10-17T13:39:16.628788Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: enter +2023-10-17T13:39:16.629051Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl::Engine<_>::elect vote=5-0:uncommitted +2023-10-17T13:39:16.629308Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: vote is changing to vote=5-0:uncommitted +2023-10-17T13:39:16.629418Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: vote is changing from 4-0:uncommitted to 5-0:uncommitted +2023-10-17T13:39:16.629558Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::handler::vote_handler::VoteHandler<_>::update_vote now=Instant { t: 1043048279168334 } +2023-10-17T13:39:16.630144Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::raft_state@calc_server_state#c000000004: new +2023-10-17T13:39:16.630164Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::raft_state@calc_server_state#c000000004: enter +2023-10-17T13:39:16.630350Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::raft_state@calc_server_state#c000000004: states is_member=true is_leader=false is_leading=true +2023-10-17T13:39:16.630377Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::raft_state@calc_server_state#c000000004: exit +2023-10-17T13:39:16.630397Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::raft_state@calc_server_state#c000000004: close time.busy=213µs time.idle=40.9µs +2023-10-17T13:39:16.630564Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: update_server_state_if_changed id=0 prev_server_state=Follower server_state=Candidate +2023-10-17T13:39:16.630691Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::leader::voting::Voting<_, _, _>::grant_by voting=Voting { starting_time: Instant { t: 1043048280223709 }, vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: false }, last_log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), progress: VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: true, voter_count: 1, vector: [(0, true)], stat: Stat { update_count: 1, move_count: 1, is_quorum_count: 1 } } } +2023-10-17T13:39:16.631098Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: new +2023-10-17T13:39:16.631118Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: enter +2023-10-17T13:39:16.631232Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::engine_impl::Engine<_>::establish_leader +2023-10-17T13:39:16.631255Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: vote is changing to vote=5-0:committed +2023-10-17T13:39:16.631274Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: vote is changing from 5-0:uncommitted to 5-0:committed +2023-10-17T13:39:16.631293Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::vote_handler::VoteHandler<_>::update_vote now=Instant { t: 1043048280904042 } +2023-10-17T13:39:16.631317Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::raft_state@calc_server_state#c000000005: new +2023-10-17T13:39:16.631337Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::raft_state@calc_server_state#c000000005: enter +2023-10-17T13:39:16.631356Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::raft_state@calc_server_state#c000000005: states is_member=true is_leader=true is_leading=true +2023-10-17T13:39:16.631379Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::raft_state@calc_server_state#c000000005: exit +2023-10-17T13:39:16.631398Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::raft_state@calc_server_state#c000000005: close time.busy=42.3µs time.idle=38.9µs +2023-10-17T13:39:16.631423Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: update_server_state_if_changed id=0 prev_server_state=Candidate server_state=Leader +2023-10-17T13:39:16.631492Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: become leader id=0 +2023-10-17T13:39:16.631802Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@rebuild_replication_streams#800c000000005: new +2023-10-17T13:39:16.631823Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@rebuild_replication_streams#800c000000005: enter +2023-10-17T13:39:16.631885Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@rebuild_replication_streams#800c000000005: exit +2023-10-17T13:39:16.631906Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@rebuild_replication_streams#800c000000005: close time.busy=62.3µs time.idle=41.6µs +2023-10-17T13:39:16.632222Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: new +2023-10-17T13:39:16.632242Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: enter +2023-10-17T13:39:16.632328Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-1 +2023-10-17T13:39:16.632445Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: update progress self_matching=None +2023-10-17T13:39:16.632619Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: new +2023-10-17T13:39:16.632642Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: enter +2023-10-17T13:39:16.632738Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-1 +2023-10-17T13:39:16.633113Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: update_matching self={[None, 1), inflight:Logs(id=0):(None, Some(5-0-1)]} request_id=0 matching=Some(5-0-1) +2023-10-17T13:39:16.633207Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: granted after updating progress granted=5-0-1 +2023-10-17T13:39:16.633332Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: new +2023-10-17T13:39:16.633354Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: enter +2023-10-17T13:39:16.633548Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: openraft::raft_state@update_committed#c000000008: new +2023-10-17T13:39:16.633572Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: openraft::raft_state@update_committed#c000000008: enter +2023-10-17T13:39:16.633930Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: openraft::raft_state@update_committed#c000000008: committed: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.633971Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: openraft::raft_state@update_committed#c000000008: exit +2023-10-17T13:39:16.633994Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: openraft::raft_state@update_committed#c000000008: close time.busy=399µs time.idle=47.3µs +2023-10-17T13:39:16.634023Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: exit +2023-10-17T13:39:16.634045Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: openraft::engine::handler::replication_handler@try_commit_granted#c000000007: close time.busy=669µs time.idle=44.0µs +2023-10-17T13:39:16.634070Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: exit +2023-10-17T13:39:16.634090Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: openraft::engine::handler::replication_handler@update_matching#c000000006: close time.busy=1.43ms time.idle=43.5µs +2023-10-17T13:39:16.634115Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: exit +2023-10-17T13:39:16.634134Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@append_blank_log#1000c000000005: close time.busy=1.87ms time.idle=39.8µs +2023-10-17T13:39:16.634259Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@initiate_replication#1800c000000005: new +2023-10-17T13:39:16.634279Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@initiate_replication#1800c000000005: enter +2023-10-17T13:39:16.634435Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@initiate_replication#1800c000000005: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }), curr_inflight_id: 0, inflight: None, searching_end: 2 })], stat: Stat { update_count: 1, move_count: 1, is_quorum_count: 1 } } +2023-10-17T13:39:16.634466Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@initiate_replication#1800c000000005: exit +2023-10-17T13:39:16.634486Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: openraft::engine::handler::replication_handler@initiate_replication#1800c000000005: close time.busy=187µs time.idle=40.3µs +2023-10-17T13:39:16.634509Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: exit +2023-10-17T13:39:16.634527Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: openraft::engine::engine_impl@establish_leader#800c000000004: close time.busy=3.39ms time.idle=39.0µs +2023-10-17T13:39:16.634550Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: exit +2023-10-17T13:39:16.634567Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: openraft::engine::engine_impl@elect#c000000003: close time.busy=5.76ms time.idle=56.3µs +2023-10-17T13:39:16.634589Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: exit +2023-10-17T13:39:16.634605Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: openraft::core::raft_core@handle_tick_election#c000000002: close time.busy=6.80ms time.idle=36.6µs +2023-10-17T13:39:16.634643Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: exit +2023-10-17T13:39:16.634667Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000000001{state=Follower id=0}: close time.busy=7.75ms time.idle=41.7µs +2023-10-17T13:39:16.634726Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: new +2023-10-17T13:39:16.634742Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: enter +2023-10-17T13:39:16.634758Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: start... +2023-10-17T13:39:16.634856Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: SaveVote { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: false } } +2023-10-17T13:39:16.635142Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: SaveVote { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } } +2023-10-17T13:39:16.635158Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: BecomeLeader +2023-10-17T13:39:16.635181Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: RebuildReplicationStreams { targets: [] } +2023-10-17T13:39:16.635205Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: AppendEntry { entry: Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }, payload: blank } } +2023-10-17T13:39:16.635229Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }) } +2023-10-17T13:39:16.635262Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: Apply { seq: 1, already_committed: None, upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 } } +2023-10-17T13:39:16.635289Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: queued commands: end... +2023-10-17T13:39:16.635352Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: SaveVote { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: false } } +2023-10-17T13:39:16.635633Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.635712Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: save_vote vote=Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: false } +2023-10-17T13:39:16.636009Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: SaveVote { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } } +2023-10-17T13:39:16.636026Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.636044Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: save_vote vote=Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } +2023-10-17T13:39:16.636063Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: BecomeLeader +2023-10-17T13:39:16.636084Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.636137Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: RebuildReplicationStreams { targets: [] } +2023-10-17T13:39:16.636157Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.636335Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@remove_all_replication#800c000000002: new +2023-10-17T13:39:16.636387Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@remove_all_replication#800c000000002: enter +2023-10-17T13:39:16.636594Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@remove_all_replication#800c000000002: remove all replication +2023-10-17T13:39:16.636675Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@remove_all_replication#800c000000002: remove all targets from replication_metrics targets=[] +2023-10-17T13:39:16.636948Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@remove_all_replication#800c000000002: exit +2023-10-17T13:39:16.636964Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@remove_all_replication#800c000000002: close time.busy=561µs time.idle=68.7µs +2023-10-17T13:39:16.636989Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: AppendEntry { entry: Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }, payload: blank } } +2023-10-17T13:39:16.637011Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.637124Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: AppendEntry: 5-0-1:blank +2023-10-17T13:39:16.637304Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@append_to_log#1000c000000002: new +2023-10-17T13:39:16.637336Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@append_to_log#1000c000000002: enter +2023-10-17T13:39:16.637557Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@append_to_log#1000c000000002: append_to_log +2023-10-17T13:39:16.638174Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@append_to_log#1000c000000002: exit +2023-10-17T13:39:16.638206Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@append_to_log#1000c000000002: close time.busy=838µs time.idle=64.5µs +2023-10-17T13:39:16.638249Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-1 +2023-10-17T13:39:16.638267Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: update progress self_matching=5-0-1 +2023-10-17T13:39:16.638287Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }) } +2023-10-17T13:39:16.638316Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.638366Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: run command: Apply { seq: 1, already_committed: None, upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 } } +2023-10-17T13:39:16.638390Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: condition: None +2023-10-17T13:39:16.638548Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: new +2023-10-17T13:39:16.638628Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: enter +2023-10-17T13:39:16.638825Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: apply_to_state_machine upto_index=1 +2023-10-17T13:39:16.639138Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: about to apply entries=[0-0-0:membership: voters:[{0:{()}}], learners:[],5-0-1:blank] +2023-10-17T13:39:16.639242Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: sending command to state machine worker: StateMachineCommand { seq: 1, payload: Apply: [0-0-0:membership: voters:[{0:{()}}], learners:[],5-0-1:blank] } +2023-10-17T13:39:16.639527Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: exit +2023-10-17T13:39:16.639560Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: openraft::core::raft_core@apply_to_state_machine#1800c000000002: close time.busy=900µs time.idle=112µs +2023-10-17T13:39:16.639613Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: exit +2023-10-17T13:39:16.639632Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#800c000000001: close time.busy=4.87ms time.idle=35.3µs +2023-10-17T13:39:16.640405Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.640706Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.640807Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: new +2023-10-17T13:39:16.640823Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: enter +2023-10-17T13:39:16.640853Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: new +2023-10-17T13:39:16.640869Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: enter +2023-10-17T13:39:16.640890Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: openraft::core::raft_core@current_leader#800c000000003: new +2023-10-17T13:39:16.640908Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: openraft::core::raft_core@current_leader#800c000000003: enter +2023-10-17T13:39:16.640926Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: openraft::core::raft_core@current_leader#800c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.640950Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: openraft::core::raft_core@current_leader#800c000000003: exit +2023-10-17T13:39:16.640968Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: openraft::core::raft_core@current_leader#800c000000003: close time.busy=42.7µs time.idle=34.5µs +2023-10-17T13:39:16.641055Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.641095Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.641090Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: exit +2023-10-17T13:39:16.641108Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: openraft::core::raft_core@report_metrics#2000c000000002: close time.busy=222µs time.idle=33.6µs +2023-10-17T13:39:16.641128Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: exit +2023-10-17T13:39:16.641146Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 server-state .state -> Leader latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.641143Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#1000c000000001: close time.busy=305µs time.idle=31.6µs +2023-10-17T13:39:16.641171Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.641189Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.641203Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.641291Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 done wait init node-0 server-state .state -> Leader latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.641455Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.641730Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 1, payload: Apply: [0-0-0:membership: voters:[{0:{()}}], learners:[],5-0-1:blank] } +2023-10-17T13:39:16.642456Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#1800c000000001: new +2023-10-17T13:39:16.642477Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#1800c000000001: enter +2023-10-17T13:39:16.642765Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 log .last_log_index -> Some(1) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.642972Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 done wait init node-0 log .last_log_index -> Some(1) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.643601Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#1800c000000001: replicate to sm entry.log_id=0-0-0 +2023-10-17T13:39:16.643646Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#1800c000000001: replicate to sm entry.log_id=5-0-1 +2023-10-17T13:39:16.643920Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#1800c000000001: exit +2023-10-17T13:39:16.643948Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#1800c000000001: close time.busy=1.44ms time.idle=50.4µs +2023-10-17T13:39:16.643999Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.644013Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.644025Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.644036Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 log .last_applied -> Some(1) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.644038Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.644065Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.644080Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.644110Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999786916s +2023-10-17T13:39:16.644150Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 1, result: Ok(Apply(ApplyResult { since: 0, end: 2, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 } })) } +2023-10-17T13:39:16.644170Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 log .last_applied -> Some(1) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:None, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.644176Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999656291s +2023-10-17T13:39:16.644194Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.644320Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 1, result: Ok(Apply(ApplyResult { since: 0, end: 2, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 } })) } +2023-10-17T13:39:16.644597Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-1 +2023-10-17T13:39:16.644704Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: new +2023-10-17T13:39:16.644722Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: enter +2023-10-17T13:39:16.644941Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-1 +2023-10-17T13:39:16.645076Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1000c000000003: new +2023-10-17T13:39:16.645096Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1000c000000003: enter +2023-10-17T13:39:16.645325Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1000c000000003: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }, membership: Some(Membership { configs: [{0}], nodes: {0: ()} }) } +2023-10-17T13:39:16.645353Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1000c000000003: exit +2023-10-17T13:39:16.645369Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1000c000000003: close time.busy=257µs time.idle=36.8µs +2023-10-17T13:39:16.645397Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1800c000000003: new +2023-10-17T13:39:16.645415Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1800c000000003: enter +2023-10-17T13:39:16.645431Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1800c000000003: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }, membership: None } +2023-10-17T13:39:16.645451Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1800c000000003: exit +2023-10-17T13:39:16.645468Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: openraft::core::raft_core@send_response#1800c000000003: close time.busy=35.7µs time.idle=34.9µs +2023-10-17T13:39:16.645492Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: exit +2023-10-17T13:39:16.645508Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#2800c000000002: close time.busy=770µs time.idle=35.0µs +2023-10-17T13:39:16.645550Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.645565Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#2000c000000001{state=Leader id=0}: close time.busy=1.47ms time.idle=30.9µs +2023-10-17T13:39:16.645587Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#2800c000000001: new +2023-10-17T13:39:16.645602Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#2800c000000001: enter +2023-10-17T13:39:16.645617Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#2800c000000001: queued commands: start... +2023-10-17T13:39:16.645632Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#2800c000000001: queued commands: end... +2023-10-17T13:39:16.645648Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#2800c000000001: exit +2023-10-17T13:39:16.645663Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#2800c000000001: close time.busy=46.5µs time.idle=29.5µs +2023-10-17T13:39:16.645683Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.645698Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.645715Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: new +2023-10-17T13:39:16.645730Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: enter +2023-10-17T13:39:16.645752Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: new +2023-10-17T13:39:16.645768Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: enter +2023-10-17T13:39:16.645788Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: openraft::core::raft_core@current_leader#2000c000000003: new +2023-10-17T13:39:16.645805Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: openraft::core::raft_core@current_leader#2000c000000003: enter +2023-10-17T13:39:16.645824Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: openraft::core::raft_core@current_leader#2000c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.645846Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: openraft::core::raft_core@current_leader#2000c000000003: exit +2023-10-17T13:39:16.645863Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: openraft::core::raft_core@current_leader#2000c000000003: close time.busy=40.8µs time.idle=33.6µs +2023-10-17T13:39:16.645903Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:5-0-1, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.645934Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: exit +2023-10-17T13:39:16.645942Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.645951Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: openraft::core::raft_core@report_metrics#3000c000000002: close time.busy=166µs time.idle=33.4µs +2023-10-17T13:39:16.645971Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: exit +2023-10-17T13:39:16.645985Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#3000c000000001: close time.busy=241µs time.idle=29.2µs +2023-10-17T13:39:16.646013Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.646027Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.646041Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.646098Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait init node-0 log .last_applied -> Some(1) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:5-0-1, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.646246Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 done wait init node-0 log .last_applied -> Some(1) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:1, last_applied:5-0-1, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-1}} +2023-10-17T13:39:16.646689Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- send just enough logs to trigger snapshot log_index=1 +2023-10-17T13:39:16.647283Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: new +2023-10-17T13:39:16.647309Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: enter +2023-10-17T13:39:16.647808Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: new +2023-10-17T13:39:16.647841Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: enter +2023-10-17T13:39:16.647998Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.648011Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.648024Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.648158Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: exit +2023-10-17T13:39:16.648199Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: exit +2023-10-17T13:39:16.648210Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.648626Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.648669Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.648802Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.649020Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: new +2023-10-17T13:39:16.649042Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: enter +2023-10-17T13:39:16.649259Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.649426Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#2800c000000003: new +2023-10-17T13:39:16.649449Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#2800c000000003: enter +2023-10-17T13:39:16.649526Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#2800c000000003: this node is a leader +2023-10-17T13:39:16.649555Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#2800c000000003: exit +2023-10-17T13:39:16.649573Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#2800c000000003: close time.busy=106µs time.idle=41.3µs +2023-10-17T13:39:16.649779Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: new +2023-10-17T13:39:16.649798Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: enter +2023-10-17T13:39:16.649864Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: assign log id: 5-0-2 +2023-10-17T13:39:16.649953Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: openraft::engine::handler::replication_handler@initiate_replication#1000c000000004: new +2023-10-17T13:39:16.649973Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: openraft::engine::handler::replication_handler@initiate_replication#1000c000000004: enter +2023-10-17T13:39:16.649993Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: openraft::engine::handler::replication_handler@initiate_replication#1000c000000004: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }), curr_inflight_id: 0, inflight: None, searching_end: 2 })], stat: Stat { update_count: 1, move_count: 1, is_quorum_count: 1 } } +2023-10-17T13:39:16.650025Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: openraft::engine::handler::replication_handler@initiate_replication#1000c000000004: exit +2023-10-17T13:39:16.650044Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: openraft::engine::handler::replication_handler@initiate_replication#1000c000000004: close time.busy=52.3µs time.idle=39.2µs +2023-10-17T13:39:16.650067Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: exit +2023-10-17T13:39:16.650085Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#3000c000000003: close time.busy=269µs time.idle=36.3µs +2023-10-17T13:39:16.650110Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: exit +2023-10-17T13:39:16.650126Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#3800c000000002{id=0}: close time.busy=1.07ms time.idle=38.2µs +2023-10-17T13:39:16.650159Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.650175Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#3800c000000001{state=Leader id=0}: close time.busy=1.49ms time.idle=59.8µs +2023-10-17T13:39:16.650200Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: new +2023-10-17T13:39:16.650215Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: enter +2023-10-17T13:39:16.650229Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: queued commands: start... +2023-10-17T13:39:16.650246Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }, payload: normal }] } +2023-10-17T13:39:16.650277Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: queued commands: end... +2023-10-17T13:39:16.650292Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }, payload: normal }] } +2023-10-17T13:39:16.650319Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: condition: None +2023-10-17T13:39:16.650462Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: AppendInputEntries: [5-0-2:normal] +2023-10-17T13:39:16.650676Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@append_to_log#4000c000000002: new +2023-10-17T13:39:16.650702Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@append_to_log#4000c000000002: enter +2023-10-17T13:39:16.650933Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@append_to_log#4000c000000002: append_to_log +2023-10-17T13:39:16.651000Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@append_to_log#4000c000000002: exit +2023-10-17T13:39:16.651030Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@append_to_log#4000c000000002: close time.busy=297µs time.idle=59.4µs +2023-10-17T13:39:16.651077Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-2 +2023-10-17T13:39:16.651095Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: update progress self_matching=5-0-1 +2023-10-17T13:39:16.651115Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: new +2023-10-17T13:39:16.651131Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: enter +2023-10-17T13:39:16.651148Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-2 +2023-10-17T13:39:16.651169Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: update_matching self={[Some(5-0-1), 2), inflight:Logs(id=0):(None, Some(5-0-2)]} request_id=0 matching=Some(5-0-2) +2023-10-17T13:39:16.651194Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: granted after updating progress granted=5-0-2 +2023-10-17T13:39:16.651215Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: new +2023-10-17T13:39:16.651232Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: enter +2023-10-17T13:39:16.651251Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: openraft::raft_state@update_committed#1800c000000004: new +2023-10-17T13:39:16.651269Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: openraft::raft_state@update_committed#1800c000000004: enter +2023-10-17T13:39:16.651287Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: openraft::raft_state@update_committed#1800c000000004: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.651312Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: openraft::raft_state@update_committed#1800c000000004: exit +2023-10-17T13:39:16.651330Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: openraft::raft_state@update_committed#1800c000000004: close time.busy=43.3µs time.idle=35.8µs +2023-10-17T13:39:16.651355Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: exit +2023-10-17T13:39:16.651372Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: openraft::engine::handler::replication_handler@try_commit_granted#3800c000000003: close time.busy=124µs time.idle=33.8µs +2023-10-17T13:39:16.651393Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: exit +2023-10-17T13:39:16.651408Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::engine::handler::replication_handler@update_matching#4800c000000002: close time.busy=261µs time.idle=31.5µs +2023-10-17T13:39:16.651431Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }) } +2023-10-17T13:39:16.651447Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: condition: None +2023-10-17T13:39:16.651462Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: run command: Apply { seq: 2, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 1 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 } } +2023-10-17T13:39:16.651477Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: condition: None +2023-10-17T13:39:16.651497Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: new +2023-10-17T13:39:16.651513Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: enter +2023-10-17T13:39:16.651528Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: apply_to_state_machine upto_index=2 +2023-10-17T13:39:16.651577Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: about to apply entries=[5-0-2:normal] +2023-10-17T13:39:16.651596Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: sending command to state machine worker: StateMachineCommand { seq: 2, payload: Apply: [5-0-2:normal] } +2023-10-17T13:39:16.651618Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: exit +2023-10-17T13:39:16.651634Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: openraft::core::raft_core@apply_to_state_machine#5000c000000002: close time.busy=105µs time.idle=32.2µs +2023-10-17T13:39:16.651655Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: exit +2023-10-17T13:39:16.651670Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#4000c000000001: close time.busy=1.44ms time.idle=30.1µs +2023-10-17T13:39:16.651691Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.651706Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.651723Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: new +2023-10-17T13:39:16.651738Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: enter +2023-10-17T13:39:16.651759Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: new +2023-10-17T13:39:16.651774Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: enter +2023-10-17T13:39:16.651794Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: openraft::core::raft_core@current_leader#4000c000000003: new +2023-10-17T13:39:16.651810Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: openraft::core::raft_core@current_leader#4000c000000003: enter +2023-10-17T13:39:16.651827Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: openraft::core::raft_core@current_leader#4000c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.651847Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: openraft::core::raft_core@current_leader#4000c000000003: exit +2023-10-17T13:39:16.651863Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: openraft::core::raft_core@current_leader#4000c000000003: close time.busy=36.9µs time.idle=32.8µs +2023-10-17T13:39:16.651906Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:2, last_applied:5-0-1, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-2}} +2023-10-17T13:39:16.651927Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: exit +2023-10-17T13:39:16.651942Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: openraft::core::raft_core@report_metrics#5800c000000002: close time.busy=153µs time.idle=30.8µs +2023-10-17T13:39:16.651962Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: exit +2023-10-17T13:39:16.651976Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#4800c000000001: close time.busy=224µs time.idle=29.0µs +2023-10-17T13:39:16.651998Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.652012Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.652023Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.652037Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.652047Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 2, payload: Apply: [5-0-2:normal] } +2023-10-17T13:39:16.652060Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#5000c000000001: new +2023-10-17T13:39:16.652067Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#5000c000000001: enter +2023-10-17T13:39:16.652082Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#5000c000000001: replicate to sm entry.log_id=5-0-2 +2023-10-17T13:39:16.652166Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#5000c000000001: exit +2023-10-17T13:39:16.652175Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#5000c000000001: close time.busy=98.5µs time.idle=17.0µs +2023-10-17T13:39:16.652192Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.652201Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.652210Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.652222Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.652243Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.652257Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.652288Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 2, result: Ok(Apply(ApplyResult { since: 2, end: 3, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 } })) } +2023-10-17T13:39:16.652307Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 2, result: Ok(Apply(ApplyResult { since: 2, end: 3, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 } })) } +2023-10-17T13:39:16.652327Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-2 +2023-10-17T13:39:16.652347Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: new +2023-10-17T13:39:16.652364Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: enter +2023-10-17T13:39:16.652379Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-2 +2023-10-17T13:39:16.652405Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core@send_response#4800c000000003: new +2023-10-17T13:39:16.652422Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core@send_response#4800c000000003: enter +2023-10-17T13:39:16.652438Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core@send_response#4800c000000003: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }, membership: None } +2023-10-17T13:39:16.652472Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.652545Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core@send_response#4800c000000003: send client response through tx, send_res is error: false +2023-10-17T13:39:16.652757Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: enter +2023-10-17T13:39:16.652750Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core@send_response#4800c000000003: exit +2023-10-17T13:39:16.652771Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: openraft::core::raft_core@send_response#4800c000000003: close time.busy=328µs time.idle=38.4µs +2023-10-17T13:39:16.652804Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: enter +2023-10-17T13:39:16.652802Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: exit +2023-10-17T13:39:16.652817Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#6000c000000002: close time.busy=439µs time.idle=31.3µs +2023-10-17T13:39:16.652839Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.652853Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#5800c000000001{state=Leader id=0}: close time.busy=581µs time.idle=29.5µs +2023-10-17T13:39:16.652876Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#6000c000000001: new +2023-10-17T13:39:16.652892Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#6000c000000001: enter +2023-10-17T13:39:16.652906Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#6000c000000001: queued commands: start... +2023-10-17T13:39:16.652921Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#6000c000000001: queued commands: end... +2023-10-17T13:39:16.652936Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#6000c000000001: exit +2023-10-17T13:39:16.652951Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#6000c000000001: close time.busy=44.6µs time.idle=31.1µs +2023-10-17T13:39:16.652970Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.652984Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.653000Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: new +2023-10-17T13:39:16.653012Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: call_core receives result is error: false +2023-10-17T13:39:16.653015Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: enter +2023-10-17T13:39:16.653033Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: new +2023-10-17T13:39:16.653048Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: exit +2023-10-17T13:39:16.653048Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: enter +2023-10-17T13:39:16.653067Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: openraft::core::raft_core@current_leader#5000c000000003: new +2023-10-17T13:39:16.653082Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: openraft::raft@call_core#7: close time.busy=559µs time.idle=4.72ms +2023-10-17T13:39:16.653084Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: openraft::core::raft_core@current_leader#5000c000000003: enter +2023-10-17T13:39:16.653100Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: openraft::core::raft_core@current_leader#5000c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.653118Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: exit +2023-10-17T13:39:16.653121Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: openraft::core::raft_core@current_leader#5000c000000003: exit +2023-10-17T13:39:16.653139Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#6: close time.busy=1.25ms time.idle=4.61ms +2023-10-17T13:39:16.653138Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: openraft::core::raft_core@current_leader#5000c000000003: close time.busy=37.3µs time.idle=34.2µs +2023-10-17T13:39:16.653177Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:2, last_applied:5-0-2, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-2}} +2023-10-17T13:39:16.653198Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: exit +2023-10-17T13:39:16.653214Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: openraft::core::raft_core@report_metrics#6800c000000002: close time.busy=150µs time.idle=32.4µs +2023-10-17T13:39:16.653232Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: new +2023-10-17T13:39:16.653239Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: enter +2023-10-17T13:39:16.653235Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: exit +2023-10-17T13:39:16.653250Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: new +2023-10-17T13:39:16.653248Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#6800c000000001: close time.busy=220µs time.idle=28.4µs +2023-10-17T13:39:16.653260Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: enter +2023-10-17T13:39:16.653272Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: exit +2023-10-17T13:39:16.653269Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.653284Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: exit +2023-10-17T13:39:16.653287Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.653294Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.653298Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.653365Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.653376Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.653387Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.653411Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.653426Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.653439Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.653459Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: new +2023-10-17T13:39:16.653475Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: enter +2023-10-17T13:39:16.653489Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.653510Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#5800c000000003: new +2023-10-17T13:39:16.653527Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#5800c000000003: enter +2023-10-17T13:39:16.653543Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#5800c000000003: this node is a leader +2023-10-17T13:39:16.653561Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#5800c000000003: exit +2023-10-17T13:39:16.653578Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#5800c000000003: close time.busy=34.5µs time.idle=33.8µs +2023-10-17T13:39:16.653602Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: new +2023-10-17T13:39:16.653619Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: enter +2023-10-17T13:39:16.653635Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: assign log id: 5-0-3 +2023-10-17T13:39:16.653656Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: openraft::engine::handler::replication_handler@initiate_replication#2000c000000004: new +2023-10-17T13:39:16.653674Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: openraft::engine::handler::replication_handler@initiate_replication#2000c000000004: enter +2023-10-17T13:39:16.653693Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: openraft::engine::handler::replication_handler@initiate_replication#2000c000000004: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }), curr_inflight_id: 0, inflight: None, searching_end: 3 })], stat: Stat { update_count: 2, move_count: 2, is_quorum_count: 2 } } +2023-10-17T13:39:16.653721Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: openraft::engine::handler::replication_handler@initiate_replication#2000c000000004: exit +2023-10-17T13:39:16.653739Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: openraft::engine::handler::replication_handler@initiate_replication#2000c000000004: close time.busy=46.8µs time.idle=35.8µs +2023-10-17T13:39:16.653761Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: exit +2023-10-17T13:39:16.653778Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#6000c000000003: close time.busy=142µs time.idle=35.0µs +2023-10-17T13:39:16.653805Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: exit +2023-10-17T13:39:16.653820Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#7000c000000002{id=0}: close time.busy=330µs time.idle=31.9µs +2023-10-17T13:39:16.653841Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.653856Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#7000c000000001{state=Leader id=0}: close time.busy=415µs time.idle=30.2µs +2023-10-17T13:39:16.653877Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: new +2023-10-17T13:39:16.653892Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: enter +2023-10-17T13:39:16.653913Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: queued commands: start... +2023-10-17T13:39:16.653928Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }, payload: normal }] } +2023-10-17T13:39:16.653946Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: queued commands: end... +2023-10-17T13:39:16.653958Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }, payload: normal }] } +2023-10-17T13:39:16.653973Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: condition: None +2023-10-17T13:39:16.653987Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: AppendInputEntries: [5-0-3:normal] +2023-10-17T13:39:16.654007Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@append_to_log#7800c000000002: new +2023-10-17T13:39:16.654023Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@append_to_log#7800c000000002: enter +2023-10-17T13:39:16.654037Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@append_to_log#7800c000000002: append_to_log +2023-10-17T13:39:16.654070Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@append_to_log#7800c000000002: exit +2023-10-17T13:39:16.654086Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@append_to_log#7800c000000002: close time.busy=47.0µs time.idle=32.2µs +2023-10-17T13:39:16.654106Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-3 +2023-10-17T13:39:16.654122Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: update progress self_matching=5-0-2 +2023-10-17T13:39:16.654140Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: new +2023-10-17T13:39:16.654156Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: enter +2023-10-17T13:39:16.654171Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-3 +2023-10-17T13:39:16.654190Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: update_matching self={[Some(5-0-2), 3), inflight:Logs(id=0):(None, Some(5-0-3)]} request_id=0 matching=Some(5-0-3) +2023-10-17T13:39:16.654212Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: granted after updating progress granted=5-0-3 +2023-10-17T13:39:16.654231Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: new +2023-10-17T13:39:16.654248Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: enter +2023-10-17T13:39:16.654267Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: openraft::raft_state@update_committed#2800c000000004: new +2023-10-17T13:39:16.654284Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: openraft::raft_state@update_committed#2800c000000004: enter +2023-10-17T13:39:16.654301Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: openraft::raft_state@update_committed#2800c000000004: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.654326Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: openraft::raft_state@update_committed#2800c000000004: exit +2023-10-17T13:39:16.654344Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: openraft::raft_state@update_committed#2800c000000004: close time.busy=41.2µs time.idle=36.1µs +2023-10-17T13:39:16.654366Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: exit +2023-10-17T13:39:16.654383Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#6800c000000003: close time.busy=118µs time.idle=34.3µs +2023-10-17T13:39:16.654404Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: exit +2023-10-17T13:39:16.654423Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::engine::handler::replication_handler@update_matching#8000c000000002: close time.busy=248µs time.idle=34.0µs +2023-10-17T13:39:16.654441Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }) } +2023-10-17T13:39:16.654456Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: condition: None +2023-10-17T13:39:16.654470Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: run command: Apply { seq: 3, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 2 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 } } +2023-10-17T13:39:16.654484Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: condition: None +2023-10-17T13:39:16.654502Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: new +2023-10-17T13:39:16.654518Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: enter +2023-10-17T13:39:16.654532Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: apply_to_state_machine upto_index=3 +2023-10-17T13:39:16.654566Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: about to apply entries=[5-0-3:normal] +2023-10-17T13:39:16.654585Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: sending command to state machine worker: StateMachineCommand { seq: 3, payload: Apply: [5-0-3:normal] } +2023-10-17T13:39:16.654605Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: exit +2023-10-17T13:39:16.654620Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: openraft::core::raft_core@apply_to_state_machine#8800c000000002: close time.busy=86.8µs time.idle=31.9µs +2023-10-17T13:39:16.654640Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: exit +2023-10-17T13:39:16.654655Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#7800c000000001: close time.busy=749µs time.idle=29.8µs +2023-10-17T13:39:16.654672Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.654686Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.654706Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: new +2023-10-17T13:39:16.654722Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: enter +2023-10-17T13:39:16.654747Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: new +2023-10-17T13:39:16.654794Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: enter +2023-10-17T13:39:16.654832Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: openraft::core::raft_core@current_leader#7000c000000003: new +2023-10-17T13:39:16.654852Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: openraft::core::raft_core@current_leader#7000c000000003: enter +2023-10-17T13:39:16.654875Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: openraft::core::raft_core@current_leader#7000c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.654905Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: openraft::core::raft_core@current_leader#7000c000000003: exit +2023-10-17T13:39:16.654926Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: openraft::core::raft_core@current_leader#7000c000000003: close time.busy=52.8µs time.idle=41.4µs +2023-10-17T13:39:16.654991Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:3, last_applied:5-0-2, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-3}} +2023-10-17T13:39:16.655020Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: exit +2023-10-17T13:39:16.655038Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: openraft::core::raft_core@report_metrics#9000c000000002: close time.busy=227µs time.idle=64.5µs +2023-10-17T13:39:16.655063Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: exit +2023-10-17T13:39:16.655080Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#8000c000000001: close time.busy=340µs time.idle=33.8µs +2023-10-17T13:39:16.655110Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.655127Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.655140Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.655160Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.655177Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 3, payload: Apply: [5-0-3:normal] } +2023-10-17T13:39:16.655195Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#8800c000000001: new +2023-10-17T13:39:16.655206Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#8800c000000001: enter +2023-10-17T13:39:16.655226Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#8800c000000001: replicate to sm entry.log_id=5-0-3 +2023-10-17T13:39:16.655252Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#8800c000000001: exit +2023-10-17T13:39:16.655264Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#8800c000000001: close time.busy=45.8µs time.idle=22.9µs +2023-10-17T13:39:16.655284Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.655298Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.655308Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.655323Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.655350Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.655367Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.655387Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 3, result: Ok(Apply(ApplyResult { since: 3, end: 4, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 } })) } +2023-10-17T13:39:16.655405Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 3, result: Ok(Apply(ApplyResult { since: 3, end: 4, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 } })) } +2023-10-17T13:39:16.655428Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-3 +2023-10-17T13:39:16.655450Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: new +2023-10-17T13:39:16.655469Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: enter +2023-10-17T13:39:16.655484Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-3 +2023-10-17T13:39:16.655511Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core@send_response#7800c000000003: new +2023-10-17T13:39:16.655530Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core@send_response#7800c000000003: enter +2023-10-17T13:39:16.655549Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core@send_response#7800c000000003: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }, membership: None } +2023-10-17T13:39:16.655594Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.655605Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: enter +2023-10-17T13:39:16.655577Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core@send_response#7800c000000003: send client response through tx, send_res is error: false +2023-10-17T13:39:16.655613Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: enter +2023-10-17T13:39:16.655622Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: call_core receives result is error: false +2023-10-17T13:39:16.655615Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core@send_response#7800c000000003: exit +2023-10-17T13:39:16.655632Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: exit +2023-10-17T13:39:16.655632Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: openraft::core::raft_core@send_response#7800c000000003: close time.busy=84.9µs time.idle=36.4µs +2023-10-17T13:39:16.655640Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: openraft::raft@call_core#8000000000007: close time.busy=31.6µs time.idle=2.36ms +2023-10-17T13:39:16.655658Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: exit +2023-10-17T13:39:16.655658Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: exit +2023-10-17T13:39:16.655667Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#8000000000006: close time.busy=98.6µs time.idle=2.34ms +2023-10-17T13:39:16.655672Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#9800c000000002: close time.busy=189µs time.idle=33.5µs +2023-10-17T13:39:16.655684Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: new +2023-10-17T13:39:16.655692Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: enter +2023-10-17T13:39:16.655693Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.655702Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: new +2023-10-17T13:39:16.655710Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: enter +2023-10-17T13:39:16.655707Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#9000c000000001{state=Leader id=0}: close time.busy=326µs time.idle=31.3µs +2023-10-17T13:39:16.655723Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: exit +2023-10-17T13:39:16.655732Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: exit +2023-10-17T13:39:16.655740Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.655727Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#9800c000000001: new +2023-10-17T13:39:16.655750Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#9800c000000001: enter +2023-10-17T13:39:16.655767Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#9800c000000001: queued commands: start... +2023-10-17T13:39:16.655780Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#9800c000000001: queued commands: end... +2023-10-17T13:39:16.655797Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#9800c000000001: exit +2023-10-17T13:39:16.655811Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#9800c000000001: close time.busy=46.6µs time.idle=37.7µs +2023-10-17T13:39:16.655930Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.655945Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.655959Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.655977Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: new +2023-10-17T13:39:16.655993Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: enter +2023-10-17T13:39:16.656008Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.656029Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8000c000000003: new +2023-10-17T13:39:16.656046Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8000c000000003: enter +2023-10-17T13:39:16.656061Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8000c000000003: this node is a leader +2023-10-17T13:39:16.656079Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8000c000000003: exit +2023-10-17T13:39:16.656096Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8000c000000003: close time.busy=33.4µs time.idle=33.7µs +2023-10-17T13:39:16.656119Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: new +2023-10-17T13:39:16.656136Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: enter +2023-10-17T13:39:16.656153Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: assign log id: 5-0-4 +2023-10-17T13:39:16.656174Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: openraft::engine::handler::replication_handler@initiate_replication#3000c000000004: new +2023-10-17T13:39:16.656192Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: openraft::engine::handler::replication_handler@initiate_replication#3000c000000004: enter +2023-10-17T13:39:16.656210Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: openraft::engine::handler::replication_handler@initiate_replication#3000c000000004: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }), curr_inflight_id: 0, inflight: None, searching_end: 4 })], stat: Stat { update_count: 3, move_count: 3, is_quorum_count: 3 } } +2023-10-17T13:39:16.656237Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: openraft::engine::handler::replication_handler@initiate_replication#3000c000000004: exit +2023-10-17T13:39:16.656255Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: openraft::engine::handler::replication_handler@initiate_replication#3000c000000004: close time.busy=45.1µs time.idle=36.4µs +2023-10-17T13:39:16.656277Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: exit +2023-10-17T13:39:16.656294Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#8800c000000003: close time.busy=141µs time.idle=34.0µs +2023-10-17T13:39:16.656316Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: exit +2023-10-17T13:39:16.656332Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: openraft::core::raft_core@write_entry#a000c000000002{id=0}: close time.busy=323µs time.idle=32.0µs +2023-10-17T13:39:16.656351Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.656366Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#a000c000000001{state=Leader id=0}: close time.busy=406µs time.idle=33.5µs +2023-10-17T13:39:16.656416Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: new +2023-10-17T13:39:16.656432Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: enter +2023-10-17T13:39:16.656445Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: queued commands: start... +2023-10-17T13:39:16.656457Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }, payload: normal }] } +2023-10-17T13:39:16.656473Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: queued commands: end... +2023-10-17T13:39:16.656486Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }, payload: normal }] } +2023-10-17T13:39:16.656501Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: condition: None +2023-10-17T13:39:16.656515Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: AppendInputEntries: [5-0-4:normal] +2023-10-17T13:39:16.656533Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@append_to_log#a800c000000002: new +2023-10-17T13:39:16.656549Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@append_to_log#a800c000000002: enter +2023-10-17T13:39:16.656563Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@append_to_log#a800c000000002: append_to_log +2023-10-17T13:39:16.656593Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@append_to_log#a800c000000002: exit +2023-10-17T13:39:16.656610Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@append_to_log#a800c000000002: close time.busy=44.2µs time.idle=32.9µs +2023-10-17T13:39:16.656630Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-4 +2023-10-17T13:39:16.656645Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: update progress self_matching=5-0-3 +2023-10-17T13:39:16.656664Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: new +2023-10-17T13:39:16.656680Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: enter +2023-10-17T13:39:16.656695Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-4 +2023-10-17T13:39:16.656715Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: update_matching self={[Some(5-0-3), 4), inflight:Logs(id=0):(None, Some(5-0-4)]} request_id=0 matching=Some(5-0-4) +2023-10-17T13:39:16.656736Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: granted after updating progress granted=5-0-4 +2023-10-17T13:39:16.656755Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: new +2023-10-17T13:39:16.656772Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: enter +2023-10-17T13:39:16.656791Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: openraft::raft_state@update_committed#3800c000000004: new +2023-10-17T13:39:16.656809Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: openraft::raft_state@update_committed#3800c000000004: enter +2023-10-17T13:39:16.656826Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: openraft::raft_state@update_committed#3800c000000004: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.656850Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: openraft::raft_state@update_committed#3800c000000004: exit +2023-10-17T13:39:16.656868Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: openraft::raft_state@update_committed#3800c000000004: close time.busy=41.0µs time.idle=36.3µs +2023-10-17T13:39:16.656892Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: exit +2023-10-17T13:39:16.656909Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: openraft::engine::handler::replication_handler@try_commit_granted#9000c000000003: close time.busy=120µs time.idle=34.1µs +2023-10-17T13:39:16.656929Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: exit +2023-10-17T13:39:16.656945Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::engine::handler::replication_handler@update_matching#b000c000000002: close time.busy=249µs time.idle=32.3µs +2023-10-17T13:39:16.656963Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }) } +2023-10-17T13:39:16.656979Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: condition: None +2023-10-17T13:39:16.656993Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: run command: Apply { seq: 4, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 3 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 } } +2023-10-17T13:39:16.657008Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: condition: None +2023-10-17T13:39:16.657026Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: new +2023-10-17T13:39:16.657042Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: enter +2023-10-17T13:39:16.657056Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: apply_to_state_machine upto_index=4 +2023-10-17T13:39:16.657086Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: about to apply entries=[5-0-4:normal] +2023-10-17T13:39:16.657103Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: sending command to state machine worker: StateMachineCommand { seq: 4, payload: Apply: [5-0-4:normal] } +2023-10-17T13:39:16.657122Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: exit +2023-10-17T13:39:16.657138Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: openraft::core::raft_core@apply_to_state_machine#b800c000000002: close time.busy=80.8µs time.idle=31.4µs +2023-10-17T13:39:16.657157Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: exit +2023-10-17T13:39:16.657172Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#a800c000000001: close time.busy=726µs time.idle=30.1µs +2023-10-17T13:39:16.657212Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.657226Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.657242Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: new +2023-10-17T13:39:16.657256Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: enter +2023-10-17T13:39:16.657274Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: new +2023-10-17T13:39:16.657290Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: enter +2023-10-17T13:39:16.657309Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: openraft::core::raft_core@current_leader#9800c000000003: new +2023-10-17T13:39:16.657328Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: openraft::core::raft_core@current_leader#9800c000000003: enter +2023-10-17T13:39:16.657344Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: openraft::core::raft_core@current_leader#9800c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.657365Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: openraft::core::raft_core@current_leader#9800c000000003: exit +2023-10-17T13:39:16.657381Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: openraft::core::raft_core@current_leader#9800c000000003: close time.busy=37.2µs time.idle=35.2µs +2023-10-17T13:39:16.657419Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:4, last_applied:5-0-3, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-4}} +2023-10-17T13:39:16.657440Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: exit +2023-10-17T13:39:16.657458Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: openraft::core::raft_core@report_metrics#c000c000000002: close time.busy=150µs time.idle=33.6µs +2023-10-17T13:39:16.657477Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: exit +2023-10-17T13:39:16.657491Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#b000c000000001: close time.busy=221µs time.idle=28.7µs +2023-10-17T13:39:16.657511Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.657526Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.657538Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.657553Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.657562Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 4, payload: Apply: [5-0-4:normal] } +2023-10-17T13:39:16.657564Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.657574Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#b800c000000001: new +2023-10-17T13:39:16.657574Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.657581Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#b800c000000001: enter +2023-10-17T13:39:16.657592Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#b800c000000001: replicate to sm entry.log_id=5-0-4 +2023-10-17T13:39:16.657590Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.657609Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#b800c000000001: exit +2023-10-17T13:39:16.657609Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.657617Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#b800c000000001: close time.busy=27.3µs time.idle=16.4µs +2023-10-17T13:39:16.657621Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.657631Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.657632Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.657643Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.657652Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.657664Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.657692Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: new +2023-10-17T13:39:16.657708Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.657724Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 4, result: Ok(Apply(ApplyResult { since: 4, end: 5, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 } })) } +2023-10-17T13:39:16.657740Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 4, result: Ok(Apply(ApplyResult { since: 4, end: 5, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 } })) } +2023-10-17T13:39:16.657758Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-4 +2023-10-17T13:39:16.657777Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: new +2023-10-17T13:39:16.657796Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: enter +2023-10-17T13:39:16.657811Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-4 +2023-10-17T13:39:16.657834Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core@send_response#4000000004: new +2023-10-17T13:39:16.657851Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core@send_response#4000000004: enter +2023-10-17T13:39:16.657866Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core@send_response#4000000004: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }, membership: None } +2023-10-17T13:39:16.657890Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.657887Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core@send_response#4000000004: send client response through tx, send_res is error: false +2023-10-17T13:39:16.657899Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: enter +2023-10-17T13:39:16.657906Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: enter +2023-10-17T13:39:16.657903Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core@send_response#4000000004: exit +2023-10-17T13:39:16.657917Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: call_core receives result is error: false +2023-10-17T13:39:16.657927Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: exit +2023-10-17T13:39:16.657919Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: openraft::core::raft_core@send_response#4000000004: close time.busy=52.5µs time.idle=33.1µs +2023-10-17T13:39:16.657936Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: openraft::raft@call_core#10000000000007: close time.busy=34.2µs time.idle=2.20ms +2023-10-17T13:39:16.657950Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: exit +2023-10-17T13:39:16.657942Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: exit +2023-10-17T13:39:16.657960Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#10000000000006: close time.busy=90.5µs time.idle=2.19ms +2023-10-17T13:39:16.657962Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#4000000003: close time.busy=147µs time.idle=37.6µs +2023-10-17T13:39:16.657977Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: new +2023-10-17T13:39:16.657984Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: enter +2023-10-17T13:39:16.657981Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.657994Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: new +2023-10-17T13:39:16.657995Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#4000000002{state=Leader id=0}: close time.busy=274µs time.idle=29.6µs +2023-10-17T13:39:16.658004Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: enter +2023-10-17T13:39:16.658016Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: exit +2023-10-17T13:39:16.658014Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#8004000000002: new +2023-10-17T13:39:16.658025Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: exit +2023-10-17T13:39:16.658032Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.658027Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#8004000000002: enter +2023-10-17T13:39:16.658040Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#8004000000002: queued commands: start... +2023-10-17T13:39:16.658054Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#8004000000002: queued commands: end... +2023-10-17T13:39:16.658070Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#8004000000002: exit +2023-10-17T13:39:16.658085Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#8004000000002: close time.busy=43.0µs time.idle=27.9µs +2023-10-17T13:39:16.658111Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: new +2023-10-17T13:39:16.658126Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.658140Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.658158Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: new +2023-10-17T13:39:16.658174Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: enter +2023-10-17T13:39:16.658189Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.658210Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8004000000004: new +2023-10-17T13:39:16.658227Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8004000000004: enter +2023-10-17T13:39:16.658244Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8004000000004: this node is a leader +2023-10-17T13:39:16.658262Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8004000000004: exit +2023-10-17T13:39:16.658279Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#8004000000004: close time.busy=35.3µs time.idle=34.3µs +2023-10-17T13:39:16.658303Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: new +2023-10-17T13:39:16.658321Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: enter +2023-10-17T13:39:16.658337Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: assign log id: 5-0-5 +2023-10-17T13:39:16.658358Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: openraft::engine::handler::replication_handler@initiate_replication#4000000005: new +2023-10-17T13:39:16.658380Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: openraft::engine::handler::replication_handler@initiate_replication#4000000005: enter +2023-10-17T13:39:16.658398Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: openraft::engine::handler::replication_handler@initiate_replication#4000000005: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }), curr_inflight_id: 0, inflight: None, searching_end: 5 })], stat: Stat { update_count: 4, move_count: 4, is_quorum_count: 4 } } +2023-10-17T13:39:16.658424Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: openraft::engine::handler::replication_handler@initiate_replication#4000000005: exit +2023-10-17T13:39:16.658442Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: openraft::engine::handler::replication_handler@initiate_replication#4000000005: close time.busy=44.0µs time.idle=40.2µs +2023-10-17T13:39:16.658466Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: exit +2023-10-17T13:39:16.658483Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#10004000000004: close time.busy=145µs time.idle=34.6µs +2023-10-17T13:39:16.658505Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: exit +2023-10-17T13:39:16.658521Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#8004000000003{id=0}: close time.busy=330µs time.idle=32.2µs +2023-10-17T13:39:16.658540Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.658555Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#10004000000002{state=Leader id=0}: close time.busy=414µs time.idle=30.0µs +2023-10-17T13:39:16.658575Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: new +2023-10-17T13:39:16.658590Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: enter +2023-10-17T13:39:16.658604Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: queued commands: start... +2023-10-17T13:39:16.658618Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }, payload: normal }] } +2023-10-17T13:39:16.658634Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: queued commands: end... +2023-10-17T13:39:16.658647Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }, payload: normal }] } +2023-10-17T13:39:16.658661Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: condition: None +2023-10-17T13:39:16.658674Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: AppendInputEntries: [5-0-5:normal] +2023-10-17T13:39:16.658692Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@append_to_log#10004000000003: new +2023-10-17T13:39:16.658708Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@append_to_log#10004000000003: enter +2023-10-17T13:39:16.658722Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@append_to_log#10004000000003: append_to_log +2023-10-17T13:39:16.658754Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@append_to_log#10004000000003: exit +2023-10-17T13:39:16.658770Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@append_to_log#10004000000003: close time.busy=46.5µs time.idle=32.2µs +2023-10-17T13:39:16.658793Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-5 +2023-10-17T13:39:16.658809Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: update progress self_matching=5-0-4 +2023-10-17T13:39:16.658828Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: new +2023-10-17T13:39:16.658842Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: enter +2023-10-17T13:39:16.658859Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-5 +2023-10-17T13:39:16.658878Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: update_matching self={[Some(5-0-4), 5), inflight:Logs(id=0):(None, Some(5-0-5)]} request_id=0 matching=Some(5-0-5) +2023-10-17T13:39:16.658900Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: granted after updating progress granted=5-0-5 +2023-10-17T13:39:16.658919Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: new +2023-10-17T13:39:16.658936Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: enter +2023-10-17T13:39:16.658954Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: openraft::raft_state@update_committed#8004000000005: new +2023-10-17T13:39:16.658972Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: openraft::raft_state@update_committed#8004000000005: enter +2023-10-17T13:39:16.658989Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: openraft::raft_state@update_committed#8004000000005: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.659013Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: openraft::raft_state@update_committed#8004000000005: exit +2023-10-17T13:39:16.659031Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: openraft::raft_state@update_committed#8004000000005: close time.busy=41.1µs time.idle=35.6µs +2023-10-17T13:39:16.659054Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: exit +2023-10-17T13:39:16.659071Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: openraft::engine::handler::replication_handler@try_commit_granted#18004000000004: close time.busy=118µs time.idle=33.8µs +2023-10-17T13:39:16.659091Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: exit +2023-10-17T13:39:16.659106Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::engine::handler::replication_handler@update_matching#18004000000003: close time.busy=249µs time.idle=29.8µs +2023-10-17T13:39:16.659125Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }) } +2023-10-17T13:39:16.659140Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: condition: None +2023-10-17T13:39:16.659155Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: run command: Apply { seq: 5, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 4 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 } } +2023-10-17T13:39:16.659169Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: condition: None +2023-10-17T13:39:16.659186Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: new +2023-10-17T13:39:16.659202Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: enter +2023-10-17T13:39:16.659216Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: apply_to_state_machine upto_index=5 +2023-10-17T13:39:16.659247Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: about to apply entries=[5-0-5:normal] +2023-10-17T13:39:16.659265Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: sending command to state machine worker: StateMachineCommand { seq: 5, payload: Apply: [5-0-5:normal] } +2023-10-17T13:39:16.659285Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: exit +2023-10-17T13:39:16.659300Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: openraft::core::raft_core@apply_to_state_machine#20004000000003: close time.busy=82.8µs time.idle=31.6µs +2023-10-17T13:39:16.659320Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: exit +2023-10-17T13:39:16.659335Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18004000000002: close time.busy=730µs time.idle=29.8µs +2023-10-17T13:39:16.659352Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.659366Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.659382Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: new +2023-10-17T13:39:16.659396Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: enter +2023-10-17T13:39:16.659414Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: new +2023-10-17T13:39:16.659430Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: enter +2023-10-17T13:39:16.659450Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: openraft::core::raft_core@current_leader#20004000000004: new +2023-10-17T13:39:16.659467Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: openraft::core::raft_core@current_leader#20004000000004: enter +2023-10-17T13:39:16.659483Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: openraft::core::raft_core@current_leader#20004000000004: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.659504Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: openraft::core::raft_core@current_leader#20004000000004: exit +2023-10-17T13:39:16.659520Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: openraft::core::raft_core@current_leader#20004000000004: close time.busy=36.8µs time.idle=33.8µs +2023-10-17T13:39:16.659557Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:5, last_applied:5-0-4, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-5}} +2023-10-17T13:39:16.659578Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: exit +2023-10-17T13:39:16.659593Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: openraft::core::raft_core@report_metrics#28004000000003: close time.busy=147µs time.idle=31.8µs +2023-10-17T13:39:16.659612Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: exit +2023-10-17T13:39:16.659626Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#20004000000002: close time.busy=216µs time.idle=29.0µs +2023-10-17T13:39:16.659647Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.659661Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.659673Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.659686Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.659693Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.659696Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 5, payload: Apply: [5-0-5:normal] } +2023-10-17T13:39:16.659702Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.659707Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#28004000000002: new +2023-10-17T13:39:16.659717Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.659721Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#28004000000002: enter +2023-10-17T13:39:16.659736Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#28004000000002: replicate to sm entry.log_id=5-0-5 +2023-10-17T13:39:16.659734Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.659746Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.659751Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#28004000000002: exit +2023-10-17T13:39:16.659758Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.659758Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#28004000000002: close time.busy=30.6µs time.idle=21.5µs +2023-10-17T13:39:16.659773Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.659783Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.659793Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.659805Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.659827Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: new +2023-10-17T13:39:16.659842Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.659858Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 5, result: Ok(Apply(ApplyResult { since: 5, end: 6, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 } })) } +2023-10-17T13:39:16.659873Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 5, result: Ok(Apply(ApplyResult { since: 5, end: 6, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 } })) } +2023-10-17T13:39:16.659890Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-5 +2023-10-17T13:39:16.659909Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: new +2023-10-17T13:39:16.659925Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: enter +2023-10-17T13:39:16.659941Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-5 +2023-10-17T13:39:16.659962Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core@send_response#28004000000004: new +2023-10-17T13:39:16.659979Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core@send_response#28004000000004: enter +2023-10-17T13:39:16.659994Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core@send_response#28004000000004: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }, membership: None } +2023-10-17T13:39:16.660018Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.660024Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: enter +2023-10-17T13:39:16.660015Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core@send_response#28004000000004: send client response through tx, send_res is error: false +2023-10-17T13:39:16.660031Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: enter +2023-10-17T13:39:16.660039Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: call_core receives result is error: false +2023-10-17T13:39:16.660036Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core@send_response#28004000000004: exit +2023-10-17T13:39:16.660049Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: exit +2023-10-17T13:39:16.660057Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: openraft::raft@call_core#18000000000007: close time.busy=30.1µs time.idle=2.03ms +2023-10-17T13:39:16.660052Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: openraft::core::raft_core@send_response#28004000000004: close time.busy=57.3µs time.idle=33.0µs +2023-10-17T13:39:16.660071Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: exit +2023-10-17T13:39:16.660072Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: exit +2023-10-17T13:39:16.660078Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#18000000000006: close time.busy=88.0µs time.idle=2.01ms +2023-10-17T13:39:16.660098Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: new +2023-10-17T13:39:16.660091Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#30004000000003: close time.busy=147µs time.idle=35.0µs +2023-10-17T13:39:16.660107Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: enter +2023-10-17T13:39:16.660110Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.660116Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: new +2023-10-17T13:39:16.660128Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: enter +2023-10-17T13:39:16.660126Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#30004000000002{state=Leader id=0}: close time.busy=268µs time.idle=31.4µs +2023-10-17T13:39:16.660139Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: exit +2023-10-17T13:39:16.660148Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: exit +2023-10-17T13:39:16.660145Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#38004000000002: new +2023-10-17T13:39:16.660156Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.660158Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#38004000000002: enter +2023-10-17T13:39:16.660172Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#38004000000002: queued commands: start... +2023-10-17T13:39:16.660185Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#38004000000002: queued commands: end... +2023-10-17T13:39:16.660201Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#38004000000002: exit +2023-10-17T13:39:16.660215Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#38004000000002: close time.busy=43.0µs time.idle=27.9µs +2023-10-17T13:39:16.660240Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: new +2023-10-17T13:39:16.660255Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.660268Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.660287Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: new +2023-10-17T13:39:16.660303Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: enter +2023-10-17T13:39:16.660318Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.660337Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#30004000000004: new +2023-10-17T13:39:16.660354Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#30004000000004: enter +2023-10-17T13:39:16.660371Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#30004000000004: this node is a leader +2023-10-17T13:39:16.660388Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#30004000000004: exit +2023-10-17T13:39:16.660404Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#30004000000004: close time.busy=33.3µs time.idle=33.8µs +2023-10-17T13:39:16.660428Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: new +2023-10-17T13:39:16.660444Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: enter +2023-10-17T13:39:16.660460Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: assign log id: 5-0-6 +2023-10-17T13:39:16.660483Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: openraft::engine::handler::replication_handler@initiate_replication#10004000000005: new +2023-10-17T13:39:16.660501Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: openraft::engine::handler::replication_handler@initiate_replication#10004000000005: enter +2023-10-17T13:39:16.660518Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: openraft::engine::handler::replication_handler@initiate_replication#10004000000005: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }), curr_inflight_id: 0, inflight: None, searching_end: 6 })], stat: Stat { update_count: 5, move_count: 5, is_quorum_count: 5 } } +2023-10-17T13:39:16.660544Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: openraft::engine::handler::replication_handler@initiate_replication#10004000000005: exit +2023-10-17T13:39:16.660562Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: openraft::engine::handler::replication_handler@initiate_replication#10004000000005: close time.busy=42.9µs time.idle=36.9µs +2023-10-17T13:39:16.660585Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: exit +2023-10-17T13:39:16.660602Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#38004000000004: close time.busy=140µs time.idle=34.4µs +2023-10-17T13:39:16.660623Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: exit +2023-10-17T13:39:16.660639Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#38004000000003{id=0}: close time.busy=320µs time.idle=32.3µs +2023-10-17T13:39:16.660659Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.660674Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#40004000000002{state=Leader id=0}: close time.busy=404µs time.idle=30.0µs +2023-10-17T13:39:16.660694Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: new +2023-10-17T13:39:16.660708Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: enter +2023-10-17T13:39:16.660722Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: queued commands: start... +2023-10-17T13:39:16.660736Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }, payload: normal }] } +2023-10-17T13:39:16.660751Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: queued commands: end... +2023-10-17T13:39:16.660765Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }, payload: normal }] } +2023-10-17T13:39:16.660781Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: condition: None +2023-10-17T13:39:16.660796Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: AppendInputEntries: [5-0-6:normal] +2023-10-17T13:39:16.660814Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@append_to_log#40004000000003: new +2023-10-17T13:39:16.660829Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@append_to_log#40004000000003: enter +2023-10-17T13:39:16.660843Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@append_to_log#40004000000003: append_to_log +2023-10-17T13:39:16.660875Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@append_to_log#40004000000003: exit +2023-10-17T13:39:16.660893Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@append_to_log#40004000000003: close time.busy=46.2µs time.idle=33.9µs +2023-10-17T13:39:16.660914Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-6 +2023-10-17T13:39:16.660930Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: update progress self_matching=5-0-5 +2023-10-17T13:39:16.660949Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: new +2023-10-17T13:39:16.660963Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: enter +2023-10-17T13:39:16.660978Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-6 +2023-10-17T13:39:16.660997Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: update_matching self={[Some(5-0-5), 6), inflight:Logs(id=0):(None, Some(5-0-6)]} request_id=0 matching=Some(5-0-6) +2023-10-17T13:39:16.661019Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: granted after updating progress granted=5-0-6 +2023-10-17T13:39:16.661038Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: new +2023-10-17T13:39:16.661057Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: enter +2023-10-17T13:39:16.661076Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: openraft::raft_state@update_committed#18004000000005: new +2023-10-17T13:39:16.661095Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: openraft::raft_state@update_committed#18004000000005: enter +2023-10-17T13:39:16.661112Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: openraft::raft_state@update_committed#18004000000005: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.661136Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: openraft::raft_state@update_committed#18004000000005: exit +2023-10-17T13:39:16.661154Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: openraft::raft_state@update_committed#18004000000005: close time.busy=41.5µs time.idle=35.8µs +2023-10-17T13:39:16.661176Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: exit +2023-10-17T13:39:16.661192Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: openraft::engine::handler::replication_handler@try_commit_granted#40004000000004: close time.busy=119µs time.idle=34.9µs +2023-10-17T13:39:16.661212Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: exit +2023-10-17T13:39:16.661227Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::engine::handler::replication_handler@update_matching#48004000000003: close time.busy=249µs time.idle=30.0µs +2023-10-17T13:39:16.661246Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }) } +2023-10-17T13:39:16.661261Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: condition: None +2023-10-17T13:39:16.661275Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: run command: Apply { seq: 6, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 5 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 } } +2023-10-17T13:39:16.661291Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: condition: None +2023-10-17T13:39:16.661308Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: new +2023-10-17T13:39:16.661324Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: enter +2023-10-17T13:39:16.661337Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: apply_to_state_machine upto_index=6 +2023-10-17T13:39:16.661367Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: about to apply entries=[5-0-6:normal] +2023-10-17T13:39:16.661384Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: sending command to state machine worker: StateMachineCommand { seq: 6, payload: Apply: [5-0-6:normal] } +2023-10-17T13:39:16.661404Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: exit +2023-10-17T13:39:16.661419Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: openraft::core::raft_core@apply_to_state_machine#50004000000003: close time.busy=80.0µs time.idle=31.6µs +2023-10-17T13:39:16.661439Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: exit +2023-10-17T13:39:16.661455Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48004000000002: close time.busy=731µs time.idle=31.0µs +2023-10-17T13:39:16.661472Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.661485Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.661502Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: new +2023-10-17T13:39:16.661516Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: enter +2023-10-17T13:39:16.661534Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: new +2023-10-17T13:39:16.661550Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: enter +2023-10-17T13:39:16.661570Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: openraft::core::raft_core@current_leader#48004000000004: new +2023-10-17T13:39:16.661587Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: openraft::core::raft_core@current_leader#48004000000004: enter +2023-10-17T13:39:16.661603Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: openraft::core::raft_core@current_leader#48004000000004: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.661623Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: openraft::core::raft_core@current_leader#48004000000004: exit +2023-10-17T13:39:16.661639Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: openraft::core::raft_core@current_leader#48004000000004: close time.busy=35.9µs time.idle=33.8µs +2023-10-17T13:39:16.661677Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:6, last_applied:5-0-5, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-6}} +2023-10-17T13:39:16.661698Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: exit +2023-10-17T13:39:16.661714Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: openraft::core::raft_core@report_metrics#58004000000003: close time.busy=148µs time.idle=31.1µs +2023-10-17T13:39:16.661733Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: exit +2023-10-17T13:39:16.661748Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#50004000000002: close time.busy=217µs time.idle=29.6µs +2023-10-17T13:39:16.661768Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.661782Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.661793Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.661808Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.661814Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.661815Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 6, payload: Apply: [5-0-6:normal] } +2023-10-17T13:39:16.661826Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#58004000000002: new +2023-10-17T13:39:16.661824Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.661834Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#58004000000002: enter +2023-10-17T13:39:16.661834Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.661843Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#58004000000002: replicate to sm entry.log_id=5-0-6 +2023-10-17T13:39:16.661849Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.661856Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#58004000000002: exit +2023-10-17T13:39:16.661863Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#58004000000002: close time.busy=22.5µs time.idle=15.0µs +2023-10-17T13:39:16.661863Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.661874Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.661876Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.661886Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.661896Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.661908Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.661928Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: new +2023-10-17T13:39:16.661942Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.661957Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 6, result: Ok(Apply(ApplyResult { since: 6, end: 7, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 } })) } +2023-10-17T13:39:16.661972Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 6, result: Ok(Apply(ApplyResult { since: 6, end: 7, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 } })) } +2023-10-17T13:39:16.661989Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-6 +2023-10-17T13:39:16.662007Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: new +2023-10-17T13:39:16.662023Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: enter +2023-10-17T13:39:16.662037Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-6 +2023-10-17T13:39:16.662057Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core@send_response#50004000000004: new +2023-10-17T13:39:16.662074Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core@send_response#50004000000004: enter +2023-10-17T13:39:16.662090Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core@send_response#50004000000004: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }, membership: None } +2023-10-17T13:39:16.662115Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.662111Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core@send_response#50004000000004: send client response through tx, send_res is error: false +2023-10-17T13:39:16.662123Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: enter +2023-10-17T13:39:16.662130Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: enter +2023-10-17T13:39:16.662127Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core@send_response#50004000000004: exit +2023-10-17T13:39:16.662138Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: call_core receives result is error: false +2023-10-17T13:39:16.662148Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: exit +2023-10-17T13:39:16.662143Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: openraft::core::raft_core@send_response#50004000000004: close time.busy=53.2µs time.idle=32.2µs +2023-10-17T13:39:16.662156Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: openraft::raft@call_core#20000000000007: close time.busy=29.1µs time.idle=2.01ms +2023-10-17T13:39:16.662169Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: exit +2023-10-17T13:39:16.662163Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: exit +2023-10-17T13:39:16.662177Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#20000000000006: close time.busy=87.8µs time.idle=1.99ms +2023-10-17T13:39:16.662177Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#60004000000003: close time.busy=140µs time.idle=30.6µs +2023-10-17T13:39:16.662192Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: new +2023-10-17T13:39:16.662199Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: enter +2023-10-17T13:39:16.662196Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.662208Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: new +2023-10-17T13:39:16.662216Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: enter +2023-10-17T13:39:16.662210Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#60004000000002{state=Leader id=0}: close time.busy=254µs time.idle=28.4µs +2023-10-17T13:39:16.662228Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: exit +2023-10-17T13:39:16.662237Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: exit +2023-10-17T13:39:16.662244Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.662239Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#68004000000002: new +2023-10-17T13:39:16.662253Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#68004000000002: enter +2023-10-17T13:39:16.662268Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#68004000000002: queued commands: start... +2023-10-17T13:39:16.662281Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#68004000000002: queued commands: end... +2023-10-17T13:39:16.662296Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#68004000000002: exit +2023-10-17T13:39:16.662311Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#68004000000002: close time.busy=43.0µs time.idle=29.0µs +2023-10-17T13:39:16.662336Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: new +2023-10-17T13:39:16.662351Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.662364Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.662383Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: new +2023-10-17T13:39:16.662400Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: enter +2023-10-17T13:39:16.662414Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.662434Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#58004000000004: new +2023-10-17T13:39:16.662451Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#58004000000004: enter +2023-10-17T13:39:16.662466Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#58004000000004: this node is a leader +2023-10-17T13:39:16.662484Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#58004000000004: exit +2023-10-17T13:39:16.662502Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#58004000000004: close time.busy=32.8µs time.idle=35.3µs +2023-10-17T13:39:16.662525Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: new +2023-10-17T13:39:16.662542Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: enter +2023-10-17T13:39:16.662558Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: assign log id: 5-0-7 +2023-10-17T13:39:16.662578Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: openraft::engine::handler::replication_handler@initiate_replication#20004000000005: new +2023-10-17T13:39:16.662595Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: openraft::engine::handler::replication_handler@initiate_replication#20004000000005: enter +2023-10-17T13:39:16.662613Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: openraft::engine::handler::replication_handler@initiate_replication#20004000000005: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }), curr_inflight_id: 0, inflight: None, searching_end: 7 })], stat: Stat { update_count: 6, move_count: 6, is_quorum_count: 6 } } +2023-10-17T13:39:16.662638Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: openraft::engine::handler::replication_handler@initiate_replication#20004000000005: exit +2023-10-17T13:39:16.662657Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: openraft::engine::handler::replication_handler@initiate_replication#20004000000005: close time.busy=42.5µs time.idle=36.7µs +2023-10-17T13:39:16.662680Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: exit +2023-10-17T13:39:16.662696Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#60004000000004: close time.busy=137µs time.idle=34.2µs +2023-10-17T13:39:16.662718Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: exit +2023-10-17T13:39:16.662733Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: openraft::core::raft_core@write_entry#68004000000003{id=0}: close time.busy=318µs time.idle=32.1µs +2023-10-17T13:39:16.662754Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.662768Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#70004000000002{state=Leader id=0}: close time.busy=402µs time.idle=29.7µs +2023-10-17T13:39:16.662788Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: new +2023-10-17T13:39:16.662803Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: enter +2023-10-17T13:39:16.662817Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: queued commands: start... +2023-10-17T13:39:16.662830Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }, payload: normal }] } +2023-10-17T13:39:16.662844Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: queued commands: end... +2023-10-17T13:39:16.662860Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }, payload: normal }] } +2023-10-17T13:39:16.662874Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: condition: None +2023-10-17T13:39:16.662888Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: AppendInputEntries: [5-0-7:normal] +2023-10-17T13:39:16.662906Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@append_to_log#70004000000003: new +2023-10-17T13:39:16.662921Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@append_to_log#70004000000003: enter +2023-10-17T13:39:16.662936Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@append_to_log#70004000000003: append_to_log +2023-10-17T13:39:16.662966Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@append_to_log#70004000000003: exit +2023-10-17T13:39:16.662982Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@append_to_log#70004000000003: close time.busy=44.1µs time.idle=32.2µs +2023-10-17T13:39:16.663001Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-7 +2023-10-17T13:39:16.663017Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: update progress self_matching=5-0-6 +2023-10-17T13:39:16.663035Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: new +2023-10-17T13:39:16.663051Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: enter +2023-10-17T13:39:16.663066Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-7 +2023-10-17T13:39:16.663085Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: update_matching self={[Some(5-0-6), 7), inflight:Logs(id=0):(None, Some(5-0-7)]} request_id=0 matching=Some(5-0-7) +2023-10-17T13:39:16.663105Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: granted after updating progress granted=5-0-7 +2023-10-17T13:39:16.663124Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: new +2023-10-17T13:39:16.663141Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: enter +2023-10-17T13:39:16.663160Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: openraft::raft_state@update_committed#28004000000005: new +2023-10-17T13:39:16.663178Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: openraft::raft_state@update_committed#28004000000005: enter +2023-10-17T13:39:16.663195Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: openraft::raft_state@update_committed#28004000000005: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.663218Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: openraft::raft_state@update_committed#28004000000005: exit +2023-10-17T13:39:16.663235Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: openraft::raft_state@update_committed#28004000000005: close time.busy=40.4µs time.idle=35.6µs +2023-10-17T13:39:16.663257Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: exit +2023-10-17T13:39:16.663273Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: openraft::engine::handler::replication_handler@try_commit_granted#68004000000004: close time.busy=116µs time.idle=33.5µs +2023-10-17T13:39:16.663294Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: exit +2023-10-17T13:39:16.663310Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::engine::handler::replication_handler@update_matching#78004000000003: close time.busy=243µs time.idle=31.9µs +2023-10-17T13:39:16.663326Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }) } +2023-10-17T13:39:16.663342Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: condition: None +2023-10-17T13:39:16.663356Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: run command: Apply { seq: 7, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 6 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 } } +2023-10-17T13:39:16.663371Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: condition: None +2023-10-17T13:39:16.663388Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: new +2023-10-17T13:39:16.663404Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: enter +2023-10-17T13:39:16.663418Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: apply_to_state_machine upto_index=7 +2023-10-17T13:39:16.663446Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: about to apply entries=[5-0-7:normal] +2023-10-17T13:39:16.663463Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: sending command to state machine worker: StateMachineCommand { seq: 7, payload: Apply: [5-0-7:normal] } +2023-10-17T13:39:16.663482Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: exit +2023-10-17T13:39:16.663497Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: openraft::core::raft_core@apply_to_state_machine#80004000000003: close time.busy=77.9µs time.idle=31.6µs +2023-10-17T13:39:16.663517Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: exit +2023-10-17T13:39:16.663531Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#78004000000002: close time.busy=714µs time.idle=29.1µs +2023-10-17T13:39:16.663548Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.663561Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.663577Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: new +2023-10-17T13:39:16.663592Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: enter +2023-10-17T13:39:16.663609Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: new +2023-10-17T13:39:16.663625Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: enter +2023-10-17T13:39:16.663644Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: openraft::core::raft_core@current_leader#70004000000004: new +2023-10-17T13:39:16.663661Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: openraft::core::raft_core@current_leader#70004000000004: enter +2023-10-17T13:39:16.663677Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: openraft::core::raft_core@current_leader#70004000000004: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.663697Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: openraft::core::raft_core@current_leader#70004000000004: exit +2023-10-17T13:39:16.663714Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: openraft::core::raft_core@current_leader#70004000000004: close time.busy=36.2µs time.idle=33.5µs +2023-10-17T13:39:16.663749Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:7, last_applied:5-0-6, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-7}} +2023-10-17T13:39:16.663769Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: exit +2023-10-17T13:39:16.663785Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: openraft::core::raft_core@report_metrics#88004000000003: close time.busy=144µs time.idle=31.8µs +2023-10-17T13:39:16.663805Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: exit +2023-10-17T13:39:16.663820Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#80004000000002: close time.busy=213µs time.idle=29.8µs +2023-10-17T13:39:16.663840Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.663852Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.663864Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.663878Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.663886Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 7, payload: Apply: [5-0-7:normal] } +2023-10-17T13:39:16.663890Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.663898Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#88004000000002: new +2023-10-17T13:39:16.663905Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#88004000000002: enter +2023-10-17T13:39:16.663901Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.663925Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#88004000000002: replicate to sm entry.log_id=5-0-7 +2023-10-17T13:39:16.663950Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#88004000000002: exit +2023-10-17T13:39:16.663944Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.663960Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#88004000000002: close time.busy=44.8µs time.idle=17.8µs +2023-10-17T13:39:16.663975Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.663970Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.663985Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.664001Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.664023Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.664036Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.664049Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.664092Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: new +2023-10-17T13:39:16.664109Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: enter +2023-10-17T13:39:16.664133Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 7, result: Ok(Apply(ApplyResult { since: 7, end: 8, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 } })) } +2023-10-17T13:39:16.664156Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 7, result: Ok(Apply(ApplyResult { since: 7, end: 8, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 } })) } +2023-10-17T13:39:16.664179Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-7 +2023-10-17T13:39:16.664203Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: new +2023-10-17T13:39:16.664221Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: enter +2023-10-17T13:39:16.664242Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-7 +2023-10-17T13:39:16.664269Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core@send_response#8008000000005: new +2023-10-17T13:39:16.664297Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core@send_response#8008000000005: enter +2023-10-17T13:39:16.664316Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core@send_response#8008000000005: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }, membership: None } +2023-10-17T13:39:16.664347Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.664342Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core@send_response#8008000000005: send client response through tx, send_res is error: false +2023-10-17T13:39:16.664357Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: enter +2023-10-17T13:39:16.664365Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: enter +2023-10-17T13:39:16.664360Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core@send_response#8008000000005: exit +2023-10-17T13:39:16.664376Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: call_core receives result is error: false +2023-10-17T13:39:16.664386Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: exit +2023-10-17T13:39:16.664379Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: openraft::core::raft_core@send_response#8008000000005: close time.busy=63.2µs time.idle=46.9µs +2023-10-17T13:39:16.664395Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: openraft::raft@call_core#28000000000007: close time.busy=33.0µs time.idle=2.15ms +2023-10-17T13:39:16.664408Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: exit +2023-10-17T13:39:16.664416Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#28000000000006: close time.busy=87.8µs time.idle=2.14ms +2023-10-17T13:39:16.664414Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: exit +2023-10-17T13:39:16.664434Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: new +2023-10-17T13:39:16.664430Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#8008000000004: close time.busy=193µs time.idle=35.0µs +2023-10-17T13:39:16.664444Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: enter +2023-10-17T13:39:16.664454Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: new +2023-10-17T13:39:16.664452Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: exit +2023-10-17T13:39:16.664466Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: enter +2023-10-17T13:39:16.664468Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#10008000000003{state=Leader id=0}: close time.busy=343µs time.idle=33.2µs +2023-10-17T13:39:16.664477Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: exit +2023-10-17T13:39:16.664496Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: exit +2023-10-17T13:39:16.664506Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.664499Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18008000000003: new +2023-10-17T13:39:16.664515Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18008000000003: enter +2023-10-17T13:39:16.664535Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18008000000003: queued commands: start... +2023-10-17T13:39:16.664552Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18008000000003: queued commands: end... +2023-10-17T13:39:16.664570Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18008000000003: exit +2023-10-17T13:39:16.664588Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#18008000000003: close time.busy=55.8µs time.idle=33.5µs +2023-10-17T13:39:16.664625Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: new +2023-10-17T13:39:16.664642Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: enter +2023-10-17T13:39:16.664660Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.664687Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: new +2023-10-17T13:39:16.664706Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: enter +2023-10-17T13:39:16.664723Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.664747Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#10008000000005: new +2023-10-17T13:39:16.664767Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#10008000000005: enter +2023-10-17T13:39:16.664787Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#10008000000005: this node is a leader +2023-10-17T13:39:16.664807Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#10008000000005: exit +2023-10-17T13:39:16.664830Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#10008000000005: close time.busy=40.4µs time.idle=42.6µs +2023-10-17T13:39:16.664858Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: new +2023-10-17T13:39:16.664877Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: enter +2023-10-17T13:39:16.664896Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: assign log id: 5-0-8 +2023-10-17T13:39:16.664931Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: openraft::engine::handler::replication_handler@initiate_replication#8000000006: new +2023-10-17T13:39:16.664951Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: openraft::engine::handler::replication_handler@initiate_replication#8000000006: enter +2023-10-17T13:39:16.664972Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: openraft::engine::handler::replication_handler@initiate_replication#8000000006: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }), curr_inflight_id: 0, inflight: None, searching_end: 8 })], stat: Stat { update_count: 7, move_count: 7, is_quorum_count: 7 } } +2023-10-17T13:39:16.665016Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: openraft::engine::handler::replication_handler@initiate_replication#8000000006: exit +2023-10-17T13:39:16.665036Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: openraft::engine::handler::replication_handler@initiate_replication#8000000006: close time.busy=65.0µs time.idle=41.0µs +2023-10-17T13:39:16.665064Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: exit +2023-10-17T13:39:16.665082Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#18008000000005: close time.busy=187µs time.idle=38.0µs +2023-10-17T13:39:16.665108Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: exit +2023-10-17T13:39:16.665125Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#10008000000004{id=0}: close time.busy=402µs time.idle=37.0µs +2023-10-17T13:39:16.665151Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: exit +2023-10-17T13:39:16.665168Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#20008000000003{state=Leader id=0}: close time.busy=508µs time.idle=35.2µs +2023-10-17T13:39:16.665192Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: new +2023-10-17T13:39:16.665209Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: enter +2023-10-17T13:39:16.665225Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: queued commands: start... +2023-10-17T13:39:16.665242Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }, payload: normal }] } +2023-10-17T13:39:16.665261Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: queued commands: end... +2023-10-17T13:39:16.665280Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }, payload: normal }] } +2023-10-17T13:39:16.665299Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: condition: None +2023-10-17T13:39:16.665316Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: AppendInputEntries: [5-0-8:normal] +2023-10-17T13:39:16.665337Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@append_to_log#18008000000004: new +2023-10-17T13:39:16.665356Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@append_to_log#18008000000004: enter +2023-10-17T13:39:16.665373Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@append_to_log#18008000000004: append_to_log +2023-10-17T13:39:16.665433Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@append_to_log#18008000000004: exit +2023-10-17T13:39:16.665451Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@append_to_log#18008000000004: close time.busy=77.1µs time.idle=37.1µs +2023-10-17T13:39:16.665478Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-8 +2023-10-17T13:39:16.665501Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: update progress self_matching=5-0-7 +2023-10-17T13:39:16.665523Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: new +2023-10-17T13:39:16.665541Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: enter +2023-10-17T13:39:16.665564Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-8 +2023-10-17T13:39:16.665588Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: update_matching self={[Some(5-0-7), 8), inflight:Logs(id=0):(None, Some(5-0-8)]} request_id=0 matching=Some(5-0-8) +2023-10-17T13:39:16.665621Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: granted after updating progress granted=5-0-8 +2023-10-17T13:39:16.665644Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: new +2023-10-17T13:39:16.665665Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: enter +2023-10-17T13:39:16.665687Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: openraft::raft_state@update_committed#8008000000006: new +2023-10-17T13:39:16.665707Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: openraft::raft_state@update_committed#8008000000006: enter +2023-10-17T13:39:16.665728Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: openraft::raft_state@update_committed#8008000000006: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.665759Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: openraft::raft_state@update_committed#8008000000006: exit +2023-10-17T13:39:16.665779Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: openraft::raft_state@update_committed#8008000000006: close time.busy=51.3µs time.idle=40.9µs +2023-10-17T13:39:16.665806Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: exit +2023-10-17T13:39:16.665825Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: openraft::engine::handler::replication_handler@try_commit_granted#20008000000005: close time.busy=141µs time.idle=40.0µs +2023-10-17T13:39:16.665849Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: exit +2023-10-17T13:39:16.665866Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::engine::handler::replication_handler@update_matching#20008000000004: close time.busy=308µs time.idle=37.3µs +2023-10-17T13:39:16.665889Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }) } +2023-10-17T13:39:16.665907Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: condition: None +2023-10-17T13:39:16.665929Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: run command: Apply { seq: 8, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 7 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 } } +2023-10-17T13:39:16.665945Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: condition: None +2023-10-17T13:39:16.665965Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: new +2023-10-17T13:39:16.665984Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: enter +2023-10-17T13:39:16.666001Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: apply_to_state_machine upto_index=8 +2023-10-17T13:39:16.666049Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: about to apply entries=[5-0-8:normal] +2023-10-17T13:39:16.666069Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: sending command to state machine worker: StateMachineCommand { seq: 8, payload: Apply: [5-0-8:normal] } +2023-10-17T13:39:16.666104Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: exit +2023-10-17T13:39:16.666122Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: openraft::core::raft_core@apply_to_state_machine#28008000000004: close time.busy=120µs time.idle=37.1µs +2023-10-17T13:39:16.666145Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: exit +2023-10-17T13:39:16.666161Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#28008000000003: close time.busy=936µs time.idle=33.9µs +2023-10-17T13:39:16.666182Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.666199Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.666219Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: new +2023-10-17T13:39:16.666235Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: enter +2023-10-17T13:39:16.666262Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: new +2023-10-17T13:39:16.666281Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: enter +2023-10-17T13:39:16.666302Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: openraft::core::raft_core@current_leader#28008000000005: new +2023-10-17T13:39:16.666320Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: openraft::core::raft_core@current_leader#28008000000005: enter +2023-10-17T13:39:16.666345Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: openraft::core::raft_core@current_leader#28008000000005: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.666372Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: openraft::core::raft_core@current_leader#28008000000005: exit +2023-10-17T13:39:16.666391Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: openraft::core::raft_core@current_leader#28008000000005: close time.busy=51.3µs time.idle=37.4µs +2023-10-17T13:39:16.666447Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:8, last_applied:5-0-7, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-8}} +2023-10-17T13:39:16.666477Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: exit +2023-10-17T13:39:16.666494Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: openraft::core::raft_core@report_metrics#30008000000004: close time.busy=195µs time.idle=38.0µs +2023-10-17T13:39:16.666516Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: exit +2023-10-17T13:39:16.666532Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#30008000000003: close time.busy=281µs time.idle=33.3µs +2023-10-17T13:39:16.666559Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.666576Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.666591Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.666607Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.666616Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.666621Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 8, payload: Apply: [5-0-8:normal] } +2023-10-17T13:39:16.666629Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.666634Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#38008000000003: new +2023-10-17T13:39:16.666641Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.666647Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#38008000000003: enter +2023-10-17T13:39:16.666657Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.666663Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#38008000000003: replicate to sm entry.log_id=5-0-8 +2023-10-17T13:39:16.666670Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.666684Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#38008000000003: exit +2023-10-17T13:39:16.666684Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.666691Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#38008000000003: close time.busy=36.9µs time.idle=21.3µs +2023-10-17T13:39:16.666710Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.666722Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.666734Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.666749Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.666772Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: new +2023-10-17T13:39:16.666791Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: enter +2023-10-17T13:39:16.666809Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 8, result: Ok(Apply(ApplyResult { since: 8, end: 9, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 } })) } +2023-10-17T13:39:16.666826Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 8, result: Ok(Apply(ApplyResult { since: 8, end: 9, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 } })) } +2023-10-17T13:39:16.666850Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-8 +2023-10-17T13:39:16.666871Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: new +2023-10-17T13:39:16.666889Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: enter +2023-10-17T13:39:16.666907Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-8 +2023-10-17T13:39:16.666931Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core@send_response#30008000000005: new +2023-10-17T13:39:16.666953Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core@send_response#30008000000005: enter +2023-10-17T13:39:16.666972Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core@send_response#30008000000005: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }, membership: None } +2023-10-17T13:39:16.666998Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.666993Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core@send_response#30008000000005: send client response through tx, send_res is error: false +2023-10-17T13:39:16.667008Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: enter +2023-10-17T13:39:16.667016Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: enter +2023-10-17T13:39:16.667012Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core@send_response#30008000000005: exit +2023-10-17T13:39:16.667026Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: call_core receives result is error: false +2023-10-17T13:39:16.667035Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: exit +2023-10-17T13:39:16.667029Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: openraft::core::raft_core@send_response#30008000000005: close time.busy=58.4µs time.idle=41.2µs +2023-10-17T13:39:16.667045Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: openraft::raft@call_core#30000000000007: close time.busy=30.9µs time.idle=2.56ms +2023-10-17T13:39:16.667059Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: exit +2023-10-17T13:39:16.667051Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: exit +2023-10-17T13:39:16.667072Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#30000000000006: close time.busy=102µs time.idle=2.54ms +2023-10-17T13:39:16.667087Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: new +2023-10-17T13:39:16.667078Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#38008000000004: close time.busy=162µs time.idle=45.5µs +2023-10-17T13:39:16.667096Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: enter +2023-10-17T13:39:16.667105Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: new +2023-10-17T13:39:16.667101Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: exit +2023-10-17T13:39:16.667116Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: enter +2023-10-17T13:39:16.667129Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: exit +2023-10-17T13:39:16.667122Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#40008000000003{state=Leader id=0}: close time.busy=309µs time.idle=40.3µs +2023-10-17T13:39:16.667140Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: exit +2023-10-17T13:39:16.667147Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.667145Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48008000000003: new +2023-10-17T13:39:16.667159Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48008000000003: enter +2023-10-17T13:39:16.667173Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48008000000003: queued commands: start... +2023-10-17T13:39:16.667186Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48008000000003: queued commands: end... +2023-10-17T13:39:16.667202Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48008000000003: exit +2023-10-17T13:39:16.667216Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#48008000000003: close time.busy=43.2µs time.idle=28.2µs +2023-10-17T13:39:16.667242Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: new +2023-10-17T13:39:16.667257Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: enter +2023-10-17T13:39:16.667270Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: recv from rx_api: ClientWriteRequest +2023-10-17T13:39:16.667289Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: new +2023-10-17T13:39:16.667305Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: enter +2023-10-17T13:39:16.667320Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: write_entry payload=0-0-0:normal +2023-10-17T13:39:16.667340Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#38008000000005: new +2023-10-17T13:39:16.667356Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#38008000000005: enter +2023-10-17T13:39:16.667373Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#38008000000005: this node is a leader +2023-10-17T13:39:16.667391Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#38008000000005: exit +2023-10-17T13:39:16.667407Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::engine_impl@get_leader_handler_or_reject#38008000000005: close time.busy=34.4µs time.idle=33.8µs +2023-10-17T13:39:16.667430Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: new +2023-10-17T13:39:16.667448Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: enter +2023-10-17T13:39:16.667463Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: assign log id: 5-0-9 +2023-10-17T13:39:16.667484Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: openraft::engine::handler::replication_handler@initiate_replication#10008000000006: new +2023-10-17T13:39:16.667502Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: openraft::engine::handler::replication_handler@initiate_replication#10008000000006: enter +2023-10-17T13:39:16.667520Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: openraft::engine::handler::replication_handler@initiate_replication#10008000000006: openraft::engine::handler::replication_handler::ReplicationHandler<_>::initiate_replication progress=VecProgress { quorum_set: Joint { data: [[0]], _p: PhantomData<(u64, alloc::vec::Vec)> }, granted: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }), voter_count: 1, vector: [(0, ProgressEntry { matching: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }), curr_inflight_id: 0, inflight: None, searching_end: 9 })], stat: Stat { update_count: 8, move_count: 8, is_quorum_count: 8 } } +2023-10-17T13:39:16.667545Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: openraft::engine::handler::replication_handler@initiate_replication#10008000000006: exit +2023-10-17T13:39:16.667563Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: openraft::engine::handler::replication_handler@initiate_replication#10008000000006: close time.busy=42.5µs time.idle=36.7µs +2023-10-17T13:39:16.667585Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: exit +2023-10-17T13:39:16.667602Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: openraft::engine::handler::leader_handler@leader_append_entries#40008000000005: close time.busy=137µs time.idle=34.6µs +2023-10-17T13:39:16.667623Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: exit +2023-10-17T13:39:16.667639Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: openraft::core::raft_core@write_entry#40008000000004{id=0}: close time.busy=318µs time.idle=31.5µs +2023-10-17T13:39:16.667659Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: exit +2023-10-17T13:39:16.667673Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_api_msg#50008000000003{state=Leader id=0}: close time.busy=402µs time.idle=30.0µs +2023-10-17T13:39:16.667694Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: new +2023-10-17T13:39:16.667709Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: enter +2023-10-17T13:39:16.667722Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: queued commands: start... +2023-10-17T13:39:16.667736Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }, payload: normal }] } +2023-10-17T13:39:16.667751Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: queued commands: end... +2023-10-17T13:39:16.667765Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }, payload: normal }] } +2023-10-17T13:39:16.667781Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: condition: None +2023-10-17T13:39:16.667795Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: AppendInputEntries: [5-0-9:normal] +2023-10-17T13:39:16.667813Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@append_to_log#48008000000004: new +2023-10-17T13:39:16.667829Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@append_to_log#48008000000004: enter +2023-10-17T13:39:16.667843Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@append_to_log#48008000000004: append_to_log +2023-10-17T13:39:16.667872Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@append_to_log#48008000000004: exit +2023-10-17T13:39:16.667888Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@append_to_log#48008000000004: close time.busy=43.1µs time.idle=32.3µs +2023-10-17T13:39:16.667909Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_local_progress upto=5-0-9 +2023-10-17T13:39:16.667926Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: update progress self_matching=5-0-8 +2023-10-17T13:39:16.667944Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: new +2023-10-17T13:39:16.667959Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: enter +2023-10-17T13:39:16.667975Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler::ReplicationHandler<_>::update_matching node_id=0 inflight_id=0 log_id=5-0-9 +2023-10-17T13:39:16.667995Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: update_matching self={[Some(5-0-8), 9), inflight:Logs(id=0):(None, Some(5-0-9)]} request_id=0 matching=Some(5-0-9) +2023-10-17T13:39:16.668016Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: granted after updating progress granted=5-0-9 +2023-10-17T13:39:16.668035Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: new +2023-10-17T13:39:16.668052Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: enter +2023-10-17T13:39:16.668070Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::raft_state@update_committed#18008000000006: new +2023-10-17T13:39:16.668088Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::raft_state@update_committed#18008000000006: enter +2023-10-17T13:39:16.668105Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::raft_state@update_committed#18008000000006: committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} }, voter_ids: {0} } +2023-10-17T13:39:16.668128Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::raft_state@update_committed#18008000000006: exit +2023-10-17T13:39:16.668146Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::raft_state@update_committed#18008000000006: close time.busy=40.1µs time.idle=35.9µs +2023-10-17T13:39:16.668377Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: new +2023-10-17T13:39:16.668396Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: enter +2023-10-17T13:39:16.668491Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: openraft::engine::handler::snapshot_handler::SnapshotHandler<_>::trigger_snapshot +2023-10-17T13:39:16.668559Z INFO ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: push snapshot building command +2023-10-17T13:39:16.668786Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: next_seq: 10 +2023-10-17T13:39:16.668831Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: exit +2023-10-17T13:39:16.668849Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: openraft::engine::handler::snapshot_handler@trigger_snapshot#20008000000006: close time.busy=434µs time.idle=37.8µs +2023-10-17T13:39:16.668874Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: exit +2023-10-17T13:39:16.668892Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: openraft::engine::handler::replication_handler@try_commit_granted#48008000000005: close time.busy=822µs time.idle=34.6µs +2023-10-17T13:39:16.668912Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: exit +2023-10-17T13:39:16.668928Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::engine::handler::replication_handler@update_matching#50008000000004: close time.busy=953µs time.idle=32.4µs +2023-10-17T13:39:16.668947Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: run command: ReplicateCommitted { committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }) } +2023-10-17T13:39:16.668962Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: condition: None +2023-10-17T13:39:16.668976Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: run command: Apply { seq: 9, already_committed: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 8 }), upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } } +2023-10-17T13:39:16.668992Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: condition: None +2023-10-17T13:39:16.669007Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: new +2023-10-17T13:39:16.669023Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: enter +2023-10-17T13:39:16.669038Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: apply_to_state_machine upto_index=9 +2023-10-17T13:39:16.669065Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: about to apply entries=[5-0-9:normal] +2023-10-17T13:39:16.669082Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: sending command to state machine worker: StateMachineCommand { seq: 9, payload: Apply: [5-0-9:normal] } +2023-10-17T13:39:16.669102Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: exit +2023-10-17T13:39:16.669118Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: openraft::core::raft_core@apply_to_state_machine#58008000000004: close time.busy=78.5µs time.idle=32.6µs +2023-10-17T13:39:16.669135Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: run command: StateMachine { command: StateMachineCommand { seq: 10, payload: BuildSnapshot } } +2023-10-17T13:39:16.669184Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: condition: None +2023-10-17T13:39:16.669218Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: sending command to state machine worker: StateMachineCommand { seq: 10, payload: BuildSnapshot } +2023-10-17T13:39:16.669255Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: exit +2023-10-17T13:39:16.669270Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#58008000000003: close time.busy=1.55ms time.idle=30.2µs +2023-10-17T13:39:16.669288Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.669301Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.669317Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: new +2023-10-17T13:39:16.669332Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: enter +2023-10-17T13:39:16.669349Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: new +2023-10-17T13:39:16.669365Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: enter +2023-10-17T13:39:16.669384Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: openraft::core::raft_core@current_leader#50008000000005: new +2023-10-17T13:39:16.669401Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: openraft::core::raft_core@current_leader#50008000000005: enter +2023-10-17T13:39:16.669418Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: openraft::core::raft_core@current_leader#50008000000005: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.669438Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: openraft::core::raft_core@current_leader#50008000000005: exit +2023-10-17T13:39:16.669455Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: openraft::core::raft_core@current_leader#50008000000005: close time.busy=37.7µs time.idle=33.4µs +2023-10-17T13:39:16.669490Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-8, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.669510Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: exit +2023-10-17T13:39:16.669526Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: openraft::core::raft_core@report_metrics#60008000000004: close time.busy=146µs time.idle=32.1µs +2023-10-17T13:39:16.669546Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: exit +2023-10-17T13:39:16.669560Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#60008000000003: close time.busy=214µs time.idle=29.3µs +2023-10-17T13:39:16.669581Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.669595Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.669608Z DEBUG ThreadId(10) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.669622Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: enter +2023-10-17T13:39:16.669628Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.669630Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 9, payload: Apply: [5-0-9:normal] } +2023-10-17T13:39:16.669641Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#68008000000003: new +2023-10-17T13:39:16.669638Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.669649Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#68008000000003: enter +2023-10-17T13:39:16.669649Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.669659Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#68008000000003: replicate to sm entry.log_id=5-0-9 +2023-10-17T13:39:16.669664Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.669675Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#68008000000003: exit +2023-10-17T13:39:16.669678Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.669683Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@apply#68008000000003: close time.busy=26.4µs time.idle=15.5µs +2023-10-17T13:39:16.669691Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.669707Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.669713Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 10, payload: BuildSnapshot } +2023-10-17T13:39:16.669718Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.669729Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.669752Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: new +2023-10-17T13:39:16.669770Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: enter +2023-10-17T13:39:16.669786Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 9, result: Ok(Apply(ApplyResult { since: 9, end: 10, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } })) } +2023-10-17T13:39:16.669802Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 9, result: Ok(Apply(ApplyResult { since: 9, end: 10, last_applied: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } })) } +2023-10-17T13:39:16.669819Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-9 +2023-10-17T13:39:16.669839Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: new +2023-10-17T13:39:16.669856Z INFO ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm::Worker<_, _>::worker_loop: build snapshot +2023-10-17T13:39:16.669855Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: enter +2023-10-17T13:39:16.669870Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=5-0-9 +2023-10-17T13:39:16.669892Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core@send_response#78004000000004: new +2023-10-17T13:39:16.669910Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core@send_response#78004000000004: enter +2023-10-17T13:39:16.669926Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core@send_response#78004000000004: send_response entry=ApplyingEntry { log_id: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }, membership: None } +2023-10-17T13:39:16.669952Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.669948Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core@send_response#78004000000004: send client response through tx, send_res is error: false +2023-10-17T13:39:16.669961Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: enter +2023-10-17T13:39:16.669968Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: enter +2023-10-17T13:39:16.669965Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core@send_response#78004000000004: exit +2023-10-17T13:39:16.669977Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: call_core receives result is error: false +2023-10-17T13:39:16.669987Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: exit +2023-10-17T13:39:16.669981Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: openraft::core::raft_core@send_response#78004000000004: close time.busy=55.6µs time.idle=34.3µs +2023-10-17T13:39:16.669996Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: openraft::raft@call_core#38000000000007: close time.busy=31.7µs time.idle=2.86ms +2023-10-17T13:39:16.670009Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: exit +2023-10-17T13:39:16.670004Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: exit +2023-10-17T13:39:16.670018Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@client_write#38000000000006: close time.busy=91.9µs time.idle=2.84ms +2023-10-17T13:39:16.670018Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: openraft::core::raft_core@handle_apply_result#90004000000003: close time.busy=148µs time.idle=31.2µs +2023-10-17T13:39:16.670038Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: exit +2023-10-17T13:39:16.670053Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#90004000000002{state=Leader id=0}: close time.busy=268µs time.idle=34.3µs +2023-10-17T13:39:16.670069Z INFO ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@build_snapshot#70008000000003: new +2023-10-17T13:39:16.670071Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#98004000000002: new +2023-10-17T13:39:16.670087Z INFO ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@build_snapshot#70008000000003: enter +2023-10-17T13:39:16.670090Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait trigger snapshot .last_log_index -> Some(9) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-8, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.670085Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#98004000000002: enter +2023-10-17T13:39:16.670097Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#98004000000002: queued commands: start... +2023-10-17T13:39:16.670113Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 done wait trigger snapshot .last_log_index -> Some(9) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-8, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.670111Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#98004000000002: queued commands: end... +2023-10-17T13:39:16.670125Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#98004000000002: exit +2023-10-17T13:39:16.670141Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait trigger snapshot .last_applied -> Some(9) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-8, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.670140Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#98004000000002: close time.busy=40.4µs time.idle=28.3µs +2023-10-17T13:39:16.670149Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999978s +2023-10-17T13:39:16.670157Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.670172Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.670179Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait trigger snapshot .last_applied -> Some(9) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-8, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.670185Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999940667s +2023-10-17T13:39:16.670187Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: new +2023-10-17T13:39:16.670199Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.670201Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: enter +2023-10-17T13:39:16.670222Z INFO ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@build_snapshot#70008000000003: openraft::core::sm::Worker<_, _>::build_snapshot +2023-10-17T13:39:16.670219Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: new +2023-10-17T13:39:16.670234Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: enter +2023-10-17T13:39:16.670254Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: openraft::core::raft_core@current_leader#80004000000004: new +2023-10-17T13:39:16.670271Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: openraft::core::raft_core@current_leader#80004000000004: enter +2023-10-17T13:39:16.670288Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: openraft::core::raft_core@current_leader#80004000000004: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.670309Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: openraft::core::raft_core@current_leader#80004000000004: exit +2023-10-17T13:39:16.670332Z INFO ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@build_snapshot#70008000000003: exit +2023-10-17T13:39:16.670325Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: openraft::core::raft_core@current_leader#80004000000004: close time.busy=37.3µs time.idle=34.1µs +2023-10-17T13:39:16.670343Z INFO ThreadId(10) openraft::core::sm@worker_loop#4000000001: openraft::core::sm@build_snapshot#70008000000003: close time.busy=244µs time.idle=29.6µs +2023-10-17T13:39:16.670362Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.670391Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: exit +2023-10-17T13:39:16.670397Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.670408Z DEBUG ThreadId(10) openraft::core::sm@worker_loop#4000000001: exit +2023-10-17T13:39:16.670412Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: openraft::core::raft_core@report_metrics#98004000000003: close time.busy=156µs time.idle=36.7µs +2023-10-17T13:39:16.670432Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: exit +2023-10-17T13:39:16.670445Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#a0004000000002: close time.busy=231µs time.idle=27.7µs +2023-10-17T13:39:16.670467Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.670482Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.670493Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.670518Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait trigger snapshot .last_applied -> Some(9) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.670543Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 done wait trigger snapshot .last_applied -> Some(9) latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.671571Z INFO ThreadId(06) log compaction complete snapshot_size=275 +2023-10-17T13:39:16.671874Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait build snapshot .snapshot -> 5-0-9 latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.671960Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999807042s +2023-10-17T13:39:16.672019Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait build snapshot .snapshot -> 5-0-9 latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:None, purged:None, replication:{0:5-0-9}} +2023-10-17T13:39:16.672027Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.99967425s +2023-10-17T13:39:16.672040Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.672046Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: enter +2023-10-17T13:39:16.672057Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: enter +2023-10-17T13:39:16.672069Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: enter +2023-10-17T13:39:16.672092Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: new +2023-10-17T13:39:16.672106Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: enter +2023-10-17T13:39:16.672255Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 10, result: Ok(BuildSnapshot(SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-1" })) } +2023-10-17T13:39:16.672271Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: sm::StateMachine command result: CommandResult { command_seq: 10, result: Ok(BuildSnapshot(SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-1" })) } +2023-10-17T13:39:16.672464Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: sm::StateMachine command done: BuildSnapshot: {snapshot_id: 5-0-9-1, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}}: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_notify +2023-10-17T13:39:16.672695Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: new +2023-10-17T13:39:16.672712Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: enter +2023-10-17T13:39:16.672807Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl::Engine<_>::finish_building_snapshot snapshot_meta={snapshot_id: 5-0-9-1, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.672970Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::snapshot_handler@update_snapshot#a000c000000003: new +2023-10-17T13:39:16.672988Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::snapshot_handler@update_snapshot#a000c000000003: enter +2023-10-17T13:39:16.673123Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::snapshot_handler@update_snapshot#a000c000000003: update_snapshot: SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-1" } +2023-10-17T13:39:16.673166Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::snapshot_handler@update_snapshot#a000c000000003: exit +2023-10-17T13:39:16.673184Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::snapshot_handler@update_snapshot#a000c000000003: close time.busy=178µs time.idle=38.8µs +2023-10-17T13:39:16.673324Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: new +2023-10-17T13:39:16.673342Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: enter +2023-10-17T13:39:16.673532Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@calc_purge_upto#4000c000000004: new +2023-10-17T13:39:16.673555Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@calc_purge_upto#4000c000000004: enter +2023-10-17T13:39:16.673640Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@calc_purge_upto#4000c000000004: try purge: (-oo, 10) snapshot_last_log_id=Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }) max_keep=0 +2023-10-17T13:39:16.673849Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@calc_purge_upto#4000c000000004: exit +2023-10-17T13:39:16.673868Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@calc_purge_upto#4000c000000004: close time.busy=295µs time.idle=41.8µs +2023-10-17T13:39:16.674010Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@update_purge_upto#4800c000000004: new +2023-10-17T13:39:16.674028Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@update_purge_upto#4800c000000004: enter +2023-10-17T13:39:16.674050Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@update_purge_upto#4800c000000004: exit +2023-10-17T13:39:16.674068Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: openraft::engine::handler::log_handler@update_purge_upto#4800c000000004: close time.busy=21.5µs time.idle=37.3µs +2023-10-17T13:39:16.674089Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: exit +2023-10-17T13:39:16.674111Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::handler::log_handler@schedule_policy_based_purge#a800c000000003: close time.busy=747µs time.idle=39.2µs +2023-10-17T13:39:16.674249Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: new +2023-10-17T13:39:16.674266Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: enter +2023-10-17T13:39:16.674368Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::engine_impl::Engine<_>::try_purge_log purge_upto=5-0-9 +2023-10-17T13:39:16.674621Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: new +2023-10-17T13:39:16.674639Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: enter +2023-10-17T13:39:16.674814Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: try_purge_log last_purged_log_id=None purge_upto=Some(5-0-9) +2023-10-17T13:39:16.674981Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: new +2023-10-17T13:39:16.675001Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: enter +2023-10-17T13:39:16.675098Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: purge_log last_purged_log_id=None purge_upto=Some(5-0-9) +2023-10-17T13:39:16.675260Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: openraft::raft_state@purge_log#800c000000006: new +2023-10-17T13:39:16.675286Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: openraft::raft_state@purge_log#800c000000006: enter +2023-10-17T13:39:16.675310Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: openraft::raft_state@purge_log#800c000000006: exit +2023-10-17T13:39:16.675330Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: openraft::raft_state@purge_log#800c000000006: close time.busy=24.4µs time.idle=48.9µs +2023-10-17T13:39:16.675354Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: exit +2023-10-17T13:39:16.675372Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: openraft::engine::handler::log_handler@purge_log#2000c000000005: close time.busy=353µs time.idle=38.7µs +2023-10-17T13:39:16.675395Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: exit +2023-10-17T13:39:16.675413Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: openraft::engine::handler::replication_handler@try_purge_log#5000c000000004: close time.busy=756µs time.idle=36.7µs +2023-10-17T13:39:16.675434Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: exit +2023-10-17T13:39:16.675450Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: openraft::engine::engine_impl@try_purge_log#b000c000000003: close time.busy=1.17ms time.idle=33.9µs +2023-10-17T13:39:16.675470Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: exit +2023-10-17T13:39:16.675486Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: openraft::engine::engine_impl@finish_building_snapshot#c800c000000002: close time.busy=2.76ms time.idle=33.2µs +2023-10-17T13:39:16.675514Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: exit +2023-10-17T13:39:16.675529Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@handle_notify#c000c000000001{state=Leader id=0}: close time.busy=3.41ms time.idle=29.5µs +2023-10-17T13:39:16.675552Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: new +2023-10-17T13:39:16.675567Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: enter +2023-10-17T13:39:16.675581Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: queued commands: start... +2023-10-17T13:39:16.675595Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: queued commands: PurgeLog { upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } } +2023-10-17T13:39:16.675626Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: queued commands: end... +2023-10-17T13:39:16.675640Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: run command: PurgeLog { upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } } +2023-10-17T13:39:16.675666Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: condition: None +2023-10-17T13:39:16.675951Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: openraft_memstore@purge_logs_upto#d000c000000002: new +2023-10-17T13:39:16.675985Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: openraft_memstore@purge_logs_upto#d000c000000002: enter +2023-10-17T13:39:16.676248Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: openraft_memstore@purge_logs_upto#d000c000000002: purge_log_upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } +2023-10-17T13:39:16.676504Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: openraft_memstore@purge_logs_upto#d000c000000002: exit +2023-10-17T13:39:16.676552Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: openraft_memstore@purge_logs_upto#d000c000000002: close time.busy=519µs time.idle=82.0µs +2023-10-17T13:39:16.676642Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: exit +2023-10-17T13:39:16.676656Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@run_engine_commands#c800c000000001: close time.busy=1.07ms time.idle=29.8µs +2023-10-17T13:39:16.676674Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.676687Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: all Notify are processed, wait for more +2023-10-17T13:39:16.676701Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: new +2023-10-17T13:39:16.676714Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: enter +2023-10-17T13:39:16.676732Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: new +2023-10-17T13:39:16.676747Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: enter +2023-10-17T13:39:16.676766Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: openraft::core::raft_core@current_leader#b800c000000003: new +2023-10-17T13:39:16.676783Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: openraft::core::raft_core@current_leader#b800c000000003: enter +2023-10-17T13:39:16.676799Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: openraft::core::raft_core@current_leader#b800c000000003: get current_leader self_id=0 vote=5-0:committed +2023-10-17T13:39:16.676823Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: openraft::core::raft_core@current_leader#b800c000000003: exit +2023-10-17T13:39:16.676839Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: openraft::core::raft_core@current_leader#b800c000000003: close time.busy=39.8µs time.idle=33.2µs +2023-10-17T13:39:16.676875Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: report_metrics: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{0:5-0-9}} +2023-10-17T13:39:16.676904Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.676900Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: exit +2023-10-17T13:39:16.676914Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: openraft::core::raft_core@report_metrics#d800c000000002: close time.busy=152µs time.idle=30.3µs +2023-10-17T13:39:16.676935Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: exit +2023-10-17T13:39:16.676951Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: openraft::core::raft_core@flush_metrics#d000c000000001: close time.busy=221µs time.idle=29.8µs +2023-10-17T13:39:16.676973Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: exit +2023-10-17T13:39:16.676987Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: exit +2023-10-17T13:39:16.676998Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: exit +2023-10-17T13:39:16.677093Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 wait build snapshot .snapshot -> 5-0-9 latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{0:5-0-9}} +2023-10-17T13:39:16.677225Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=0 done wait build snapshot .snapshot -> 5-0-9 latest: Metrics{id:0, Leader, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:0, membership:{log_id:0-0-0, voters:[{0:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{0:5-0-9}} +2023-10-17T13:39:16.677520Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- create node-1 and add conflicting logs log_index=9 +2023-10-17T13:39:16.677579Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: new +2023-10-17T13:39:16.677589Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: enter +2023-10-17T13:39:16.677604Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: new +2023-10-17T13:39:16.677615Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.677644Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::core::tick@tick#8: new +2023-10-17T13:39:16.677672Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: new +2023-10-17T13:39:16.677676Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::core::tick@tick#8: enter +2023-10-17T13:39:16.677692Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::core::tick@tick#8: exit +2023-10-17T13:39:16.677722Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: RaftStorage::get_membership membership_in_sm=StoredMembership { log_id: None, membership: Membership { configs: [], nodes: {} } } membership_in_log=[] +2023-10-17T13:39:16.677795Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: save_vote vote=Vote { leader_id: LeaderId { term: 0, node_id: 0 }, committed: false } +2023-10-17T13:39:16.677841Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#d800c000000001: new +2023-10-17T13:39:16.677845Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.677851Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#d800c000000001: enter +2023-10-17T13:39:16.677848Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.677859Z DEBUG ThreadId(06) openraft::core::sm@worker_loop#d800c000000001: exit +2023-10-17T13:39:16.677859Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: exit +2023-10-17T13:39:16.677860Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: new +2023-10-17T13:39:16.677872Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.677885Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: raft node is initializing +2023-10-17T13:39:16.677901Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: new +2023-10-17T13:39:16.677915Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: enter +2023-10-17T13:39:16.677928Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: startup: state: RaftState { vote: UTime { data: Vote { leader_id: LeaderId { term: 0, node_id: 0 }, committed: false }, utime: Some(Instant { t: 1043048327399750 }) }, committed: None, purged_next: 0, log_ids: LogIdList { key_log_ids: [] }, membership_state: MembershipState { committed: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} }, effective: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} } }, snapshot_meta: SnapshotMeta { last_log_id: None, last_membership: StoredMembership { log_id: None, membership: Membership { configs: [], nodes: {} } }, snapshot_id: "" }, server_state: Learner, accepted: Accepted { leader_id: LeaderId { term: 0, node_id: 0 }, log_id: None }, io_state: IOState { building_snapshot: false, vote: Vote { leader_id: LeaderId { term: 0, node_id: 0 }, committed: false }, flushed: LogIOId { leader_id: LeaderId { term: 0, node_id: 0 }, log_id: None }, applied: None, purged: None }, snapshot_streaming: None, purge_upto: None } +2023-10-17T13:39:16.677957Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: startup: is_leader: false +2023-10-17T13:39:16.677971Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: startup: is_voter: false +2023-10-17T13:39:16.677985Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: startup: id=1 target_state: Learner +2023-10-17T13:39:16.678000Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: exit +2023-10-17T13:39:16.678012Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::engine::engine_impl@startup#a0004000000003: close time.busy=84.4µs time.idle=27.4µs +2023-10-17T13:39:16.678032Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@run_engine_commands#a8004000000003: new +2023-10-17T13:39:16.678046Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@run_engine_commands#a8004000000003: enter +2023-10-17T13:39:16.678058Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@run_engine_commands#a8004000000003: queued commands: start... +2023-10-17T13:39:16.678071Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@run_engine_commands#a8004000000003: queued commands: end... +2023-10-17T13:39:16.678085Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@run_engine_commands#a8004000000003: exit +2023-10-17T13:39:16.678097Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@run_engine_commands#a8004000000003: close time.busy=39.0µs time.idle=25.9µs +2023-10-17T13:39:16.678118Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: new +2023-10-17T13:39:16.678131Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: enter +2023-10-17T13:39:16.678147Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: openraft::core::raft_core@current_leader#88004000000004: new +2023-10-17T13:39:16.678163Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: openraft::core::raft_core@current_leader#88004000000004: enter +2023-10-17T13:39:16.678177Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: openraft::core::raft_core@current_leader#88004000000004: get current_leader self_id=1 vote=0-0:uncommitted +2023-10-17T13:39:16.678194Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: openraft::core::raft_core@current_leader#88004000000004: exit +2023-10-17T13:39:16.678207Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: openraft::core::raft_core@current_leader#88004000000004: close time.busy=30.8µs time.idle=29.3µs +2023-10-17T13:39:16.678233Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: report_metrics: Metrics{id:1, Learner, term:0, vote:0-0:uncommitted, last_log:None, last_applied:None, leader:None, membership:{log_id:None, voters:[], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.678253Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: exit +2023-10-17T13:39:16.678267Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@report_metrics#b0004000000003: close time.busy=122µs time.idle=27.5µs +2023-10-17T13:39:16.678289Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: new +2023-10-17T13:39:16.678303Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.678319Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: new +2023-10-17T13:39:16.678333Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: enter +2023-10-17T13:39:16.678350Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: new +2023-10-17T13:39:16.678365Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: enter +2023-10-17T13:39:16.678384Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: openraft::core::raft_core@current_leader#4000000006: new +2023-10-17T13:39:16.678401Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: openraft::core::raft_core@current_leader#4000000006: enter +2023-10-17T13:39:16.678417Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: openraft::core::raft_core@current_leader#4000000006: get current_leader self_id=1 vote=0-0:uncommitted +2023-10-17T13:39:16.678437Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: openraft::core::raft_core@current_leader#4000000006: exit +2023-10-17T13:39:16.678454Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: openraft::core::raft_core@current_leader#4000000006: close time.busy=36.2µs time.idle=34.0µs +2023-10-17T13:39:16.678481Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: report_metrics: Metrics{id:1, Learner, term:0, vote:0-0:uncommitted, last_log:None, last_applied:None, leader:None, membership:{log_id:None, voters:[], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.678500Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: exit +2023-10-17T13:39:16.678515Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: openraft::core::raft_core@report_metrics#30004000000005: close time.busy=135µs time.idle=30.8µs +2023-10-17T13:39:16.678534Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: exit +2023-10-17T13:39:16.678549Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#90004000000004: close time.busy=201µs time.idle=29.1µs +2023-10-17T13:39:16.678569Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.678583Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.678595Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.678693Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: append_entries to id=1 vote=1-0:committed, prev_log_id=None, leader_commit=Some(1-0-2), entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] +2023-10-17T13:39:16.678820Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#a{id=0 target=1}: new +2023-10-17T13:39:16.678830Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#a{id=0 target=1}: enter +2023-10-17T13:39:16.678879Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#a{id=0 target=1}: exit +2023-10-17T13:39:16.678888Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#a{id=0 target=1}: close time.busy=49.0µs time.idle=19.6µs +2023-10-17T13:39:16.679002Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#800000000000a{id=0 target=1}: new +2023-10-17T13:39:16.679012Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#800000000000a{id=0 target=1}: enter +2023-10-17T13:39:16.679059Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#800000000000a{id=0 target=1}: exit +2023-10-17T13:39:16.679068Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#800000000000a{id=0 target=1}: close time.busy=47.2µs time.idle=19.5µs +2023-10-17T13:39:16.679179Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#1000000000000a: new +2023-10-17T13:39:16.679233Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#1000000000000a: enter +2023-10-17T13:39:16.679272Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#1000000000000a: exit +2023-10-17T13:39:16.679281Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#1000000000000a: close time.busy=39.3µs time.idle=63.8µs +2023-10-17T13:39:16.679547Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: current quota: None +2023-10-17T13:39:16.679596Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: quota after consumption: None +2023-10-17T13:39:16.679856Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: append_entries truncated: None +2023-10-17T13:39:16.679992Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#1800000000000a{node_id=1}: new +2023-10-17T13:39:16.680002Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#1800000000000a{node_id=1}: enter +2023-10-17T13:39:16.680046Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#1800000000000a{node_id=1}: exit +2023-10-17T13:39:16.680055Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#1800000000000a{node_id=1}: close time.busy=44.2µs time.idle=19.5µs +2023-10-17T13:39:16.680182Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: new +2023-10-17T13:39:16.680249Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: enter +2023-10-17T13:39:16.680437Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: Raft::append_entries rpc=vote=1-0:committed, prev_log_id=None, leader_commit=Some(1-0-2), entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] +2023-10-17T13:39:16.680617Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: new +2023-10-17T13:39:16.680657Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: enter +2023-10-17T13:39:16.680922Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: exit +2023-10-17T13:39:16.680925Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.680932Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: exit +2023-10-17T13:39:16.680935Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.680941Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.680948Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.680973Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: new +2023-10-17T13:39:16.680989Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: enter +2023-10-17T13:39:16.681029Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: recv from rx_api: AppendEntries: vote=1-0:committed, prev_log_id=None, leader_commit=Some(1-0-2), entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] +2023-10-17T13:39:16.681166Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: new +2023-10-17T13:39:16.681184Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: enter +2023-10-17T13:39:16.681394Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: req=vote=1-0:committed, prev_log_id=None, leader_commit=Some(1-0-2), entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] func="openraft::core::raft_core::RaftCore<_, _, _, _>::handle_append_entries_request" +2023-10-17T13:39:16.681721Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: new +2023-10-17T13:39:16.681740Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: enter +2023-10-17T13:39:16.682015Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::engine_impl::Engine<_>::handle_append_entries vote=1-0:committed prev_log_id=None entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] my_vote=0-0:uncommitted my_last_log_id=None +2023-10-17T13:39:16.682069Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: vote is changing to vote=1-0:committed +2023-10-17T13:39:16.682088Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: vote is changing from 0-0:uncommitted to 1-0:committed +2023-10-17T13:39:16.682109Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::vote_handler::VoteHandler<_>::update_vote now=Instant { t: 1043048331719709 } +2023-10-17T13:39:16.682715Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: new +2023-10-17T13:39:16.682735Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: enter +2023-10-17T13:39:16.682816Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: append-entries request prev_log_id=None entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] +2023-10-17T13:39:16.683133Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: local state my_last_log_id=None my_committed=None +2023-10-17T13:39:16.683217Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: prev_log_id matches, skip matching entries committed=None entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] +2023-10-17T13:39:16.683361Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: found nonexistent log id at=0 entry_log_id=0-0-0 +2023-10-17T13:39:16.683639Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@truncate_logs#4000000008{since=0}: new +2023-10-17T13:39:16.683660Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@truncate_logs#4000000008{since=0}: enter +2023-10-17T13:39:16.683929Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@truncate_logs#4000000008{since=0}: truncate_logs since=0 +2023-10-17T13:39:16.683993Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@truncate_logs#4000000008{since=0}: trying to delete absent log at: 0 +2023-10-17T13:39:16.684015Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@truncate_logs#4000000008{since=0}: exit +2023-10-17T13:39:16.684035Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@truncate_logs#4000000008{since=0}: close time.busy=356µs time.idle=40.9µs +2023-10-17T13:39:16.684301Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: new +2023-10-17T13:39:16.684323Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: enter +2023-10-17T13:39:16.684475Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: applying 0-th new membership configs received from leader last={log_id:1-0-2, voters:[{2:{()},3:{()}}], learners:[]} +2023-10-17T13:39:16.684552Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: append: committed: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} } effective: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} } +2023-10-17T13:39:16.684733Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: applying 1-th new membership configs received from leader last={log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]} +2023-10-17T13:39:16.684761Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: append: committed: EffectiveMembership { log_id: None, membership: Membership { configs: [], nodes: {} }, voter_ids: {} } effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }), membership: Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} }, voter_ids: {2, 3} } +2023-10-17T13:39:16.684860Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: updated membership state membership_state=MembershipState{committed: {log_id:Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }) membership:voters:[{2:{()},3:{()}}], learners:[]}, effective: {log_id:Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 11 }) membership:voters:[{4:{()},5:{()}}], learners:[]}} +2023-10-17T13:39:16.684888Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: openraft::raft_state@calc_server_state#4000000009: new +2023-10-17T13:39:16.684910Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: openraft::raft_state@calc_server_state#4000000009: enter +2023-10-17T13:39:16.684932Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: openraft::raft_state@calc_server_state#4000000009: states is_member=false is_leader=false is_leading=false +2023-10-17T13:39:16.684957Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: openraft::raft_state@calc_server_state#4000000009: exit +2023-10-17T13:39:16.684976Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: openraft::raft_state@calc_server_state#4000000009: close time.busy=46.3µs time.idle=42.0µs +2023-10-17T13:39:16.685002Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: update_server_state_if_changed id=1 prev_server_state=Learner server_state=Learner +2023-10-17T13:39:16.685026Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: exit +2023-10-17T13:39:16.685046Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: openraft::engine::handler::following_handler@do_append_entries#8004000000008{since=0}: close time.busy=703µs time.idle=42.0µs +2023-10-17T13:39:16.685073Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: exit +2023-10-17T13:39:16.685091Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: openraft::engine::handler::following_handler@append_entries#4000000007: close time.busy=2.34ms time.idle=41.2µs +2023-10-17T13:39:16.685115Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: exit +2023-10-17T13:39:16.685132Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_append_entries#8004000000006: close time.busy=3.38ms time.idle=35.7µs +2023-10-17T13:39:16.685320Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: new +2023-10-17T13:39:16.685338Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: enter +2023-10-17T13:39:16.685453Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::engine_impl::Engine<_>::handle_commit_entries leader_committed=Some(1-0-2) my_accepted=Some(1-0-11) my_committed=None +2023-10-17T13:39:16.685619Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: new +2023-10-17T13:39:16.685638Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: enter +2023-10-17T13:39:16.685766Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: openraft::engine::handler::following_handler::FollowingHandler<_>::commit_entries leader_committed=1-0-2 accepted=1-0-11 committed=1-0-2 +2023-10-17T13:39:16.685792Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: openraft::raft_state@update_committed#10004000000008: new +2023-10-17T13:39:16.685812Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: openraft::raft_state@update_committed#10004000000008: enter +2023-10-17T13:39:16.685831Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: openraft::raft_state@update_committed#10004000000008: exit +2023-10-17T13:39:16.685850Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: openraft::raft_state@update_committed#10004000000008: close time.busy=20.0µs time.idle=38.4µs +2023-10-17T13:39:16.685875Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: exit +2023-10-17T13:39:16.685893Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: openraft::engine::handler::following_handler@commit_entries#8004000000007: close time.busy=237µs time.idle=37.9µs +2023-10-17T13:39:16.685915Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: exit +2023-10-17T13:39:16.685932Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: openraft::engine::engine_impl@handle_commit_entries#10004000000006: close time.busy=577µs time.idle=35.5µs +2023-10-17T13:39:16.685953Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: exit +2023-10-17T13:39:16.685969Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: openraft::core::raft_core@handle_append_entries_request#38004000000005: close time.busy=4.77ms time.idle=34.0µs +2023-10-17T13:39:16.685989Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: exit +2023-10-17T13:39:16.686005Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#98004000000004{state=Learner id=1}: close time.busy=5.00ms time.idle=32.3µs +2023-10-17T13:39:16.686027Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: new +2023-10-17T13:39:16.686042Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: enter +2023-10-17T13:39:16.686058Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: queued commands: start... +2023-10-17T13:39:16.686073Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: queued commands: SaveVote { vote: Vote { leader_id: LeaderId { term: 1, node_id: 0 }, committed: true } } +2023-10-17T13:39:16.686089Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: queued commands: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 1 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }, payload: membership:Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} } }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 4 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 5 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 6 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 7 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 8 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 9 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 10 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 11 }, payload: membership:Membership { configs: [{4, 5}], nodes: {4: (), 5: ()} } }] } +2023-10-17T13:39:16.686119Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: queued commands: Respond { when: None, resp: AppendEntries(ValueSender { value: Ok(Success), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.686187Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: queued commands: Apply { seq: 1, already_committed: None, upto: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 } } +2023-10-17T13:39:16.686203Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: queued commands: end... +2023-10-17T13:39:16.686217Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: run command: SaveVote { vote: Vote { leader_id: LeaderId { term: 1, node_id: 0 }, committed: true } } +2023-10-17T13:39:16.686234Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: condition: None +2023-10-17T13:39:16.686256Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: save_vote vote=Vote { leader_id: LeaderId { term: 1, node_id: 0 }, committed: true } +2023-10-17T13:39:16.686279Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: run command: AppendInputEntries { entries: [Entry { log_id: LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 1 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }, payload: membership:Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} } }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 4 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 5 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 6 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 7 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 8 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 9 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 10 }, payload: blank }, Entry { log_id: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 11 }, payload: membership:Membership { configs: [{4, 5}], nodes: {4: (), 5: ()} } }] } +2023-10-17T13:39:16.686306Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: condition: None +2023-10-17T13:39:16.686320Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: AppendInputEntries: [0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[],1-0-3:blank,..,1-0-11:membership: voters:[{4:{()},5:{()}}], learners:[]] +2023-10-17T13:39:16.686366Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@append_to_log#40004000000005: new +2023-10-17T13:39:16.686382Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@append_to_log#40004000000005: enter +2023-10-17T13:39:16.686397Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@append_to_log#40004000000005: append_to_log +2023-10-17T13:39:16.686498Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@append_to_log#40004000000005: exit +2023-10-17T13:39:16.686514Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@append_to_log#40004000000005: close time.busy=116µs time.idle=32.9µs +2023-10-17T13:39:16.686577Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: this node is NOT a leader: Learner +2023-10-17T13:39:16.686629Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: id=0 is not in membership, when getting leader id +2023-10-17T13:39:16.686656Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: run command: Respond { when: None, resp: AppendEntries(ValueSender { value: Ok(Success), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.686686Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: condition: None +2023-10-17T13:39:16.686756Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.686770Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: run command: Apply { seq: 1, already_committed: None, upto: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 } } +2023-10-17T13:39:16.686786Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: condition: None +2023-10-17T13:39:16.686805Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: new +2023-10-17T13:39:16.686821Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: enter +2023-10-17T13:39:16.686836Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: apply_to_state_machine upto_index=2 +2023-10-17T13:39:16.686899Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: about to apply entries=[0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[]] +2023-10-17T13:39:16.686931Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: sending command to state machine worker: StateMachineCommand { seq: 1, payload: Apply: [0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[]] } +2023-10-17T13:39:16.686952Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: enter +2023-10-17T13:39:16.686972Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: exit +2023-10-17T13:39:16.686988Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: openraft::core::raft_core@apply_to_state_machine#48004000000005: close time.busy=150µs time.idle=33.4µs +2023-10-17T13:39:16.687009Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: enter +2023-10-17T13:39:16.687009Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: exit +2023-10-17T13:39:16.687023Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#a0004000000004: close time.busy=967µs time.idle=29.5µs +2023-10-17T13:39:16.687044Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.687060Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.687077Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: new +2023-10-17T13:39:16.687092Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: enter +2023-10-17T13:39:16.687109Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: new +2023-10-17T13:39:16.687125Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: enter +2023-10-17T13:39:16.687145Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: openraft::core::raft_core@current_leader#18004000000006: new +2023-10-17T13:39:16.687162Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: openraft::core::raft_core@current_leader#18004000000006: enter +2023-10-17T13:39:16.687175Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: call_core receives result is error: false +2023-10-17T13:39:16.687212Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: exit +2023-10-17T13:39:16.687180Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: openraft::core::raft_core@current_leader#18004000000006: get current_leader self_id=1 vote=1-0:committed +2023-10-17T13:39:16.687352Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: openraft::core::raft_core@current_leader#18004000000006: id=0 is not a voter +2023-10-17T13:39:16.687362Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: openraft::raft@call_core#b: close time.busy=468µs time.idle=6.28ms +2023-10-17T13:39:16.687377Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: openraft::core::raft_core@current_leader#18004000000006: exit +2023-10-17T13:39:16.687395Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: openraft::core::raft_core@current_leader#18004000000006: close time.busy=215µs time.idle=35.0µs +2023-10-17T13:39:16.687426Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: exit +2023-10-17T13:39:16.687437Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: report_metrics: Metrics{id:1, Learner, term:1, vote:1-0:committed, last_log:11, last_applied:None, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.687454Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@append_entries#2000000000000a: close time.busy=1.16ms time.idle=6.11ms +2023-10-17T13:39:16.687461Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: exit +2023-10-17T13:39:16.687482Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: openraft::core::raft_core@report_metrics#50004000000005: close time.busy=336µs time.idle=37.0µs +2023-10-17T13:39:16.687502Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: exit +2023-10-17T13:39:16.687516Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#a8004000000004: close time.busy=410µs time.idle=29.6µs +2023-10-17T13:39:16.687540Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.687548Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: append_entries: recv resp from id=1 Ok(Success) +2023-10-17T13:39:16.687555Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.687567Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.687583Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: enter +2023-10-17T13:39:16.687595Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 1, payload: Apply: [0-0-0:blank,1-0-1:blank,1-0-2:membership: voters:[{2:{()},3:{()}}], learners:[]] } +2023-10-17T13:39:16.687625Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: new +2023-10-17T13:39:16.687637Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: enter +2023-10-17T13:39:16.687655Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: replicate to sm entry.log_id=0-0-0 +2023-10-17T13:39:16.687665Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: replicate to sm entry.log_id=1-0-1 +2023-10-17T13:39:16.687674Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: replicate to sm entry.log_id=1-0-2 +2023-10-17T13:39:16.687692Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: exit +2023-10-17T13:39:16.687706Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@apply#b0004000000004: close time.busy=54.8µs time.idle=25.8µs +2023-10-17T13:39:16.687718Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- check that learner membership is affected log_index=9 +2023-10-17T13:39:16.687725Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: exit +2023-10-17T13:39:16.687735Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.687745Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.687761Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.687785Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: new +2023-10-17T13:39:16.687802Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: enter +2023-10-17T13:39:16.687823Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 1, result: Ok(Apply(ApplyResult { since: 0, end: 3, last_applied: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 } })) } +2023-10-17T13:39:16.687828Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: RaftStorage::get_membership membership_in_sm=StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }), membership: Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} } } membership_in_log=[StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 11 }), membership: Membership { configs: [{4, 5}], nodes: {4: (), 5: ()} } }] +2023-10-17T13:39:16.687841Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: sm::StateMachine command result: CommandResult { command_seq: 1, result: Ok(Apply(ApplyResult { since: 0, end: 3, last_applied: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 } })) } +2023-10-17T13:39:16.687862Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: openraft::raft_state::io_state::IOState<_>::update_applied applied=1-0-2 +2023-10-17T13:39:16.687887Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: openraft::core::raft_core@handle_apply_result#58004000000005: new +2023-10-17T13:39:16.687906Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: openraft::core::raft_core@handle_apply_result#58004000000005: enter +2023-10-17T13:39:16.687924Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: openraft::core::raft_core@handle_apply_result#58004000000005: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_apply_result last_applied=1-0-2 +2023-10-17T13:39:16.687945Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: openraft::core::raft_core@handle_apply_result#58004000000005: exit +2023-10-17T13:39:16.687963Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: openraft::core::raft_core@handle_apply_result#58004000000005: close time.busy=39.4µs time.idle=37.2µs +2023-10-17T13:39:16.688081Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: got membership of node-1: MembershipState { committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }), membership: Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} }, voter_ids: {2, 3} }, effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 11 }), membership: Membership { configs: [{4, 5}], nodes: {4: (), 5: ()} }, voter_ids: {4, 5} } } +2023-10-17T13:39:16.688093Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: exit +2023-10-17T13:39:16.688107Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#b8004000000004{state=Learner id=1}: close time.busy=291µs time.idle=31.6µs +2023-10-17T13:39:16.688132Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#c0004000000004: new +2023-10-17T13:39:16.688149Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#c0004000000004: enter +2023-10-17T13:39:16.688166Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#c0004000000004: queued commands: start... +2023-10-17T13:39:16.688184Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- manually build and install snapshot to node-1 log_index=9 +2023-10-17T13:39:16.688183Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#c0004000000004: queued commands: end... +2023-10-17T13:39:16.688198Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#c0004000000004: exit +2023-10-17T13:39:16.688216Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#c0004000000004: close time.busy=49.7µs time.idle=35.0µs +2023-10-17T13:39:16.688237Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.688253Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.688272Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: new +2023-10-17T13:39:16.688289Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: enter +2023-10-17T13:39:16.688308Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: new +2023-10-17T13:39:16.688326Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: enter +2023-10-17T13:39:16.688355Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: log compaction complete snapshot_size=275 +2023-10-17T13:39:16.688350Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: openraft::core::raft_core@current_leader#20004000000006: new +2023-10-17T13:39:16.688372Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: openraft::core::raft_core@current_leader#20004000000006: enter +2023-10-17T13:39:16.688391Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: openraft::core::raft_core@current_leader#20004000000006: get current_leader self_id=1 vote=1-0:committed +2023-10-17T13:39:16.688414Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: openraft::core::raft_core@current_leader#20004000000006: id=0 is not a voter +2023-10-17T13:39:16.688434Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: openraft::core::raft_core@current_leader#20004000000006: exit +2023-10-17T13:39:16.688452Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: openraft::core::raft_core@current_leader#20004000000006: close time.busy=61.5µs time.idle=41.3µs +2023-10-17T13:39:16.688489Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: report_metrics: Metrics{id:1, Learner, term:1, vote:1-0:committed, last_log:11, last_applied:1-0-2, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.688509Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: exit +2023-10-17T13:39:16.688525Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: openraft::core::raft_core@report_metrics#60004000000005: close time.busy=183µs time.idle=34.6µs +2023-10-17T13:39:16.688544Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: exit +2023-10-17T13:39:16.688559Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#c8004000000004: close time.busy=255µs time.idle=32.0µs +2023-10-17T13:39:16.688580Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.688595Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.688610Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.688782Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#2800000000000a{id=0 target=1}: new +2023-10-17T13:39:16.688792Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#2800000000000a{id=0 target=1}: enter +2023-10-17T13:39:16.688802Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#2800000000000a{id=0 target=1}: exit +2023-10-17T13:39:16.688808Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#2800000000000a{id=0 target=1}: close time.busy=9.12µs time.idle=18.1µs +2023-10-17T13:39:16.688827Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#3000000000000a{id=0 target=1}: new +2023-10-17T13:39:16.688836Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#3000000000000a{id=0 target=1}: enter +2023-10-17T13:39:16.688845Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#3000000000000a{id=0 target=1}: exit +2023-10-17T13:39:16.688853Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#3000000000000a{id=0 target=1}: close time.busy=8.54µs time.idle=16.8µs +2023-10-17T13:39:16.688879Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#3800000000000a: new +2023-10-17T13:39:16.688888Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#3800000000000a: enter +2023-10-17T13:39:16.688896Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#3800000000000a: exit +2023-10-17T13:39:16.688904Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#3800000000000a: close time.busy=8.29µs time.idle=17.6µs +2023-10-17T13:39:16.688923Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#4000000000000a{node_id=1}: new +2023-10-17T13:39:16.688931Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#4000000000000a{node_id=1}: enter +2023-10-17T13:39:16.688940Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#4000000000000a{node_id=1}: exit +2023-10-17T13:39:16.688948Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#4000000000000a{node_id=1}: close time.busy=8.83µs time.idle=17.2µs +2023-10-17T13:39:16.689155Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: new +2023-10-17T13:39:16.689193Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: enter +2023-10-17T13:39:16.689333Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: Raft::install_snapshot() rpc=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.689585Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: new +2023-10-17T13:39:16.689626Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: enter +2023-10-17T13:39:16.689852Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.689861Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: exit +2023-10-17T13:39:16.689865Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.689870Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: exit +2023-10-17T13:39:16.689878Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.689877Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.689900Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: new +2023-10-17T13:39:16.689915Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: enter +2023-10-17T13:39:16.689940Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: recv from rx_api: InstallSnapshot: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.690158Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: received RaftMst::InstallSnapshot: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_api_msg req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.690382Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: new +2023-10-17T13:39:16.690400Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: enter +2023-10-17T13:39:16.690535Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_install_snapshot_request req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.690686Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: new +2023-10-17T13:39:16.690704Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: enter +2023-10-17T13:39:16.690829Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl::Engine<_>::handle_install_snapshot req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.690965Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: new +2023-10-17T13:39:16.690984Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: enter +2023-10-17T13:39:16.691007Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: vote is changing to vote=5-0:committed +2023-10-17T13:39:16.691027Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: vote is changing from 1-0:committed to 5-0:committed +2023-10-17T13:39:16.691047Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: openraft::engine::handler::vote_handler::VoteHandler<_>::update_vote now=Instant { t: 1043048340657500 } +2023-10-17T13:39:16.691088Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: exit +2023-10-17T13:39:16.691107Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::handler::vote_handler@accept_vote#10004000000007: close time.busy=104µs time.idle=38.3µs +2023-10-17T13:39:16.691288Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: new +2023-10-17T13:39:16.691307Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: enter +2023-10-17T13:39:16.691407Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::engine_impl::Engine<_>::install_snapshot req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.691632Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#18004000000008: new +2023-10-17T13:39:16.691652Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#18004000000008: enter +2023-10-17T13:39:16.691757Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#18004000000008: openraft::engine::handler::following_handler::FollowingHandler<_>::receive_snapshot_chunk req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.691894Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#18004000000008: next_seq: 2 +2023-10-17T13:39:16.691920Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#18004000000008: exit +2023-10-17T13:39:16.691939Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#18004000000008: close time.busy=268µs time.idle=39.4µs +2023-10-17T13:39:16.691964Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: exit +2023-10-17T13:39:16.691982Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: openraft::engine::engine_impl@install_snapshot#18004000000007: close time.busy=657µs time.idle=37.9µs +2023-10-17T13:39:16.692006Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: exit +2023-10-17T13:39:16.692023Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: openraft::engine::engine_impl@handle_install_snapshot#28004000000006: close time.busy=1.30ms time.idle=35.5µs +2023-10-17T13:39:16.692065Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: exit +2023-10-17T13:39:16.692081Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#68004000000005: close time.busy=1.66ms time.idle=36.9µs +2023-10-17T13:39:16.692107Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: exit +2023-10-17T13:39:16.692122Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#d0004000000004{state=Learner id=1}: close time.busy=2.19ms time.idle=30.6µs +2023-10-17T13:39:16.692143Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: new +2023-10-17T13:39:16.692158Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: enter +2023-10-17T13:39:16.692171Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: queued commands: start... +2023-10-17T13:39:16.692186Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: queued commands: SaveVote { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } } +2023-10-17T13:39:16.692201Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: queued commands: StateMachine { command: StateMachineCommand { seq: 2, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } } +2023-10-17T13:39:16.692247Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: queued commands: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.692285Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: queued commands: end... +2023-10-17T13:39:16.692299Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: run command: SaveVote { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } } +2023-10-17T13:39:16.692314Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: condition: None +2023-10-17T13:39:16.692332Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: save_vote vote=Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } +2023-10-17T13:39:16.692352Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: run command: StateMachine { command: StateMachineCommand { seq: 2, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } } +2023-10-17T13:39:16.692376Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: condition: None +2023-10-17T13:39:16.692391Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: sending command to state machine worker: StateMachineCommand { seq: 2, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } +2023-10-17T13:39:16.692417Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: run command: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.692436Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: condition: Some(StateMachineCommand { command_seq: 2 }) +2023-10-17T13:39:16.692658Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: sm::Command(2) has not yet finished(1), postpone cmd: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.692799Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: early return: postpone command: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.692866Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: postpone command: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.693075Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: postponed, first 8 queued commands: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.693113Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: exit +2023-10-17T13:39:16.693128Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#d8004000000004: close time.busy=956µs time.idle=29.9µs +2023-10-17T13:39:16.693148Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.693162Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.693179Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: new +2023-10-17T13:39:16.693193Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: enter +2023-10-17T13:39:16.693210Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: new +2023-10-17T13:39:16.693226Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: enter +2023-10-17T13:39:16.693246Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: openraft::core::raft_core@current_leader#30004000000006: new +2023-10-17T13:39:16.693263Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: openraft::core::raft_core@current_leader#30004000000006: enter +2023-10-17T13:39:16.693279Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: openraft::core::raft_core@current_leader#30004000000006: get current_leader self_id=1 vote=5-0:committed +2023-10-17T13:39:16.693300Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: openraft::core::raft_core@current_leader#30004000000006: id=0 is not a voter +2023-10-17T13:39:16.693318Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: openraft::core::raft_core@current_leader#30004000000006: exit +2023-10-17T13:39:16.693335Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: openraft::core::raft_core@current_leader#30004000000006: close time.busy=55.1µs time.idle=33.6µs +2023-10-17T13:39:16.693374Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: report_metrics: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:11, last_applied:1-0-2, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.693393Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: exit +2023-10-17T13:39:16.693409Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: openraft::core::raft_core@report_metrics#70004000000005: close time.busy=168µs time.idle=31.2µs +2023-10-17T13:39:16.693428Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: exit +2023-10-17T13:39:16.693443Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#e0004000000004: close time.busy=235µs time.idle=28.9µs +2023-10-17T13:39:16.693464Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.693478Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.693490Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.693503Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: enter +2023-10-17T13:39:16.693512Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 2, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } +2023-10-17T13:39:16.693701Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: received snapshot chunk req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.693931Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#e8004000000004: new +2023-10-17T13:39:16.693970Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#e8004000000004: enter +2023-10-17T13:39:16.694203Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#e8004000000004: openraft::core::sm::Worker<_, _>::receive_snapshot_chunk req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} snapshot_req_id=SnapshotRequestId { leader_id: LeaderId { term: 5, node_id: 0 }, snapshot_id: "5-0-9-2", chunk_id: None } +2023-10-17T13:39:16.694616Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#e8004000000004: exit +2023-10-17T13:39:16.694644Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#e8004000000004: close time.busy=645µs time.idle=67.8µs +2023-10-17T13:39:16.694752Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: exit +2023-10-17T13:39:16.694762Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.694772Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.694784Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.694806Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: new +2023-10-17T13:39:16.694821Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: enter +2023-10-17T13:39:16.694852Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 2, result: Ok(ReceiveSnapshotChunk(None)) } +2023-10-17T13:39:16.694867Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: sm::StateMachine command result: CommandResult { command_seq: 2, result: Ok(ReceiveSnapshotChunk(None)) } +2023-10-17T13:39:16.695050Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: sm::StateMachine command done: ReceiveSnapshotChunk: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_notify +2023-10-17T13:39:16.695088Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: exit +2023-10-17T13:39:16.695103Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#f0004000000004{state=Learner id=1}: close time.busy=267µs time.idle=30.4µs +2023-10-17T13:39:16.695124Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: new +2023-10-17T13:39:16.695140Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: enter +2023-10-17T13:39:16.695154Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: queued commands: start... +2023-10-17T13:39:16.695168Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: queued commands: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.695186Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: queued commands: end... +2023-10-17T13:39:16.695199Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: run command: Respond { when: Some(StateMachineCommand { command_seq: 2 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.695216Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: condition: Some(StateMachineCommand { command_seq: 2 }) +2023-10-17T13:39:16.695263Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: exit +2023-10-17T13:39:16.695272Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.695278Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#f8004000000004: close time.busy=123µs time.idle=30.8µs +2023-10-17T13:39:16.695296Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.695310Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.695327Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: new +2023-10-17T13:39:16.695342Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: enter +2023-10-17T13:39:16.695357Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: new +2023-10-17T13:39:16.695373Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: enter +2023-10-17T13:39:16.695396Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: enter +2023-10-17T13:39:16.695393Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: openraft::core::raft_core@current_leader#38004000000006: new +2023-10-17T13:39:16.695408Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: openraft::core::raft_core@current_leader#38004000000006: enter +2023-10-17T13:39:16.695425Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: openraft::core::raft_core@current_leader#38004000000006: get current_leader self_id=1 vote=5-0:committed +2023-10-17T13:39:16.695436Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: enter +2023-10-17T13:39:16.695444Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: openraft::core::raft_core@current_leader#38004000000006: id=0 is not a voter +2023-10-17T13:39:16.695461Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: openraft::core::raft_core@current_leader#38004000000006: exit +2023-10-17T13:39:16.695482Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: openraft::core::raft_core@current_leader#38004000000006: close time.busy=53.2µs time.idle=35.9µs +2023-10-17T13:39:16.695557Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: report_metrics: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:11, last_applied:1-0-2, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.695601Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: exit +2023-10-17T13:39:16.695622Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: openraft::core::raft_core@report_metrics#78004000000005: close time.busy=227µs time.idle=37.3µs +2023-10-17T13:39:16.695653Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: exit +2023-10-17T13:39:16.695670Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#100004000000004: close time.busy=311µs time.idle=32.5µs +2023-10-17T13:39:16.695699Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.695716Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.695725Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: call_core receives result is error: false +2023-10-17T13:39:16.695733Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.695754Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: exit +2023-10-17T13:39:16.695797Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: openraft::raft@call_core#800000000000b: close time.busy=553µs time.idle=5.66ms +2023-10-17T13:39:16.695840Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: exit +2023-10-17T13:39:16.695869Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#4800000000000a: close time.busy=1.12ms time.idle=5.60ms +2023-10-17T13:39:16.696210Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#5000000000000a{id=0 target=1}: new +2023-10-17T13:39:16.696220Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#5000000000000a{id=0 target=1}: enter +2023-10-17T13:39:16.696229Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#5000000000000a{id=0 target=1}: exit +2023-10-17T13:39:16.696238Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_network_error#5000000000000a{id=0 target=1}: close time.busy=9.50µs time.idle=18.4µs +2023-10-17T13:39:16.696255Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#5800000000000a{id=0 target=1}: new +2023-10-17T13:39:16.696264Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#5800000000000a{id=0 target=1}: enter +2023-10-17T13:39:16.696272Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#5800000000000a{id=0 target=1}: exit +2023-10-17T13:39:16.696279Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@check_unreachable#5800000000000a{id=0 target=1}: close time.busy=8.08µs time.idle=15.7µs +2023-10-17T13:39:16.696294Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#6000000000000a: new +2023-10-17T13:39:16.696302Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#6000000000000a: enter +2023-10-17T13:39:16.696310Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#6000000000000a: exit +2023-10-17T13:39:16.696317Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@rand_send_delay#6000000000000a: close time.busy=7.46µs time.idle=16.3µs +2023-10-17T13:39:16.696332Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#6800000000000a{node_id=1}: new +2023-10-17T13:39:16.696341Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#6800000000000a{node_id=1}: enter +2023-10-17T13:39:16.696349Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#6800000000000a{node_id=1}: exit +2023-10-17T13:39:16.696357Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@get_raft_handle#6800000000000a{node_id=1}: close time.busy=8.25µs time.idle=16.7µs +2023-10-17T13:39:16.696372Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: new +2023-10-17T13:39:16.696381Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: enter +2023-10-17T13:39:16.696398Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: Raft::install_snapshot() rpc=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696412Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: new +2023-10-17T13:39:16.696422Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: enter +2023-10-17T13:39:16.696448Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: exit +2023-10-17T13:39:16.696458Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: exit +2023-10-17T13:39:16.696457Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.696465Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.696468Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.696479Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.696502Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: new +2023-10-17T13:39:16.696517Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: enter +2023-10-17T13:39:16.696542Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: recv from rx_api: InstallSnapshot: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696568Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: received RaftMst::InstallSnapshot: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_api_msg req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696588Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: new +2023-10-17T13:39:16.696604Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: enter +2023-10-17T13:39:16.696629Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_install_snapshot_request req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696650Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: new +2023-10-17T13:39:16.696667Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: enter +2023-10-17T13:39:16.696693Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl::Engine<_>::handle_install_snapshot req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696716Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::handler::vote_handler@accept_vote#20004000000007: new +2023-10-17T13:39:16.696736Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::handler::vote_handler@accept_vote#20004000000007: enter +2023-10-17T13:39:16.696753Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::handler::vote_handler@accept_vote#20004000000007: vote is changing to vote=5-0:committed +2023-10-17T13:39:16.696776Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::handler::vote_handler@accept_vote#20004000000007: openraft::engine::handler::vote_handler::VoteHandler<_>::update_vote now=Instant { t: 1043048346386000 } +2023-10-17T13:39:16.696796Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::handler::vote_handler@accept_vote#20004000000007: exit +2023-10-17T13:39:16.696814Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::handler::vote_handler@accept_vote#20004000000007: close time.busy=60.6µs time.idle=37.8µs +2023-10-17T13:39:16.696839Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: new +2023-10-17T13:39:16.696857Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: enter +2023-10-17T13:39:16.696884Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::engine_impl::Engine<_>::install_snapshot req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696908Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#20004000000008: new +2023-10-17T13:39:16.696928Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#20004000000008: enter +2023-10-17T13:39:16.696956Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#20004000000008: openraft::engine::handler::following_handler::FollowingHandler<_>::receive_snapshot_chunk req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.696978Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#20004000000008: next_seq: 3 +2023-10-17T13:39:16.696998Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#20004000000008: exit +2023-10-17T13:39:16.697017Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: openraft::engine::handler::following_handler@receive_snapshot_chunk#20004000000008: close time.busy=69.7µs time.idle=39.4µs +2023-10-17T13:39:16.697041Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: exit +2023-10-17T13:39:16.697059Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: openraft::engine::engine_impl@install_snapshot#28004000000007: close time.busy=183µs time.idle=36.7µs +2023-10-17T13:39:16.697082Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: exit +2023-10-17T13:39:16.697099Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: openraft::engine::engine_impl@handle_install_snapshot#40004000000006: close time.busy=414µs time.idle=34.7µs +2023-10-17T13:39:16.697120Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: exit +2023-10-17T13:39:16.697136Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: openraft::core::raft_core@handle_install_snapshot_request#80004000000005: close time.busy=515µs time.idle=32.6µs +2023-10-17T13:39:16.697156Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: exit +2023-10-17T13:39:16.697170Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_api_msg#108004000000004{state=Learner id=1}: close time.busy=638µs time.idle=30.1µs +2023-10-17T13:39:16.697191Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: new +2023-10-17T13:39:16.697206Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: enter +2023-10-17T13:39:16.697220Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: queued commands: start... +2023-10-17T13:39:16.697233Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: queued commands: StateMachine { command: StateMachineCommand { seq: 3, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } } +2023-10-17T13:39:16.697257Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: queued commands: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.697274Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: queued commands: end... +2023-10-17T13:39:16.697287Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: run command: StateMachine { command: StateMachineCommand { seq: 3, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } } +2023-10-17T13:39:16.697311Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: condition: None +2023-10-17T13:39:16.697325Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: sending command to state machine worker: StateMachineCommand { seq: 3, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } +2023-10-17T13:39:16.697351Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: run command: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.697368Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: condition: Some(StateMachineCommand { command_seq: 3 }) +2023-10-17T13:39:16.697383Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: sm::Command(3) has not yet finished(2), postpone cmd: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.697400Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: early return: postpone command: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.697417Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: postpone command: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.697435Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: postponed, first 8 queued commands: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.697454Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: exit +2023-10-17T13:39:16.697468Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#110004000000004: close time.busy=247µs time.idle=29.9µs +2023-10-17T13:39:16.697486Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.697500Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.697516Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: new +2023-10-17T13:39:16.697530Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: enter +2023-10-17T13:39:16.697546Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: new +2023-10-17T13:39:16.697561Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: enter +2023-10-17T13:39:16.697580Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: openraft::core::raft_core@current_leader#48004000000006: new +2023-10-17T13:39:16.697597Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: openraft::core::raft_core@current_leader#48004000000006: enter +2023-10-17T13:39:16.697614Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: openraft::core::raft_core@current_leader#48004000000006: get current_leader self_id=1 vote=5-0:committed +2023-10-17T13:39:16.697633Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: openraft::core::raft_core@current_leader#48004000000006: id=0 is not a voter +2023-10-17T13:39:16.697651Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: openraft::core::raft_core@current_leader#48004000000006: exit +2023-10-17T13:39:16.697667Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: openraft::core::raft_core@current_leader#48004000000006: close time.busy=53.2µs time.idle=33.9µs +2023-10-17T13:39:16.697704Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: report_metrics: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:11, last_applied:1-0-2, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.697723Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: exit +2023-10-17T13:39:16.697739Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: openraft::core::raft_core@report_metrics#88004000000005: close time.busy=162µs time.idle=31.0µs +2023-10-17T13:39:16.697758Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: exit +2023-10-17T13:39:16.697772Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#118004000000004: close time.busy=228µs time.idle=28.5µs +2023-10-17T13:39:16.697792Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.697806Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.697817Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.697828Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: enter +2023-10-17T13:39:16.697835Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 3, payload: ReceiveSnapshotChunk: vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} } +2023-10-17T13:39:16.697863Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: received snapshot chunk req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.697874Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#120004000000004: new +2023-10-17T13:39:16.697883Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#120004000000004: enter +2023-10-17T13:39:16.697901Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#120004000000004: openraft::core::sm::Worker<_, _>::receive_snapshot_chunk req=vote=5-0:committed, meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} snapshot_req_id=SnapshotRequestId { leader_id: LeaderId { term: 5, node_id: 0 }, snapshot_id: "5-0-9-2", chunk_id: Some(ExampleChunkId { offset: 0, len: 275 }) } +2023-10-17T13:39:16.698227Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#120004000000004: store completed streaming snapshot: SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } +2023-10-17T13:39:16.698313Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#120004000000004: exit +2023-10-17T13:39:16.698322Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@receive_snapshot_chunk#120004000000004: close time.busy=430µs time.idle=17.4µs +2023-10-17T13:39:16.698339Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: exit +2023-10-17T13:39:16.698348Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.698357Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.698369Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.698388Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: new +2023-10-17T13:39:16.698404Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: enter +2023-10-17T13:39:16.698424Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 3, result: Ok(ReceiveSnapshotChunk(Some(SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" }))) } +2023-10-17T13:39:16.698440Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: sm::StateMachine command result: CommandResult { command_seq: 3, result: Ok(ReceiveSnapshotChunk(Some(SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" }))) } +2023-10-17T13:39:16.698460Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: sm::StateMachine command done: ReceiveSnapshotChunk: openraft::core::raft_core::RaftCore<_, _, _, _>::handle_notify +2023-10-17T13:39:16.698604Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: new +2023-10-17T13:39:16.698621Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: enter +2023-10-17T13:39:16.698855Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: install_snapshot: meta:SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } +2023-10-17T13:39:16.698886Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::snapshot_handler@update_snapshot#50004000000006: new +2023-10-17T13:39:16.698903Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::snapshot_handler@update_snapshot#50004000000006: enter +2023-10-17T13:39:16.698919Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::snapshot_handler@update_snapshot#50004000000006: update_snapshot: SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } +2023-10-17T13:39:16.698941Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::snapshot_handler@update_snapshot#50004000000006: exit +2023-10-17T13:39:16.698957Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::snapshot_handler@update_snapshot#50004000000006: close time.busy=37.4µs time.idle=33.8µs +2023-10-17T13:39:16.698984Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: new +2023-10-17T13:39:16.699001Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: enter +2023-10-17T13:39:16.699017Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: truncate_logs since=3 +2023-10-17T13:39:16.699117Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: effective membership is in conflicting logs, revert it to last committed effective={log_id:Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 11 }) membership:voters:[{4:{()},5:{()}}], learners:[]} committed={log_id:Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }) membership:voters:[{2:{()},3:{()}}], learners:[]} +2023-10-17T13:39:16.699148Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: openraft::raft_state@calc_server_state#30004000000007: new +2023-10-17T13:39:16.699166Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: openraft::raft_state@calc_server_state#30004000000007: enter +2023-10-17T13:39:16.699183Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: openraft::raft_state@calc_server_state#30004000000007: states is_member=false is_leader=false is_leading=false +2023-10-17T13:39:16.699205Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: openraft::raft_state@calc_server_state#30004000000007: exit +2023-10-17T13:39:16.699222Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: openraft::raft_state@calc_server_state#30004000000007: close time.busy=38.6µs time.idle=36.1µs +2023-10-17T13:39:16.699243Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: update_server_state_if_changed id=1 prev_server_state=Learner server_state=Learner +2023-10-17T13:39:16.699265Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: exit +2023-10-17T13:39:16.699281Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@truncate_logs#58004000000006{since=3}: close time.busy=263µs time.idle=34.3µs +2023-10-17T13:39:16.699425Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: new +2023-10-17T13:39:16.699443Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: enter +2023-10-17T13:39:16.699590Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: update committed membership: {log_id:Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }) membership:voters:[{0:{()}}], learners:[]} +2023-10-17T13:39:16.699614Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: openraft::raft_state@calc_server_state#38004000000007: new +2023-10-17T13:39:16.699632Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: openraft::raft_state@calc_server_state#38004000000007: enter +2023-10-17T13:39:16.699648Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: openraft::raft_state@calc_server_state#38004000000007: states is_member=false is_leader=false is_leading=false +2023-10-17T13:39:16.699669Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: openraft::raft_state@calc_server_state#38004000000007: exit +2023-10-17T13:39:16.699686Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: openraft::raft_state@calc_server_state#38004000000007: close time.busy=36.8µs time.idle=35.2µs +2023-10-17T13:39:16.699706Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: update_server_state_if_changed id=1 prev_server_state=Learner server_state=Learner +2023-10-17T13:39:16.699726Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: exit +2023-10-17T13:39:16.699742Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::following_handler@update_committed_membership#60004000000006: close time.busy=283µs time.idle=34.8µs +2023-10-17T13:39:16.699773Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: next_seq: 4 +2023-10-17T13:39:16.699792Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: new +2023-10-17T13:39:16.699809Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: enter +2023-10-17T13:39:16.699826Z INFO ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: purge_log last_purged_log_id=None purge_upto=Some(5-0-9) +2023-10-17T13:39:16.699847Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: openraft::raft_state@purge_log#40004000000007: new +2023-10-17T13:39:16.699865Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: openraft::raft_state@purge_log#40004000000007: enter +2023-10-17T13:39:16.699883Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: openraft::raft_state@purge_log#40004000000007: exit +2023-10-17T13:39:16.699900Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: openraft::raft_state@purge_log#40004000000007: close time.busy=18.5µs time.idle=34.8µs +2023-10-17T13:39:16.699922Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: exit +2023-10-17T13:39:16.699939Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: openraft::engine::handler::log_handler@purge_log#68004000000006: close time.busy=113µs time.idle=33.5µs +2023-10-17T13:39:16.699959Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: exit +2023-10-17T13:39:16.699974Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: openraft::engine::handler::following_handler@install_snapshot#90004000000005: close time.busy=1.34ms time.idle=32.1µs +2023-10-17T13:39:16.700022Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: exit +2023-10-17T13:39:16.700036Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#128004000000004{state=Learner id=1}: close time.busy=1.62ms time.idle=30.4µs +2023-10-17T13:39:16.700056Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: new +2023-10-17T13:39:16.700070Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: enter +2023-10-17T13:39:16.700083Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: queued commands: start... +2023-10-17T13:39:16.700096Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: queued commands: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.700113Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: queued commands: DeleteConflictLog { since: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 } } +2023-10-17T13:39:16.700145Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: queued commands: StateMachine { command: StateMachineCommand { seq: 4, payload: FinalizeSnapshot: install:true SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } } } +2023-10-17T13:39:16.700185Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: queued commands: PurgeLog { upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } } +2023-10-17T13:39:16.700199Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: queued commands: end... +2023-10-17T13:39:16.700212Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: run command: Respond { when: Some(StateMachineCommand { command_seq: 3 }), resp: InstallSnapshot(ValueSender { value: Ok(InstallSnapshotResponse { vote: Vote { leader_id: LeaderId { term: 5, node_id: 0 }, committed: true } }), tx: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }) } +2023-10-17T13:39:16.700228Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: condition: Some(StateMachineCommand { command_seq: 3 }) +2023-10-17T13:39:16.700254Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.700247Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: run command: DeleteConflictLog { since: LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 } } +2023-10-17T13:39:16.700288Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: condition: None +2023-10-17T13:39:16.700308Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: enter +2023-10-17T13:39:16.700315Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: enter +2023-10-17T13:39:16.700326Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: call_core receives result is error: false +2023-10-17T13:39:16.700338Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: exit +2023-10-17T13:39:16.700347Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: openraft::raft@call_core#1000000000000b: close time.busy=48.7µs time.idle=3.89ms +2023-10-17T13:39:16.700362Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: exit +2023-10-17T13:39:16.700370Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: openraft::raft@install_snapshot#7000000000000a: close time.busy=131µs time.idle=3.87ms +2023-10-17T13:39:16.700492Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- DONE installing snapshot log_index=9 +2023-10-17T13:39:16.700742Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=1 wait node-1 snapshot .snapshot -> 5-0-9 latest: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:11, last_applied:1-0-2, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.700752Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999971375s +2023-10-17T13:39:16.700792Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=1 wait node-1 snapshot .snapshot -> 5-0-9 latest: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:11, last_applied:1-0-2, leader:None, membership:{log_id:1-0-11, voters:[{4:{()},5:{()}}], learners:[]}, snapshot:None, purged:None, replication:{}} +2023-10-17T13:39:16.700800Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: wait timeout sleep_time=4.999921916s +2023-10-17T13:39:16.700813Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.700806Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@delete_conflict_logs_since#98004000000005{log_id=LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }}: new +2023-10-17T13:39:16.700843Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@delete_conflict_logs_since#98004000000005{log_id=LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }}: enter +2023-10-17T13:39:16.701077Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@delete_conflict_logs_since#98004000000005{log_id=LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }}: delete_log: [LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }, +oo) +2023-10-17T13:39:16.701186Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@delete_conflict_logs_since#98004000000005{log_id=LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }}: exit +2023-10-17T13:39:16.701222Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@delete_conflict_logs_since#98004000000005{log_id=LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 3 }}: close time.busy=342µs time.idle=74.0µs +2023-10-17T13:39:16.701307Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: run command: StateMachine { command: StateMachineCommand { seq: 4, payload: FinalizeSnapshot: install:true SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } } } +2023-10-17T13:39:16.701327Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: condition: None +2023-10-17T13:39:16.701341Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: sending command to state machine worker: StateMachineCommand { seq: 4, payload: FinalizeSnapshot: install:true SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } } +2023-10-17T13:39:16.701360Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: run command: PurgeLog { upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } } +2023-10-17T13:39:16.701375Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: condition: None +2023-10-17T13:39:16.701393Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@purge_logs_upto#a0004000000005: new +2023-10-17T13:39:16.701409Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@purge_logs_upto#a0004000000005: enter +2023-10-17T13:39:16.701424Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@purge_logs_upto#a0004000000005: purge_log_upto: LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 } +2023-10-17T13:39:16.701450Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@purge_logs_upto#a0004000000005: exit +2023-10-17T13:39:16.701465Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: openraft_memstore@purge_logs_upto#a0004000000005: close time.busy=40.8µs time.idle=31.3µs +2023-10-17T13:39:16.701486Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: exit +2023-10-17T13:39:16.701501Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#130004000000004: close time.busy=1.42ms time.idle=28.8µs +2023-10-17T13:39:16.701519Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.701532Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.701548Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: new +2023-10-17T13:39:16.701562Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: enter +2023-10-17T13:39:16.701577Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: new +2023-10-17T13:39:16.701593Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: enter +2023-10-17T13:39:16.701612Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: openraft::core::raft_core@current_leader#70004000000006: new +2023-10-17T13:39:16.701628Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: openraft::core::raft_core@current_leader#70004000000006: enter +2023-10-17T13:39:16.701644Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: openraft::core::raft_core@current_leader#70004000000006: get current_leader self_id=1 vote=5-0:committed +2023-10-17T13:39:16.701663Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: openraft::core::raft_core@current_leader#70004000000006: id=0 is not a voter +2023-10-17T13:39:16.701680Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: openraft::core::raft_core@current_leader#70004000000006: exit +2023-10-17T13:39:16.701696Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: openraft::core::raft_core@current_leader#70004000000006: close time.busy=51.7µs time.idle=32.4µs +2023-10-17T13:39:16.701733Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: report_metrics: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:9, last_applied:1-0-2, leader:None, membership:{log_id:1-0-2, voters:[{2:{()},3:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{}} +2023-10-17T13:39:16.701765Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: enter +2023-10-17T13:39:16.701756Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: exit +2023-10-17T13:39:16.701776Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: openraft::core::raft_core@report_metrics#a8004000000005: close time.busy=163µs time.idle=36.1µs +2023-10-17T13:39:16.701796Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: exit +2023-10-17T13:39:16.701810Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#138004000000004: close time.busy=234µs time.idle=28.3µs +2023-10-17T13:39:16.701824Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=1 wait node-1 snapshot .snapshot -> 5-0-9 latest: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:9, last_applied:1-0-2, leader:None, membership:{log_id:1-0-2, voters:[{2:{()},3:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{}} +2023-10-17T13:39:16.701832Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.701848Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: id=1 done wait node-1 snapshot .snapshot -> 5-0-9 latest: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:9, last_applied:1-0-2, leader:None, membership:{log_id:1-0-2, voters:[{2:{()},3:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{}} +2023-10-17T13:39:16.701846Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.701859Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.701869Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: enter +2023-10-17T13:39:16.701877Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: received command: StateMachineCommand { seq: 4, payload: FinalizeSnapshot: install:true SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } } +2023-10-17T13:39:16.702003Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: --- check that learner membership is affected, conflict log are deleted log_index=9 +2023-10-17T13:39:16.702042Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: RaftStorage::get_membership membership_in_sm=StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }), membership: Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} } } membership_in_log=[] +2023-10-17T13:39:16.702248Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm::Worker<_, _>::worker_loop: finalize and install snapshot install=true snapshot_meta={snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}} +2023-10-17T13:39:16.702377Z INFO ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: got membership of node-1: MembershipState { committed: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }), membership: Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} }, voter_ids: {2, 3} }, effective: EffectiveMembership { log_id: Some(LogId { leader_id: LeaderId { term: 1, node_id: 0 }, index: 2 }), membership: Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} }, voter_ids: {2, 3} } } +2023-10-17T13:39:16.702547Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: new +2023-10-17T13:39:16.702586Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: enter +2023-10-17T13:39:16.702858Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: installing snapshot: SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } +2023-10-17T13:39:16.702874Z ERROR ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: panicked at 'assertion failed: `(left == right)` + left: `Membership { configs: [{0}], nodes: {0: ()} }`, + right: `Membership { configs: [{2, 3}], nodes: {2: (), 3: ()} }`: membership should be overridden by the snapshot', tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs:194:9 backtrace=backtrace is disabled without --features 'bt' panic.file="tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs" panic.line=194 panic.column=9 +2023-10-17T13:39:16.703506Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: decoding snapshot for installation snapshot_size=275 +2023-10-17T13:39:16.703788Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: SNAP META:SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } +2023-10-17T13:39:16.703906Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: JSON SNAP DATA:{"last_applied_log":{"leader_id":{"term":5,"node_id":0},"index":9},"last_membership":{"log_id":{"leader_id":{"term":0,"node_id":0},"index":0},"membership":{"configs":[[0]],"nodes":{"0":null}}},"client_serial_responses":{"0":[7,"request-6"]},"client_status":{"0":"request-7"}} +2023-10-17T13:39:16.704831Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: Done install_snapshot, meta: SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" } +2023-10-17T13:39:16.704877Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: exit +2023-10-17T13:39:16.704928Z INFO ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: openraft::core::sm@install_snapshot#140004000000004: close time.busy=2.29ms time.idle=91.4µs +2023-10-17T13:39:16.705059Z DEBUG ThreadId(05) openraft::core::sm@worker_loop#d800c000000001: exit +2023-10-17T13:39:16.705065Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: enter +2023-10-17T13:39:16.705076Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: enter +2023-10-17T13:39:16.705088Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: enter +2023-10-17T13:39:16.705114Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: new +2023-10-17T13:39:16.705129Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: enter +2023-10-17T13:39:16.705169Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: recv from rx_notify: StateMachine command done: CommandResult { command_seq: 4, result: Ok(InstallSnapshot(Some(SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" }))) } +2023-10-17T13:39:16.705184Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: sm::StateMachine command result: CommandResult { command_seq: 4, result: Ok(InstallSnapshot(Some(SnapshotMeta { last_log_id: Some(LogId { leader_id: LeaderId { term: 5, node_id: 0 }, index: 9 }), last_membership: StoredMembership { log_id: Some(LogId { leader_id: LeaderId { term: 0, node_id: 0 }, index: 0 }), membership: Membership { configs: [{0}], nodes: {0: ()} } }, snapshot_id: "5-0-9-2" }))) } +2023-10-17T13:39:16.705463Z INFO ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: sm::StateMachine command done: InstallSnapshot: Some({snapshot_id: 5-0-9-2, last_log:5-0-9, last_membership: {log_id:0-0-0, voters:[{0:{()}}], learners:[]}}): openraft::core::raft_core::RaftCore<_, _, _, _>::handle_notify +2023-10-17T13:39:16.705497Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: openraft::raft_state::io_state::IOState<_>::update_applied applied=5-0-9 +2023-10-17T13:39:16.705522Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: exit +2023-10-17T13:39:16.705537Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@handle_notify#e000c000000002{state=Learner id=1}: close time.busy=393µs time.idle=30.7µs +2023-10-17T13:39:16.705559Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#e800c000000002: new +2023-10-17T13:39:16.705576Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#e800c000000002: enter +2023-10-17T13:39:16.705591Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#e800c000000002: queued commands: start... +2023-10-17T13:39:16.705605Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#e800c000000002: queued commands: end... +2023-10-17T13:39:16.705621Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#e800c000000002: exit +2023-10-17T13:39:16.705635Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@run_engine_commands#e800c000000002: close time.busy=45.2µs time.idle=30.8µs +2023-10-17T13:39:16.705654Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all RaftMsg are processed, wait for more +2023-10-17T13:39:16.705668Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: all Notify are processed, wait for more +2023-10-17T13:39:16.705684Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: new +2023-10-17T13:39:16.705698Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: enter +2023-10-17T13:39:16.705715Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: new +2023-10-17T13:39:16.705730Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: enter +2023-10-17T13:39:16.705751Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: openraft::core::raft_core@current_leader#5800c000000004: new +2023-10-17T13:39:16.705768Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: openraft::core::raft_core@current_leader#5800c000000004: enter +2023-10-17T13:39:16.705784Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: openraft::core::raft_core@current_leader#5800c000000004: get current_leader self_id=1 vote=5-0:committed +2023-10-17T13:39:16.705804Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: openraft::core::raft_core@current_leader#5800c000000004: id=0 is not a voter +2023-10-17T13:39:16.705821Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: openraft::core::raft_core@current_leader#5800c000000004: exit +2023-10-17T13:39:16.705838Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: openraft::core::raft_core@current_leader#5800c000000004: close time.busy=53.7µs time.idle=33.4µs +2023-10-17T13:39:16.705875Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: report_metrics: Metrics{id:1, Learner, term:5, vote:5-0:committed, last_log:9, last_applied:5-0-9, leader:None, membership:{log_id:1-0-2, voters:[{2:{()},3:{()}}], learners:[]}, snapshot:5-0-9, purged:5-0-9, replication:{}} +2023-10-17T13:39:16.705895Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: exit +2023-10-17T13:39:16.705911Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: openraft::core::raft_core@report_metrics#c000c000000003: close time.busy=165µs time.idle=31.8µs +2023-10-17T13:39:16.705931Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: exit +2023-10-17T13:39:16.705945Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: openraft::core::raft_core@flush_metrics#f000c000000002: close time.busy=232µs time.idle=29.0µs +2023-10-17T13:39:16.705966Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: exit +2023-10-17T13:39:16.705980Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: exit +2023-10-17T13:39:16.705992Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: exit +2023-10-17T13:39:16.706389Z DEBUG ThreadId(02) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: exit +2023-10-17T13:39:16.707160Z DEBUG ThreadId(05) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::core::tick@tick#4: close time.busy=660µs time.idle=163ms +2023-10-17T13:39:16.707174Z DEBUG ThreadId(06) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::core::tick@tick#8: close time.busy=15.5µs time.idle=29.5ms +2023-10-17T13:39:16.707395Z DEBUG ThreadId(11) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: openraft::core::raft_core@runtime_loop#18008000000002{id=0}: close time.busy=43.2ms time.idle=106ms +2023-10-17T13:39:16.707445Z DEBUG ThreadId(08) openraft::core::sm@worker_loop#d800c000000001: close time.busy=5.10ms time.idle=24.5ms +2023-10-17T13:39:16.707457Z DEBUG ThreadId(04) openraft::core::sm@worker_loop#4000000001: close time.busy=4.77ms time.idle=151ms +2023-10-17T13:39:16.707516Z DEBUG ThreadId(11) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: openraft::core::raft_core@main#8000000001: close time.busy=49.4ms time.idle=105ms +2023-10-17T13:39:16.707745Z DEBUG ThreadId(09) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: openraft::core::raft_core@runtime_loop#b8004000000003{id=1}: close time.busy=17.8ms time.idle=11.6ms +2023-10-17T13:39:16.707777Z DEBUG ThreadId(09) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: openraft::core::raft_core@main#a8004000000002: close time.busy=18.5ms time.idle=11.4ms +2023-10-17T13:39:16.707935Z DEBUG ThreadId(11) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: openraft::raft@RaftCore#5{id=0 cluster=foo}: close time.busy=0.00ns time.idle=164ms +2023-10-17T13:39:16.708033Z DEBUG ThreadId(11) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: openraft::raft@new#3{cluster=foo}: close time.busy=61.8ms time.idle=106ms +2023-10-17T13:39:16.708045Z DEBUG ThreadId(11) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#2: close time.busy=11.5ms time.idle=156ms +2023-10-17T13:39:16.708048Z DEBUG ThreadId(09) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: openraft::raft@RaftCore#9{id=1 cluster=foo}: close time.busy=0.00ns time.idle=30.4ms +2023-10-17T13:39:16.708062Z DEBUG ThreadId(09) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: openraft::raft@new#40000000000007{cluster=foo}: close time.busy=18.9ms time.idle=11.6ms +2023-10-17T13:39:16.708074Z DEBUG ThreadId(09) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: snapshot_streaming::fixtures@new_raft_node_with_sto#40000000000006: close time.busy=269µs time.idle=30.2ms +2023-10-17T13:39:16.708083Z DEBUG ThreadId(09) snapshot_streaming::t33_snapshot_delete_conflict_logs@snapshot_delete_conflicting_logs#1: close time.busy=53.3ms time.idle=133ms diff --git a/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs b/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs index f2e830deb..a095ff322 100644 --- a/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs +++ b/tests/tests/snapshot_streaming/t33_snapshot_delete_conflict_logs.rs @@ -8,10 +8,10 @@ use openraft::network::RPCOption; use openraft::network::RaftNetwork; use openraft::network::RaftNetworkFactory; use openraft::raft::AppendEntriesRequest; -use openraft::raft::ExampleChunkId; -use openraft::raft::ExampleSnapshotChunk; use openraft::raft::InstallSnapshotData; use openraft::raft::InstallSnapshotRequest; +use openraft::raft::SnapshotData; +use openraft::raft::SnapshotManifest; use openraft::storage::RaftLogStorage; use openraft::storage::RaftStateMachine; use openraft::testing; @@ -58,6 +58,8 @@ async fn snapshot_delete_conflicting_logs() -> Result<()> { let mut router = RaftRouter::new(config.clone()); let mut log_index; + // cargo test --package tests --test snapshot_streaming -- + // t33_snapshot_delete_conflict_logs::snapshot_delete_conflicting_logs --exact --nocapture tracing::info!("--- manually init node-0 with a higher vote, in order to override conflict log on learner later"); { @@ -151,18 +153,28 @@ async fn snapshot_delete_conflicting_logs() -> Result<()> { b.build_snapshot().await? }; + let manifest = snap.snapshot.manifest().await; + + let mut client = router.new_client(1, &()).await; + let req = InstallSnapshotRequest { vote: sto0.read_vote().await?.unwrap(), meta: snap.meta.clone(), - data: InstallSnapshotData::Chunk(ExampleSnapshotChunk { - chunk_id: ExampleChunkId { offset: 0, len: 0 }, - data: snap.snapshot.data, - }), + data: InstallSnapshotData::Manifest(manifest.clone()), }; - let option = RPCOption::new(Duration::from_millis(1_000)); + client.install_snapshot(req, RPCOption::new(Duration::from_millis(1_000))).await?; - router.new_client(1, &()).await.install_snapshot(req, option).await?; + for chunk_id in manifest.chunks_to_send() { + let chunk = snap.snapshot.get_chunk(&chunk_id).await?; + let req = InstallSnapshotRequest { + vote: sto0.read_vote().await?.unwrap(), + meta: snap.meta.clone(), + data: InstallSnapshotData::Chunk(chunk), + }; + + client.install_snapshot(req, RPCOption::new(Duration::from_millis(1_000))).await?; + } tracing::info!(log_index, "--- DONE installing snapshot");