版本见图，连接数就几个
环境，proxmox lxc docker
日志如下
2025-11-18T13:36:03.664465+00:00 [error] State machine ds_mq_state2_28C2C38C944B0CC8 terminating. Reason: {missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>}. Stack: [{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]. Last event: {internal,go}. State: {recover,[{id,{ds_mq_state2_28C2C38C944B0CC8,emqx@node}},{raft_state,recover},{leader_last_seen,undefined},{num_pending_commands,0},{num_low_priority_commands,0},{num_pending_applied_notifications,0},{election_timeout_set,false},{ra_server_state,#{id => {ds_mq_state2_28C2C38C944B0CC8,emqx@node},machine => #{schema => #{storage => {emqx_ds_storage_skipstream_lts_v2,#{lts_threshold_spec => {mf,emqx_ds_pmap,lts_threshold_cb},timestamp_bytes => 0}},backend => builtin_raft,payload_type => ttv,n_shards => 16,n_sites => 1,replication_factor => 3},latest => 0,tx_serial => 0,db_shard => {mq_state,<<“2”>>},last_schema_changes => #{},otx_leader_pid => undefined},log => #{type => ra_log,open_segments => 1,snapshot_index => undefined,last_index => 222,num_segments => 1,first_index => 0,last_term => 134,last_wal_write => 1763472963660,last_written_index_term => {222,134},latest_checkpoint_index => undefined,mem_table_info => #{name => ra_mem_table,size => 0,tid => #Ref<0.393740879.1848508422.80400>,range => undefined,has_previous => false},mem_table_range => undefined,segments_range => {0,222},snapshot_term => undefined},aux => undefined,membership => voter,uid => <<“mq_state2_AD74CD57”>>,counter => {write_concurrency,#Ref<0.393740879.1848508418.79206>},cluster => #{{ds_mq_state2_28C2C38C944B0CC8,emqx@node} => #{status => normal,next_index => 1,query_index => 0,match_index => 0,commit_index_sent => 0}},effective_machine_version => 0,last_applied => 0,current_term => 134,system_config => #{message_queue_data => off_heap,name => mq_state,names => #{directory => ra_mq_state_directory,directory_rev => ra_mq_state_directory_reverse,log_sup => ra_mq_state_log_sup,segment_writer => ra_mq_state_segment_writer,wal => ra_mq_state_log_wal,open_mem_tbls => ra_mq_state_log_open_mem_tables,log_ets => ra_mq_state_log_ets,log_meta => ra_mq_state_log_meta,server_sup => ra_mq_state_server_sup_sup,wal_sup => ra_mq_state_log_wal_sup},data_dir => “data/ds/mq_state/dsrepl”,segment_max_entries => 4096,segment_max_pending => 1024,segment_max_size_bytes => 64000000,segment_compute_checksums => true,wal_data_dir => “data/ds/mq_state/dsrepl”,wal_max_size_bytes => 256000000,wal_compute_checksums => true,wal_max_batch_size => 8192,wal_max_entries => undefined,wal_write_strategy => default,wal_sync_method => datasync,wal_garbage_collect => false,wal_pre_allocate => false,wal_min_bin_vheap_size => 46422,wal_min_heap_size => 233,server_min_heap_size => 233,server_min_bin_vheap_size => 46422,default_max_pipeline_count => 4096,default_max_append_entries_rpc_batch_size => 128,compress_mem_tables => false,snapshot_chunk_size => 1000000,receive_snapshot_timeout => 30000,low_priority_commands_flush_size => 16,low_priority_commands_in_memory_size => 16,machine_upgrade_strategy => all},metrics_key => ds_mq_state2_28C2C38C944B0CC8,commit_index => 222,machine_version => 0,log_id => “{ds_mq_state2_28C2C38C944B0CC8,emqx@node}”,voted_for => {ds_mq_state2_28C2C38C944B0CC8,emqx@node},max_pipeline_count => 4096,max_append_entries_rpc_batch_size => 128,cluster_change_permitted => false,cluster_index_term => {0,0},query_index => 0,num_pending_queries => 0,num_waiting_queries => 0,machine_versions => [{0,0}],effective_machine_module => emqx_ds_builtin_raft_machine,effective_handle_aux_fun => undefined}}]}.
2025-11-18T13:36:03.669355+00:00 [error] crasher: initial call: ra_server_proc:init/1, pid: <0.6212.0>, registered_name: ds_mq_state2_28C2C38C944B0CC8, exit: {{missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>},[{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]}, ancestors: [<0.6211.0>,ra_mq_state_server_sup_sup,<0.5269.0>,ra_systems_sup,ra_sup,<0.4246.0>], message_queue_len: 1, messages: [{‘$gen_call’,{<0.6210.0>,[alias|#Ref<0.0.794883.393740879.1848442886.80404>]},{leader_call,{state_query,members}}}], links: [<0.6211.0>], dictionary: [{‘$emqx_ds_raft_idx_need_release’,3},{‘$emqx_ds_raft_bytes_need_release’,0}], trap_exit: true, status: running, heap_size: 28690, stack_size: 29, reductions: 72138; neighbours:
2025-11-18T13:36:03.672659+00:00 [error] Supervisor: {<0.6211.0>,ra_server_sup}. Context: child_terminated. Reason: {missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>}. Offender: id=ds_mq_state2_28C2C38C944B0CC8,pid=<0.6212.0>.
2025-11-18T13:36:03.683140+00:00 [error] State machine ds_mq_message_regular2_28C2C38C944B0CC8 terminating. Reason: {missing_key,88,<<“data/ds/mq_message_regular/dsrepl/mq_message_regular2_AD74CD57/0000000000000002.segment”>>}. Stack: [{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]. Last event: {internal,go}. State: {recover,[{id,{ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node}},{raft_state,recover},{leader_last_seen,undefined},{num_pending_commands,0},{num_low_priority_commands,0},{num_pending_applied_notifications,0},{election_timeout_set,false},{ra_server_state,#{id => {ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node},machine => #{schema => #{storage => {emqx_ds_storage_skipstream_lts_v2,#{lts_threshold_spec => {simple,{100,0,0,100,0}}}},backend => builtin_raft,payload_type => ttv,n_shards => 16,n_sites => 1,replication_factor => 3},latest => 0,tx_serial => 0,db_shard => {mq_message_regular,<<“2”>>},last_schema_changes => #{},otx_leader_pid => <0.6434.0>},log => #{type => ra_log,open_segments => 1,snapshot_index => 16,last_index => 175,num_segments => 1,first_index => 17,last_term => 106,last_wal_write => 1763472963675,last_written_index_term => {175,106},latest_checkpoint_index => undefined,mem_table_info => #{name => ra_mem_table,size => 0,tid => #Ref<0.393740879.1848508419.79244>,range => undefined,has_previous => false},mem_table_range => undefined,segments_range => {17,175},snapshot_term => 2},aux => undefined,membership => voter,uid => <<“mq_message_regular2_AD74CD57”>>,counter => {write_concurrency,#Ref<0.393740879.1848508419.79233>},cluster => #{{ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node} => #{status => normal,next_index => 1,query_index => 0,match_index => 0,commit_index_sent => 0}},effective_machine_version => 0,last_applied => 16,current_term => 106,system_config => #{message_queue_data => off_heap,name => mq_message_regular,names => #{directory => ra_mq_message_regular_directory,directory_rev => ra_mq_message_regular_directory_reverse,log_sup => ra_mq_message_regular_log_sup,segment_writer => ra_mq_message_regular_segment_writer,wal => ra_mq_message_regular_log_wal,open_mem_tbls => ra_mq_message_regular_log_open_mem_tables,log_ets => ra_mq_message_regular_log_ets,log_meta => ra_mq_message_regular_log_meta,server_sup => ra_mq_message_regular_server_sup_sup,wal_sup => ra_mq_message_regular_log_wal_sup},data_dir => “data/ds/mq_message_regular/dsrepl”,segment_max_entries => 4096,segment_max_pending => 1024,segment_max_size_bytes => 64000000,segment_compute_checksums => true,wal_data_dir => “data/ds/mq_message_regular/dsrepl”,wal_max_size_bytes => 256000000,wal_compute_checksums => true,wal_max_batch_size => 8192,wal_max_entries => undefined,wal_write_strategy => default,wal_sync_method => datasync,wal_garbage_collect => false,wal_pre_allocate => false,wal_min_bin_vheap_size => 46422,wal_min_heap_size => 233,server_min_heap_size => 233,server_min_bin_vheap_size => 46422,default_max_pipeline_count => 4096,default_max_append_entries_rpc_batch_size => 128,compress_mem_tables => false,snapshot_chunk_size => 1000000,receive_snapshot_timeout => 30000,low_priority_commands_flush_size => 16,low_priority_commands_in_memory_size => 16,machine_upgrade_strategy => all},metrics_key => ds_mq_message_regular2_28C2C38C944B0CC8,commit_index => 175,machine_version => 0,log_id => “{ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node}”,voted_for => {ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node},max_pipeline_count => 4096,max_append_entries_rpc_batch_size => 128,cluster_change_permitted => false,cluster_index_term => {16,2},query_index => 0,num_pending_queries => 0,num_waiting_queries => 0,machine_versions => [{16,0}],effective_machine_module => emqx_ds_builtin_raft_machine,effective_handle_aux_fun => undefined}}]}.
2025-11-18T13:36:03.690416+00:00 [error] crasher: initial call: ra_server_proc:init/1, pid: <0.6218.0>, registered_name: ds_mq_message_regular2_28C2C38C944B0CC8, exit: {{missing_key,88,<<“data/ds/mq_message_regular/dsrepl/mq_message_regular2_AD74CD57/0000000000000002.segment”>>},[{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]}, ancestors: [<0.6216.0>,ra_mq_message_regular_server_sup_sup,<0.5243.0>,ra_systems_sup,ra_sup,<0.4246.0>], message_queue_len: 1, messages: [{‘$gen_call’,{<0.6213.0>,[alias|#Ref<0.0.795267.393740879.1848442885.79504>]},{leader_call,{state_query,members}}}], links: [<0.6216.0>], dictionary: [{‘$emqx_ds_raft_idx_need_release’,20},{‘$emqx_ds_raft_bytes_need_release’,0}], trap_exit: true, status: running, heap_size: 28690, stack_size: 29, reductions: 74602; neighbours:
2025-11-18T13:36:03.690424+00:00 [error] State machine ds_mq_state2_28C2C38C944B0CC8 terminating. Reason: {missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>}. Stack: [{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]. Last event: {internal,go}. State: {recover,[{id,{ds_mq_state2_28C2C38C944B0CC8,emqx@node}},{raft_state,recover},{leader_last_seen,undefined},{num_pending_commands,0},{num_low_priority_commands,0},{num_pending_applied_notifications,0},{election_timeout_set,false},{ra_server_state,#{id => {ds_mq_state2_28C2C38C944B0CC8,emqx@node},machine => #{schema => #{storage => {emqx_ds_storage_skipstream_lts_v2,#{lts_threshold_spec => {mf,emqx_ds_pmap,lts_threshold_cb},timestamp_bytes => 0}},backend => builtin_raft,payload_type => ttv,n_shards => 16,n_sites => 1,replication_factor => 3},latest => 0,tx_serial => 0,db_shard => {mq_state,<<“2”>>},last_schema_changes => #{},otx_leader_pid => undefined},log => #{type => ra_log,open_segments => 1,snapshot_index => undefined,last_index => 222,num_segments => 1,first_index => 0,last_term => 134,last_wal_write => 1763472963683,last_written_index_term => {222,134},latest_checkpoint_index => undefined,mem_table_info => #{name => ra_mem_table,size => 0,tid => #Ref<0.393740879.1848508422.80400>,range => undefined,has_previous => false},mem_table_range => undefined,segments_range => {0,222},snapshot_term => undefined},aux => undefined,membership => voter,uid => <<“mq_state2_AD74CD57”>>,counter => {write_concurrency,#Ref<0.393740879.1848508418.79206>},cluster => #{{ds_mq_state2_28C2C38C944B0CC8,emqx@node} => #{status => normal,next_index => 1,query_index => 0,match_index => 0,commit_index_sent => 0}},effective_machine_version => 0,last_applied => 0,current_term => 134,system_config => #{message_queue_data => off_heap,name => mq_state,names => #{directory => ra_mq_state_directory,directory_rev => ra_mq_state_directory_reverse,log_sup => ra_mq_state_log_sup,segment_writer => ra_mq_state_segment_writer,wal => ra_mq_state_log_wal,open_mem_tbls => ra_mq_state_log_open_mem_tables,log_ets => ra_mq_state_log_ets,log_meta => ra_mq_state_log_meta,server_sup => ra_mq_state_server_sup_sup,wal_sup => ra_mq_state_log_wal_sup},data_dir => “data/ds/mq_state/dsrepl”,segment_max_entries => 4096,segment_max_pending => 1024,segment_max_size_bytes => 64000000,segment_compute_checksums => true,wal_data_dir => “data/ds/mq_state/dsrepl”,wal_max_size_bytes => 256000000,wal_compute_checksums => true,wal_max_batch_size => 8192,wal_max_entries => undefined,wal_write_strategy => default,wal_sync_method => datasync,wal_garbage_collect => false,wal_pre_allocate => false,wal_min_bin_vheap_size => 46422,wal_min_heap_size => 233,server_min_heap_size => 233,server_min_bin_vheap_size => 46422,default_max_pipeline_count => 4096,default_max_append_entries_rpc_batch_size => 128,compress_mem_tables => false,snapshot_chunk_size => 1000000,receive_snapshot_timeout => 30000,low_priority_commands_flush_size => 16,low_priority_commands_in_memory_size => 16,machine_upgrade_strategy => all},metrics_key => ds_mq_state2_28C2C38C944B0CC8,commit_index => 222,machine_version => 0,log_id => “{ds_mq_state2_28C2C38C944B0CC8,emqx@node}”,voted_for => {ds_mq_state2_28C2C38C944B0CC8,emqx@node},max_pipeline_count => 4096,max_append_entries_rpc_batch_size => 128,cluster_change_permitted => false,cluster_index_term => {0,0},query_index => 0,num_pending_queries => 0,num_waiting_queries => 0,machine_versions => [{0,0}],effective_machine_module => emqx_ds_builtin_raft_machine,effective_handle_aux_fun => undefined}}]}.
2025-11-18T13:36:03.695133+00:00 [error] Supervisor: {<0.6216.0>,ra_server_sup}. Context: child_terminated. Reason: {missing_key,88,<<“data/ds/mq_message_regular/dsrepl/mq_message_regular2_AD74CD57/0000000000000002.segment”>>}. Offender: id=ds_mq_message_regular2_28C2C38C944B0CC8,pid=<0.6218.0>.
2025-11-18T13:36:03.695744+00:00 [error] crasher: initial call: ra_server_proc:init/1, pid: <0.6225.0>, registered_name: ds_mq_state2_28C2C38C944B0CC8, exit: {{missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>},[{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]}, ancestors: [<0.6211.0>,ra_mq_state_server_sup_sup,<0.5269.0>,ra_systems_sup,ra_sup,<0.4246.0>], message_queue_len: 1, messages: [{‘$gen_call’,{<0.6210.0>,[alias|#Ref<0.0.794883.393740879.1848442886.80416>]},{leader_call,{state_query,members}}}], links: [<0.6211.0>], dictionary: [{‘$emqx_ds_raft_idx_need_release’,3},{‘$emqx_ds_raft_bytes_need_release’,0}], trap_exit: true, status: running, heap_size: 28690, stack_size: 29, reductions: 71933; neighbours:
2025-11-18T13:36:03.699093+00:00 [error] Supervisor: {<0.6211.0>,ra_server_sup}. Context: child_terminated. Reason: {missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>}. Offender: id=ds_mq_state2_28C2C38C944B0CC8,pid=<0.6225.0>.
2025-11-18T13:36:03.712519+00:00 [error] State machine ds_mq_state2_28C2C38C944B0CC8 terminating. Reason: {missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>}. Stack: [{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]. Last event: {internal,go}. State: {recover,[{id,{ds_mq_state2_28C2C38C944B0CC8,emqx@node}},{raft_state,recover},{leader_last_seen,undefined},{num_pending_commands,0},{num_low_priority_commands,0},{num_pending_applied_notifications,0},{election_timeout_set,false},{ra_server_state,#{id => {ds_mq_state2_28C2C38C944B0CC8,emqx@node},machine => #{schema => #{storage => {emqx_ds_storage_skipstream_lts_v2,#{lts_threshold_spec => {mf,emqx_ds_pmap,lts_threshold_cb},timestamp_bytes => 0}},backend => builtin_raft,payload_type => ttv,n_shards => 16,n_sites => 1,replication_factor => 3},latest => 0,tx_serial => 0,db_shard => {mq_state,<<“2”>>},last_schema_changes => #{},otx_leader_pid => undefined},log => #{type => ra_log,open_segments => 1,snapshot_index => undefined,last_index => 222,num_segments => 1,first_index => 0,last_term => 134,last_wal_write => 1763472963706,last_written_index_term => {222,134},latest_checkpoint_index => undefined,mem_table_info => #{name => ra_mem_table,size => 0,tid => #Ref<0.393740879.1848508422.80400>,range => undefined,has_previous => false},mem_table_range => undefined,segments_range => {0,222},snapshot_term => undefined},aux => undefined,membership => voter,uid => <<“mq_state2_AD74CD57”>>,counter => {write_concurrency,#Ref<0.393740879.1848508418.79206>},cluster => #{{ds_mq_state2_28C2C38C944B0CC8,emqx@node} => #{status => normal,next_index => 1,query_index => 0,match_index => 0,commit_index_sent => 0}},effective_machine_version => 0,last_applied => 0,current_term => 134,system_config => #{message_queue_data => off_heap,name => mq_state,names => #{directory => ra_mq_state_directory,directory_rev => ra_mq_state_directory_reverse,log_sup => ra_mq_state_log_sup,segment_writer => ra_mq_state_segment_writer,wal => ra_mq_state_log_wal,open_mem_tbls => ra_mq_state_log_open_mem_tables,log_ets => ra_mq_state_log_ets,log_meta => ra_mq_state_log_meta,server_sup => ra_mq_state_server_sup_sup,wal_sup => ra_mq_state_log_wal_sup},data_dir => “data/ds/mq_state/dsrepl”,segment_max_entries => 4096,segment_max_pending => 1024,segment_max_size_bytes => 64000000,segment_compute_checksums => true,wal_data_dir => “data/ds/mq_state/dsrepl”,wal_max_size_bytes => 256000000,wal_compute_checksums => true,wal_max_batch_size => 8192,wal_max_entries => undefined,wal_write_strategy => default,wal_sync_method => datasync,wal_garbage_collect => false,wal_pre_allocate => false,wal_min_bin_vheap_size => 46422,wal_min_heap_size => 233,server_min_heap_size => 233,server_min_bin_vheap_size => 46422,default_max_pipeline_count => 4096,default_max_append_entries_rpc_batch_size => 128,compress_mem_tables => false,snapshot_chunk_size => 1000000,receive_snapshot_timeout => 30000,low_priority_commands_flush_size => 16,low_priority_commands_in_memory_size => 16,machine_upgrade_strategy => all},metrics_key => ds_mq_state2_28C2C38C944B0CC8,commit_index => 222,machine_version => 0,log_id => “{ds_mq_state2_28C2C38C944B0CC8,emqx@node}”,voted_for => {ds_mq_state2_28C2C38C944B0CC8,emqx@node},max_pipeline_count => 4096,max_append_entries_rpc_batch_size => 128,cluster_change_permitted => false,cluster_index_term => {0,0},query_index => 0,num_pending_queries => 0,num_waiting_queries => 0,machine_versions => [{0,0}],effective_machine_module => emqx_ds_builtin_raft_machine,effective_handle_aux_fun => undefined}}]}.
2025-11-18T13:36:03.719228+00:00 [error] crasher: initial call: ra_server_proc:init/1, pid: <0.6242.0>, registered_name: ds_mq_state2_28C2C38C944B0CC8, exit: {{missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>},[{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]}, ancestors: [<0.6211.0>,ra_mq_state_server_sup_sup,<0.5269.0>,ra_systems_sup,ra_sup,<0.4246.0>], message_queue_len: 1, messages: [{‘$gen_call’,{<0.6210.0>,[alias|#Ref<0.0.794883.393740879.1848442886.80461>]},{leader_call,{state_query,members}}}], links: [<0.6211.0>], dictionary: [{‘$emqx_ds_raft_idx_need_release’,3},{‘$emqx_ds_raft_bytes_need_release’,0}], trap_exit: true, status: running, heap_size: 28690, stack_size: 29, reductions: 71964; neighbours:
2025-11-18T13:36:03.722824+00:00 [error] Supervisor: {<0.6211.0>,ra_server_sup}. Context: child_terminated. Reason: {missing_key,63,<<“data/ds/mq_state/dsrepl/mq_state2_AD74CD57/0000000000000001.segment”>>}. Offender: id=ds_mq_state2_28C2C38C944B0CC8,pid=<0.6242.0>.
2025-11-18T13:36:03.724187+00:00 [error] Supervisor: {<0.6211.0>,ra_server_sup}. Context: shutdown. Reason: reached_max_restart_intensity. Offender: id=ds_mq_state2_28C2C38C944B0CC8,pid=<0.6242.0>.
2025-11-18T13:36:03.717958+00:00 [error] State machine ds_mq_message_regular2_28C2C38C944B0CC8 terminating. Reason: {missing_key,88,<<“data/ds/mq_message_regular/dsrepl/mq_message_regular2_AD74CD57/0000000000000002.segment”>>}. Stack: [{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]. Last event: {internal,go}. State: {recover,[{id,{ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node}},{raft_state,recover},{leader_last_seen,undefined},{num_pending_commands,0},{num_low_priority_commands,0},{num_pending_applied_notifications,0},{election_timeout_set,false},{ra_server_state,#{id => {ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node},machine => #{schema => #{storage => {emqx_ds_storage_skipstream_lts_v2,#{lts_threshold_spec => {simple,{100,0,0,100,0}}}},backend => builtin_raft,payload_type => ttv,n_shards => 16,n_sites => 1,replication_factor => 3},latest => 0,tx_serial => 0,db_shard => {mq_message_regular,<<“2”>>},last_schema_changes => #{},otx_leader_pid => <0.6434.0>},log => #{type => ra_log,open_segments => 1,snapshot_index => 16,last_index => 175,num_segments => 1,first_index => 17,last_term => 106,last_wal_write => 1763472963710,last_written_index_term => {175,106},latest_checkpoint_index => undefined,mem_table_info => #{name => ra_mem_table,size => 0,tid => #Ref<0.393740879.1848508419.79244>,range => undefined,has_previous => false},mem_table_range => undefined,segments_range => {17,175},snapshot_term => 2},aux => undefined,membership => voter,uid => <<“mq_message_regular2_AD74CD57”>>,counter => {write_concurrency,#Ref<0.393740879.1848508419.79233>},cluster => #{{ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node} => #{status => normal,next_index => 1,query_index => 0,match_index => 0,commit_index_sent => 0}},effective_machine_version => 0,last_applied => 16,current_term => 106,system_config => #{message_queue_data => off_heap,name => mq_message_regular,names => #{directory => ra_mq_message_regular_directory,directory_rev => ra_mq_message_regular_directory_reverse,log_sup => ra_mq_message_regular_log_sup,segment_writer => ra_mq_message_regular_segment_writer,wal => ra_mq_message_regular_log_wal,open_mem_tbls => ra_mq_message_regular_log_open_mem_tables,log_ets => ra_mq_message_regular_log_ets,log_meta => ra_mq_message_regular_log_meta,server_sup => ra_mq_message_regular_server_sup_sup,wal_sup => ra_mq_message_regular_log_wal_sup},data_dir => “data/ds/mq_message_regular/dsrepl”,segment_max_entries => 4096,segment_max_pending => 1024,segment_max_size_bytes => 64000000,segment_compute_checksums => true,wal_data_dir => “data/ds/mq_message_regular/dsrepl”,wal_max_size_bytes => 256000000,wal_compute_checksums => true,wal_max_batch_size => 8192,wal_max_entries => undefined,wal_write_strategy => default,wal_sync_method => datasync,wal_garbage_collect => false,wal_pre_allocate => false,wal_min_bin_vheap_size => 46422,wal_min_heap_size => 233,server_min_heap_size => 233,server_min_bin_vheap_size => 46422,default_max_pipeline_count => 4096,default_max_append_entries_rpc_batch_size => 128,compress_mem_tables => false,snapshot_chunk_size => 1000000,receive_snapshot_timeout => 30000,low_priority_commands_flush_size => 16,low_priority_commands_in_memory_size => 16,machine_upgrade_strategy => all},metrics_key => ds_mq_message_regular2_28C2C38C944B0CC8,commit_index => 175,machine_version => 0,log_id => “{ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node}”,voted_for => {ds_mq_message_regular2_28C2C38C944B0CC8,emqx@node},max_pipeline_count => 4096,max_append_entries_rpc_batch_size => 128,cluster_change_permitted => false,cluster_index_term => {16,2},query_index => 0,num_pending_queries => 0,num_waiting_queries => 0,machine_versions => [{16,0}],effective_machine_module => emqx_ds_builtin_raft_machine,effective_handle_aux_fun => undefined}}]}.
2025-11-18T13:36:03.727387+00:00 [error] crasher: initial call: ra_server_proc:init/1, pid: <0.6240.0>, registered_name: ds_mq_message_regular2_28C2C38C944B0CC8, exit: {{missing_key,88,<<“data/ds/mq_message_regular/dsrepl/mq_message_regular2_AD74CD57/0000000000000002.segment”>>},[{ra_log_segment,fold0,8,[{file,“/emqx/deps/ra/src/ra_log_segment.erl”},{line,426}]},{ra_log_reader,‘-segment_fold/5-fun-1-’,4,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,344}]},{ra_log_reader,‘-segment_fold/5-lists^foldl/2-1-’,3,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log_reader,segment_fold,5,[{file,“/emqx/deps/ra/src/ra_log_reader.erl”},{line,341}]},{ra_log,fold,5,[{file,“/emqx/deps/ra/src/ra_log.erl”},{line,462}]},{ra_server,apply_to,5,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,2820}]},{ra_server,recover,1,[{file,“/emqx/deps/ra/src/ra_server.erl”},{line,436}]},{ra_server_proc,recover,3,[{file,“/emqx/deps/ra/src/ra_server_proc.erl”},{line,405}]},{gen_statem,loop_state_callback,11,[{file,“gen_statem.erl”},{line,3735}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,329}]}]}, ancestors: [<0.6216.0>,ra_mq_message_regular_server_sup_sup,<0.5243.0>,ra_systems_sup,ra_sup,<0.4246.0>], message_queue_len: 1, messages: [{‘$gen_call’,{<0.6213.0>,[alias|#Ref<0.0.795267.393740879.1848442885.79577>]},{leader_call,{state_query,members}}}], links: [<0.6216.0>], dictionary: [{‘$emqx_ds_raft_idx_need_release’,20},{‘$emqx_ds_raft_bytes_need_release’,0}], trap_exit: true, status: running, heap_size: 28690, stack_size: 29, reductions: 76707; neighbours:
