环境
- EMQX 版本：5.5.1
- 操作系统版本：ubuntu22.04
使用的了exhook插件，对数据进行了处理，然后入库
重现此问题的步骤
- 两个台服务分别通过docker-compose 部署，部署信息如下
服务器1:
emqx:
image: emqx:5.5.1
container_name: emqx
network_mode: “host”
environment:-
“EMQX_NODE_NAME=nodeA@172.20.0.14”
- “EMQX_LISTENER__TCP__EXTERNAL=172.20.0.14:1883”
-
“EMQX_CLUSTER__DISCOVERY_STRATEGY=static”
-
“EMQX_CLUSTER__STATIC__SEEDS=[nodeA@172.20.0.14,nodeB@172.20.0.201]”
-
“EMQX_LISTENERS__TCP__DEFAULT__BIND=0.0.0.0:1884”
-
“EMQX_DASHBOARD__DEFAULT_PASSWORD=Auto12345@”
- “EMQX_SETUP__DEFAULT_PASSWORD=Auto12345@”
- “EMQX_AUTH__DEFAULT_USER__USERNAME=admin”
- “EMQX_AUTH__DEFAULT_USER__PASSWORD=Auto12345@”
- “API_KEY=none”
healthcheck:
test: [“CMD”, “/opt/emqx/bin/emqx_ctl”, “status”]
interval: 5s
timeout: 25s
retries: 5
volumes:- /data/emqx/data:/opt/emqx/data
- /data/emqx/log:/opt/emqx/log
-
服务器2:
image: emqx:5.5.1
container_name: emqx
network_mode: “host”
environment:
- “EMQX_NODE_NAME=nodeB@172.20.0.201”
# - “EMQX_LISTENER__TCP__EXTERNAL=172.20.0.14:1883”
- "EMQX_CLUSTER__DISCOVERY_STRATEGY=static"
- "EMQX_CLUSTER__STATIC__SEEDS=[nodeA@172.20.0.14,nodeB@172.20.0.201]"
# - "EMQX_SETUP__DEFAULT_PASSWORD=Auto12345@"
# - "EMQX_AUTH__DEFAULT_USER__USERNAME=admin"
# - "EMQX_AUTH__DEFAULT_USER__PASSWORD=Auto12345@"
# - "API_KEY=none"
healthcheck:
test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
interval: 5s
timeout: 25s
retries: 5
volumes:
- /data/emqx/data:/opt/emqx/data
- /data/emqx/etc/emqx.conf:/opt/emqx/etc/emqx.conf
使用nginx 进行负载
stream{
upstream mqtt_servers {
server 172.20.0.201:1883;
server 172.20.0.14:1884;
}
server {
listen 1883;
proxy_pass mqtt_servers;
}
}
预期行为
之前单节点连接数在1000左右的时候一切都是正常的，现在连接数到2000左右，服务响应就变慢了，变成集群2个节点也不行
实际行为
后台产生的大量的告警
2024-03-23T02:08:34.215174+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <0.2005669.0>, stale_channel: [{status,waiting},{message_queue_len,8},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run,2,[{file,“emqx_hooks.erl”},{line,172}]},{emqx_broker,route,2,[{file,“emqx_broker.erl”},{line,289}]},{emqx_broker,publish,1,[{file,“emqx_broker.erl”},{line,246}]},{emqx_channel,do_publish,3,[{file,“emqx_channel.erl”},{line,685}]},{emqx_connection,with_channel,3,[{file,“emqx_connection.erl”},{line,843}]},{emqx_connection,process_msg,2,[{file,“emqx_connection.erl”},{line,493}]},{emqx_connection,process_msg,2,[{file,“emqx_connection.erl”},{line,499}]},{emqx_connection,handle_recv,3,[{file,“emqx_connection.erl”},{line,455}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:34.576164+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <0.1985634.0>, stale_channel: [{status,waiting},{message_queue_len,5},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run_fold,3,[{file,“emqx_hooks.erl”},{line,185}]},{emqx_channel,do_deliver,2,[{file,“emqx_channel.erl”},{line,1086}]},{emqx_channel,‘-do_deliver/2-fun-0-’,2,[{file,“emqx_channel.erl”},{line,1101}]},{lists,foldl,3,[{file,“lists.erl”},{line,1594}]},{emqx_channel,do_deliver,2,[{file,“emqx_channel.erl”},{line,1099}]},{emqx_channel,handle_out,3,[{file,“emqx_channel.erl”},{line,1011}]},{emqx_connection,with_channel,3,[{file,“emqx_connection.erl”},{line,843}]},{emqx_connection,process_msg,2,[{file,“emqx_connection.erl”},{line,493}]},{emqx_connection,handle_recv,3,[{file,“emqx_connection.erl”},{line,455}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:34.970137+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <0.1998408.0>, stale_channel: [{status,waiting},{message_queue_len,3},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run,2,[{file,“emqx_hooks.erl”},{line,172}]},{emqx_session,terminate,3,[{file,“emqx_session.erl”},{line,511}]},{emqx_connection,terminate,2,[{file,“emqx_connection.erl”},{line,669}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:36.035913+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <61893.4531444.0>, stale_channel: [{status,waiting},{message_queue_len,5},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run,2,[{file,“emqx_hooks.erl”},{line,172}]},{emqx_session,terminate,3,[{file,“emqx_session.erl”},{line,511}]},{emqx_connection,terminate,2,[{file,“emqx_connection.erl”},{line,669}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:36.608207+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <0.2021717.0>, stale_channel: [{status,waiting},{message_queue_len,3},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run,2,[{file,“emqx_hooks.erl”},{line,172}]},{emqx_session,terminate,3,[{file,“emqx_session.erl”},{line,511}]},{emqx_connection,terminate,2,[{file,“emqx_connection.erl”},{line,669}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:37.993195+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <0.1967653.0>, stale_channel: [{status,waiting},{message_queue_len,6},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run,2,[{file,“emqx_hooks.erl”},{line,172}]},{emqx_session,terminate,3,[{file,“emqx_session.erl”},{line,511}]},{emqx_connection,terminate,2,[{file,“emqx_connection.erl”},{line,669}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:38.546913+00:00 [warning] msg: session_stepdown_request_timeout, mfa: emqx_cm:request_stepdown/3(442), action: discard, stale_pid: <61893.4516795.0>, stale_channel: [{status,waiting},{message_queue_len,3},{current_stacktrace,[{grpc_client,call,3,[{file,“grpc_client.erl”},{line,757}]},{grpc_client,recv,2,[{file,“grpc_client.erl”},{line,238}]},{grpc_client,unary,4,[{file,“grpc_client.erl”},{line,169}]},{emqx_exhook_server,do_call,5,[{file,“emqx_exhook_server.erl”},{line,387}]},{emqx_exhook,cast,3,[{file,“emqx_exhook.erl”},{line,42}]},{emqx_hooks,safe_execute,2,[{file,“emqx_hooks.erl”},{line,205}]},{emqx_hooks,do_run,2,[{file,“emqx_hooks.erl”},{line,172}]},{emqx_session,terminate,3,[{file,“emqx_session.erl”},{line,511}]},{emqx_connection,terminate,2,[{file,“emqx_connection.erl”},{line,669}]},{proc_lib,wake_up,3,[{file,“proc_lib.erl”},{line,251}]}]}]
2024-03-23T02:08:39.952751+00:00 [error] supervisor: {esockd_connection_sup,<0.2001203.0>}, errorContext: connection_shutdown, reason: #{hint => malformed_packet,header_type => 7}, offender: [{pid,<0.2001203.0>},{name,connection},{mfargs,{emqx_connection,start_link,[#{listener => {tcp,default},limiter => undefined,zone => default,enable_authn => true}]}}]
麻烦求解决 ！！！感谢