环境信息
- EMQX 版本：5.0.8
- 操作系统及版本：Docker Image
- 其他 k8s 1.16.9-aliyun.1
问题描述
节点无法加入集群
配置文件及日志
dashboard.default_password = EMQX_DASHBOARD__DEFAULT_PASSWORD = ******
dashboard.default_username = EMQX_DASHBOARD__DEFAULT_USERNAME = admin
rpc.port_discovery = EMQX_RPC__PORT_DISCOVERY = manual
log.file_handlers.default.enable = EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE = false
log.console_handler.level = EMQX_LOG__CONSOLE_HANDLER__LEVEL = debug
log.console_handler.enable = EMQX_LOG__CONSOLE_HANDLER__ENABLE = true
cluster.k8s.suffix = EMQX_CLUSTER__K8S__SUFFIX = svc.cluster.local
cluster.k8s.namespace = EMQX_CLUSTER__K8S__NAMESPACE = dev
cluster.k8s.address_type = EMQX_CLUSTER__K8S__ADDRESS_TYPE = hostname
cluster.k8s.service_name = EMQX_CLUSTER__K8S__SERVICE_NAME = emqx-headless
cluster.k8s.apiserver = EMQX_CLUSTER__K8S__APISERVER = “https://kubernetes.default.svc:443”
cluster.dns.record_type = EMQX_CLUSTER__DNS__RECORD_TYPE = srv
cluster.dns.name = EMQX_CLUSTER__DNS__NAME = emqx-headless.dev.svc.cluster.local
cluster.discovery_strategy = EMQX_CLUSTER__DISCOVERY_STRATEGY = k8s
node.cookie = EMQX_NODE__COOKIE = ******
node.name = EMQX_NODE__NAME = emqx@emqx-1.emqx-headless.dev.svc.cluster.local
2022-10-09T01:34:24.693262+00:00 [info] Mria(Membership): Node emqx@emqx-0.emqx-headless.dev.svc.cluster.local up
2022-10-09T01:34:27.565211+00:00 [warning] Description: “Authenticity is not established by certificate path validation”, Reason: “Option {verify, verify_peer} and cacertfile/cacerts is missing”
2022-10-09T01:34:27.623169+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:27.624068+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:34:27.624291+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:27.624512+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:27.624768+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:34:27.626998+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.39>" result=success
2022-10-09T01:34:27.627276+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.39>"
2022-10-09T01:34:27.627427+00:00 [error] event=authentication_reception_failed socket="#Port<0.39>" reason=“closed”
2022-10-09T01:34:27.627538+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:34:27.627766+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2636.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28902; neighbours:
2022-10-09T01:34:27.627667+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:34:27.635563+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:34:31.506785+00:00 [debug] line: 362, mfa: emqx_telemetry:report_telemetry/1, msg: telemetry_data_reported
2022-10-09T01:34:32.787806+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:32.788439+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:34:32.788515+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:32.788679+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:32.788877+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:34:32.790560+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.42>" result=success
2022-10-09T01:34:32.790754+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.42>"
2022-10-09T01:34:32.790922+00:00 [error] event=authentication_reception_failed socket="#Port<0.42>" reason=“closed”
2022-10-09T01:34:32.791043+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:34:32.791219+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2650.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:34:32.791522+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:34:32.808605+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:34:38.481330+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:38.481923+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:34:38.482019+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:38.482165+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:38.482428+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:34:38.484130+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.44>" result=success
2022-10-09T01:34:38.484354+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.44>"
2022-10-09T01:34:38.484464+00:00 [error] event=authentication_reception_failed socket="#Port<0.44>" reason=“closed”
2022-10-09T01:34:38.484549+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:34:38.484647+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:34:38.484675+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2660.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:34:38.489714+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:34:45.282398+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:45.282985+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:34:45.283077+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:45.283355+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:45.283623+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:34:45.285468+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.47>" result=success
2022-10-09T01:34:45.285657+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.47>"
2022-10-09T01:34:45.285797+00:00 [error] event=authentication_reception_failed socket="#Port<0.47>" reason=“closed”
2022-10-09T01:34:45.285918+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:34:45.286120+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:34:45.286083+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2671.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:34:45.293787+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:34:52.522371+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:52.523097+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:34:52.523241+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:52.523509+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:52.523952+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:34:52.525887+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.49>" result=success
2022-10-09T01:34:52.526065+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.49>"
2022-10-09T01:34:52.526186+00:00 [error] event=authentication_reception_failed socket="#Port<0.49>" reason=“closed”
2022-10-09T01:34:52.526295+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:34:52.526410+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:34:52.526424+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2682.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:34:52.536277+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:34:58.185285+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:58.185968+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:34:58.186092+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:58.186376+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:34:58.186579+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:34:58.188418+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.51>" result=success
2022-10-09T01:34:58.188630+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.51>"
2022-10-09T01:34:58.188798+00:00 [error] event=authentication_reception_failed socket="#Port<0.51>" reason=“closed”
2022-10-09T01:34:58.188934+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:34:58.189254+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:34:58.189091+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2692.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:34:58.197150+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:06.108792+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:06.109532+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:06.109669+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:06.109910+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:06.110302+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:06.112231+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.54>" result=success
2022-10-09T01:35:06.112494+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.54>"
2022-10-09T01:35:06.112621+00:00 [error] event=authentication_reception_failed socket="#Port<0.54>" reason=“closed”
2022-10-09T01:35:06.112731+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:06.112949+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:06.112906+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2705.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:35:06.124287+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:13.071557+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:13.072355+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:13.072517+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:13.072620+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:13.072877+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:13.074594+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.56>" result=success
2022-10-09T01:35:13.074852+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.56>"
2022-10-09T01:35:13.074987+00:00 [error] event=authentication_reception_failed socket="#Port<0.56>" reason=“closed”
2022-10-09T01:35:13.075122+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:13.075307+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:13.075306+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2715.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28910; neighbours:
2022-10-09T01:35:13.081436+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:20.699912+00:00 [debug] line: 141, mfa: emqx_retainer_mnesia:store_retained/2, msg: message_retained, topic: $SYS/brokers/emqx@emqx-1.emqx-headless.dev.svc.cluster.local/version
2022-10-09T01:35:20.700632+00:00 [debug] line: 141, mfa: emqx_retainer_mnesia:store_retained/2, msg: message_retained, topic: $SYS/brokers/emqx@emqx-1.emqx-headless.dev.svc.cluster.local/sysdescr
2022-10-09T01:35:20.701245+00:00 [debug] line: 141, mfa: emqx_retainer_mnesia:store_retained/2, msg: message_retained, topic: $SYS/brokers
2022-10-09T01:35:20.817317+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:20.817937+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:20.818066+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:20.818352+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:20.818574+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:20.820397+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.59>" result=success
2022-10-09T01:35:20.820644+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.59>"
2022-10-09T01:35:20.820839+00:00 [error] event=authentication_reception_failed socket="#Port<0.59>" reason=“closed”
2022-10-09T01:35:20.820977+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:20.821213+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:20.821177+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2737.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28903; neighbours:
2022-10-09T01:35:20.827471+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:26.030322+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:26.030957+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:26.031063+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:26.031241+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:26.031451+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:26.033380+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.61>" result=success
2022-10-09T01:35:26.033606+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.61>"
2022-10-09T01:35:26.033724+00:00 [error] event=authentication_reception_failed socket="#Port<0.61>" reason=“closed”
2022-10-09T01:35:26.033840+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:26.033986+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2746.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28914; neighbours:
2022-10-09T01:35:26.034025+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:26.040691+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:32.990224+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:32.990929+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:32.991078+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:32.991309+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:32.991532+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:32.993388+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.63>" result=success
2022-10-09T01:35:32.993592+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.63>"
2022-10-09T01:35:32.993716+00:00 [error] event=authentication_reception_failed socket="#Port<0.63>" reason=“closed”
2022-10-09T01:35:32.993819+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:32.994003+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:32.994005+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2757.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28899; neighbours:
2022-10-09T01:35:33.002391+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:38.225003+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:38.225688+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:38.225791+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:38.225978+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:38.226229+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:38.228024+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.65>" result=success
2022-10-09T01:35:38.228287+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.65>"
2022-10-09T01:35:38.228439+00:00 [error] event=authentication_reception_failed socket="#Port<0.65>" reason=“closed”
2022-10-09T01:35:38.228534+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:38.228689+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:38.228695+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2767.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28899; neighbours:
2022-10-09T01:35:38.236165+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms
2022-10-09T01:35:44.170497+00:00 [info] Ekka(AutoCluster): joining with ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:44.171339+00:00 [info] event=client_process_not_found target="‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’" action=spawning_client
2022-10-09T01:35:44.171537+00:00 [debug] line: 61, mfa: gen_rpc_dispatcher:handle_call/3, msg: gen_rpc_start_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:44.171695+00:00 [debug] line: 38, mfa: gen_rpc_client_sup:start_child/1, msg: gen_rpc_starting_new_client, target: ‘emqx@emqx-0.emqx-headless.dev.svc.cluster.local’
2022-10-09T01:35:44.171902+00:00 [info] event=initializing_client driver=tcp node=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” port=5369
2022-10-09T01:35:44.173662+00:00 [debug] event=connect_to_remote_server peer=“emqx@emqx-0.emqx-headless.dev.svc.cluster.local” socket="#Port<0.68>" result=success
2022-10-09T01:35:44.173939+00:00 [debug] event=authentication_connection_succeeded socket="#Port<0.68>"
2022-10-09T01:35:44.174109+00:00 [error] event=authentication_reception_failed socket="#Port<0.68>" reason=“closed”
2022-10-09T01:35:44.174250+00:00 [error] event=client_authentication_failed driver=tcp reason="{badtcp,closed}"
2022-10-09T01:35:44.174430+00:00 [error] crasher: initial call: gen_rpc_client:init/1, pid: <0.2779.0>, registered_name: [], exit: {{badtcp,closed},[{gen_server,init_it,6,[{file,“gen_server.erl”},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,“proc_lib.erl”},{line,226}]}]}, ancestors: [gen_rpc_client_sup,gen_rpc_sup,<0.1806.0>], message_queue_len: 0, messages: [], links: [<0.1812.0>], dictionary: [], trap_exit: true, status: running, heap_size: 2586, stack_size: 29, reductions: 28900; neighbours:
2022-10-09T01:35:44.174785+00:00 [error] Ekka(AutoCluster): Discover error: {case_clause,{badtcp,closed}}, [{mria,do_join,2,[{file,“mria.erl”},{line,354}]},{ekka_autocluster,discover_and_join,2,[{file,“ekka_autocluster.erl”},{line,161}]},{ekka_autocluster,’-discover_and_join/0-fun-0-’,2,[{file,“ekka_autocluster.erl”},{line,109}]},{ekka_autocluster,’-run/1-fun-0-’,1,[{file,“ekka_autocluster.erl”},{line,54}]}]
2022-10-09T01:35:44.182783+00:00 [warning] Ekka(AutoCluster): discovery did not succeed; retrying in 5000 ms