k8s operator 部署emq 5.0 配置ssl证书报错

环境信息

  • EMQX 版本:5.0.8
  • 操作系统及版本:1.22.10
  • 其他

问题描述

k8s operator 部署emq 5.0 配置ssl证书报错

配置文件及日志

报错日志

2022-10-25T18:47:04.345028808+08:00 Listener ssl:cacertfile on :8883 started.

2022-10-25T18:47:04.347020696+08:00 2022-10-25T10:47:04.345875+00:00 [error] ssl:certfile failed to listen on 8883 - eaddrinuse (address already in use)

2022-10-25T18:47:04.347031794+08:00 2022-10-25T10:47:04.346006+00:00 [error] crasher: initial call: esockd_listener:init/1, pid: <0.2200.0>, registered_name: [], exit: {eaddrinuse,[{gen_server,init_it,6,[{file,"gen_server.erl"},{line,407}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [<0.2197.0>,esockd_sup,<0.1897.0>], message_queue_len: 0, messages: [], links: [<0.2197.0>], dictionary: [], trap_exit: true, status: running, heap_size: 1598, stack_size: 29, reductions: 6634; neighbours:

2022-10-25T18:47:04.347187363+08:00 Failed to start listener ssl:certfile on :8883: {eaddrinuse,{child,undefined,{listener_sup,{'ssl:certfile',8883}},{esockd_listener_sup,start_link,[tcp,'ssl:certfile',8883,[{tcp_options,[binary,{packet,raw},{reuseaddr,true},{backlog,1024},{nodelay,true},{buffer,4096},{high_watermark,1048576},{send_timeout,15000},{send_timeout_close,true}]},{acceptors,16},{access_rules,[{allow,all}]},{limiter,#{bucket => #{capacity => 1000,initial => 0,rate => 100.0},id => 'ssl:certfile',module => emqx_esockd_htb_limiter,type => connection}},{max_connections,infinity},{proxy_protocol,false},{proxy_protocol_timeout,3000},{ssl_options,[{certfile,<<"etc/certs/mqtt_server.crt">>},{ciphers,["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256","TLS_CHACHA20_POLY1305_SHA256","TLS_AES_128_CCM_SHA256","TLS_AES_128_CCM_8_SHA256","ECDHE-ECDSA-AES256-GCM-SHA384","ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-SHA384","ECDHE-RSA-AES256-SHA384","ECDH-ECDSA-AES256-GCM-SHA384","ECDH-RSA-AES256-GCM-SHA384","ECDH-ECDSA-AES256-SHA384","ECDH-RSA-AES256-SHA384","DHE-DSS-AES256-GCM-SHA384","DHE-DSS-AES256-SHA256","AES256-GCM-SHA384","AES256-SHA256","ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256","ECDHE-RSA-AES128-SHA256","ECDH-ECDSA-AES128-GCM-SHA256","ECDH-RSA-AES128-GCM-SHA256","ECDH-ECDSA-AES128-SHA256","ECDH-RSA-AES128-SHA256","DHE-DSS-AES128-GCM-SHA256","DHE-DSS-AES128-SHA256","AES128-GCM-SHA256","AES128-SHA256","ECDHE-ECDSA-AES256-SHA","ECDHE-RSA-AES256-SHA","DHE-DSS-AES256-SHA","ECDH-ECDSA-AES256-SHA","ECDH-RSA-AES256-SHA","ECDHE-ECDSA-AES128-SHA","ECDHE-RSA-AES128-SHA","DHE-DSS-AES128-SHA","ECDH-ECDSA-AES128-SHA","ECDH-RSA-AES128-SHA","RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384","RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256","RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA"]},{client_renegotiation,true},{depth,10},{fail_if_no_peer_cert,false},{gc_after_handshake,false},{honor_cipher_order,true},{reuse_sessions,true},{secure_renegotiate,true},{user_lookup_fun,{fun emqx_tls_psk:lookup/3,undefined}},{verify,verify_none},{versions,['tlsv1.3','tlsv1.2','tlsv1.1',tlsv1]}]},{tune_fun,{emqx_olp,backoff_new_conn,[default]}}],{emqx_connection,start_link,[#{enable_authn => true,limiter => #{bytes_in => #{capacity => 1099511627776,initial => 0,rate => infinity},client => #{bytes_in => #{capacity => 1099511627776,divisible => false,failure_strategy => force,initial => 0,low_watermark => 0,max_retry_time => 10000,rate => infinity},connection => #{capacity => 1099511627776,divisible => false,failure_strategy => force,initial => 0,low_watermark => 0,max_retry_time => 10000,rate => infinity},message_in => #{capacity => 1099511627776,divisible => false,failure_strategy => force,initial => 0,low_watermark => 0,max_retry_time => 10000,rate => infinity},message_routing => #{capacity => 1099511627776,divisible => false,failure_strategy => force,initial => 0,low_watermark => 0,max_retry_time => 10000,rate => infinity}},connection => #{capacity => 1000,initial => 0,rate => 100.0},message_in => #{capacity => 1099511627776,initial => 0,rate => infinity},message_routing => #{capacity => 1099511627776,initial => 0,rate => infinity}},listener => {ssl,certfile},zone => default}]}]},transient,false,infinity,supervisor,[esockd_listener_sup]}}.

2022-10-25T18:47:04.347604973+08:00 2022-10-25T10:47:04.347156+00:00 [error] crasher: initial call: application_master:init/4, pid: <0.2072.0>, registered_name: [], exit: {{bad_return,{{emqx_app,start,[normal,[]]},{'EXIT',{{failed_to_start,"ssl:certfile(:8883) : eaddrinuse"},[{emqx_listeners,'-foreach_listeners/1-fun-0-',2,[{file,"emqx_listeners.erl"},{line,601}]},{lists,foreach,2,[{file,"lists.erl"},{line,1342}]},{emqx_app,maybe_start_listeners,0,[{file,"emqx_app.erl"},{line,90}]},{emqx_app,start,2,[{file,"emqx_app.erl"},{line,50}]},{application_master,start_it_old,4,[{file,"application_master.erl"},{line,293}]}]}}}},[{application_master,init,4,[{file,"application_master.erl"},{line,142}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [<0.2071.0>], message_queue_len: 1, messages: [{'EXIT',<0.2073.0>,normal}], links: [<0.2071.0>,<0.1685.0>], dictionary: [], trap_exit: true, status: running, heap_size: 987, stack_size: 29, reductions: 160; neighbours:

2022-10-25T18:47:04.421114098+08:00 [os_mon] memory supervisor port (memsup): Erlang has closed

配置文件

[root@k8s-m1 emqx]# cat emqx.yaml 
apiVersion: apps.emqx.io/v2alpha1
kind: EMQX
metadata:
  name: emqx
spec:
  image: g6/emqx:5.0.8
  imagePullPolicy: IfNotPresent
  bootstrapConfig: |
    listeners.ssl {
        keyfile.ssl_options {
            keyfile = "etc/certs/mqtt_server.key"
        }, 
        certfile.ssl_options {
            certfile = "etc/certs/mqtt_server.crt"
        }, 
        cacertfile.ssl_options {
            cacertfile = "etc/certs/server_ca.crt"
        }, 
        verify.ssl_options {
            verify = verify_peer
        }, 
        fail_if_no_peer_cert.ssl_options {
            fail_if_no_peer_cert = true
        }
    }

看起来是 8883 端口占用导致无法启动:

failed to listen on 8883 - eaddrinuse (address already in use)

是需要更换一个端口吗,不能使用默认的8883?

可以给一个配置ssl证书的示例吗

证书配置没有问题,也不需要换端口,排查看是不是其他原因占用了 8883 端口

这个会是cacertfile 占用了这个端口吗,我看日志cacertfile 在8883启动,但是certfile就提示8883被占用了
Listener ssl:cacertfile on :8883 started.
[error] ssl:certfile failed to listen on 8883 - eaddrinuse (address already in use)

我把这条配置删除后,报错信息出现了变化
cacertfile.ssl_options {
cacertfile = “etc/certs/server_ca.crt”
},

Listener ssl:certfile on :8883 started.
[error] ssl:fail_if_no_peer_cert failed to listen on 8883 - eaddrinuse (address already in use)

[root@k8s-m1 emqx]# cat emqx.yaml 
apiVersion: apps.emqx.io/v2alpha1
kind: EMQX
metadata:
  name: emqx
spec:
  image: g6/emqx:5.0.8
  imagePullPolicy: IfNotPresent
  imagePullSecrets: 
    - name: huawei-token
  bootstrapConfig: |
        listeners.ssl.default {
            bind = "0.0.0.0:8883"
            ssl_options {
                keyfile = "etc/certs/mqtt_server.key"
                certfile = "etc/certs/mqtt_server.crt"
                cacertfile = "etc/certs/server_ca.crt"
                verify = verify_peer
                fail_if_no_peer_cert = true
            } 
        }

修改配置文件格式后解决