环境
- CentOS 8.1.1911 x86_64(Py3.7.4)
- EMQX 5.0.4
背景
- 2022年左右部署的服务,配置的开机启动项。至今基本没有出现问题,中间服务器也重启过几次,不过开机后服务都可以正常启动。
- 今天突然发现服务挂了,服务器是前两天刚重启过。估计是开机启动没有起起来。手动执行指令出现下面问题。
问题
- 使用emqx start指令启动,失败,无报错:
- 使用env DEBUG=1 /usr/bin/emqx console,则可以正常启动,并正常访问emqx的dashboard
无warning信息,有notice信息:
日志文件夹/var/log/emqx,也没有任何文件
这个是什么问题呢?求解答。
下面是执行指令env DEBUG=1 /usr/bin/emqx console
的输出内容
++++ readlink /usr/bin/emqx
+++ dirname /usr/lib/emqx/bin/emqx
++ cd /usr/lib/emqx/bin/..
++ pwd -P
+ RUNNER_ROOT_DIR=/usr/lib/emqx
+ . /usr/lib/emqx/releases/emqx_vars
++ REL_VSN=5.0.4
++ ERTS_VSN=12.2.1
++ ERL_OPTS=
++ RUNNER_BIN_DIR=/usr/bin
++ RUNNER_LOG_DIR=/var/log/emqx
++ RUNNER_LIB_DIR=/usr/lib/emqx/lib
++ EMQX_ETC_DIR=/etc/emqx
++ RUNNER_USER=emqx
++ IS_ELIXIR=no
++ SCHEMA_MOD=emqx_conf_schema
++ IS_ENTERPRISE=no
++ export EMQX_DESCRIPTION=EMQX
++ EMQX_DESCRIPTION=EMQX
++ REL_NAME=emqx
+ export RUNNER_ROOT_DIR
+ export EMQX_ETC_DIR
+ export REL_VSN
+ export SCHEMA_MOD
+ RUNNER_SCRIPT=/usr/bin/emqx
+ CODE_LOADING_MODE=embedded
+ REL_DIR=/usr/lib/emqx/releases/5.0.4
++ whoami
+ WHOAMI=root
+ mkdir -p /var/log/emqx
+ export HOCON_ENV_OVERRIDE_PREFIX=EMQX_
+ HOCON_ENV_OVERRIDE_PREFIX=EMQX_
+ export ERTS_DIR=/usr/lib/emqx/erts-12.2.1
+ ERTS_DIR=/usr/lib/emqx/erts-12.2.1
+ export BINDIR=/usr/lib/emqx/erts-12.2.1/bin
+ BINDIR=/usr/lib/emqx/erts-12.2.1/bin
+ export EMU=beam
+ EMU=beam
+ export PROGNAME=erl
+ PROGNAME=erl
+ export ERTS_LIB_DIR=/usr/lib/emqx/lib
+ ERTS_LIB_DIR=/usr/lib/emqx/lib
+ DYNLIBS_DIR=/usr/lib/emqx/dynlibs
+ COMMAND=console
+ '[' -z console ']'
+ '[' console = help ']'
+ '[' '' = help ']'
+ case "${COMMAND}" in
+ IS_BOOT_COMMAND=yes
+ '[' -d /usr/lib/emqx/erts-12.2.1/lib ']'
+ check_user console
+ '[' emqx ']'
+ '[' xroot '!=' xemqx ']'
+ '[' xroot '!=' xroot ']'
+ CMD='DEBUG=1 "/usr/bin/emqx" '
+ for ARG in "$@"
+ CMD='DEBUG=1 "/usr/bin/emqx" "console"'
+ exec su - emqx -c 'DEBUG=1 "/usr/bin/emqx" "console"'
++++ readlink /usr/bin/emqx
+++ dirname /usr/lib/emqx/bin/emqx
++ cd /usr/lib/emqx/bin/..
++ pwd -P
+ RUNNER_ROOT_DIR=/usr/lib/emqx
+ . /usr/lib/emqx/releases/emqx_vars
++ REL_VSN=5.0.4
++ ERTS_VSN=12.2.1
++ ERL_OPTS=
++ RUNNER_BIN_DIR=/usr/bin
++ RUNNER_LOG_DIR=/var/log/emqx
++ RUNNER_LIB_DIR=/usr/lib/emqx/lib
++ EMQX_ETC_DIR=/etc/emqx
++ RUNNER_USER=emqx
++ IS_ELIXIR=no
++ SCHEMA_MOD=emqx_conf_schema
++ IS_ENTERPRISE=no
++ export EMQX_DESCRIPTION=EMQX
++ EMQX_DESCRIPTION=EMQX
++ REL_NAME=emqx
+ export RUNNER_ROOT_DIR
+ export EMQX_ETC_DIR
+ export REL_VSN
+ export SCHEMA_MOD
+ RUNNER_SCRIPT=/usr/bin/emqx
+ CODE_LOADING_MODE=embedded
+ REL_DIR=/usr/lib/emqx/releases/5.0.4
++ whoami
+ WHOAMI=emqx
+ mkdir -p /var/log/emqx
+ export HOCON_ENV_OVERRIDE_PREFIX=EMQX_
+ HOCON_ENV_OVERRIDE_PREFIX=EMQX_
+ export ERTS_DIR=/usr/lib/emqx/erts-12.2.1
+ ERTS_DIR=/usr/lib/emqx/erts-12.2.1
+ export BINDIR=/usr/lib/emqx/erts-12.2.1/bin
+ BINDIR=/usr/lib/emqx/erts-12.2.1/bin
+ export EMU=beam
+ EMU=beam
+ export PROGNAME=erl
+ PROGNAME=erl
+ export ERTS_LIB_DIR=/usr/lib/emqx/lib
+ ERTS_LIB_DIR=/usr/lib/emqx/lib
+ DYNLIBS_DIR=/usr/lib/emqx/dynlibs
+ COMMAND=console
+ '[' -z console ']'
+ '[' console = help ']'
+ '[' '' = help ']'
+ case "${COMMAND}" in
+ IS_BOOT_COMMAND=yes
+ '[' -d /usr/lib/emqx/erts-12.2.1/lib ']'
+ check_user console
+ '[' emqx ']'
+ '[' xemqx '!=' xemqx ']'
+ ES=0
+ '[' 0 -ne 0 ']'
+ COMPATIBILITY_CHECK='
io:format("BEAM_OK~n", []),
try
[_|_] = L = crypto:info_lib(),
io:format("CRYPTO_OK ~0p~n", [L])
catch
_ : _ ->
%% so logger has the chance to log something
timer:sleep(100),
halt(1)
end,
try
mnesia_hook:module_info(),
io:format("MNESIA_OK~n", [])
catch
_ : _ ->
io:format("WARNING: Mnesia app has no post-coommit hook support~n", []),
halt(2)
end,
halt(0).
'
+ '[' yes = yes ']'
++ cat /usr/lib/emqx/releases/5.0.4/BUILD_INFO
+ BUILD_INFO='arch: "x86_64-pc-linux-gnu"
wordsize: 64
os: "el8"
erlang: "24.2.1-1"
elixir: "none"
relform: "rpm"'
++ compatiblity_info
+ COMPATIBILITY_INFO='BEAM_OK
CRYPTO_OK [{<<"OpenSSL">>,269488319,<<"OpenSSL 1.1.1k FIPS 25 Mar 2021">>}]
MNESIA_OK'
+ echo -e 'BEAM_OK
CRYPTO_OK [{<<"OpenSSL">>,269488319,<<"OpenSSL 1.1.1k FIPS 25 Mar 2021">>}]
MNESIA_OK'
+ grep -q CRYPTO_OK
+ '[' 1 -eq 1 ']'
+ set -x
++ ulimit -n
+ ULIMIT_F=1000001
+ '[' 1000001 -lt 1024 ']'
+ SED_REPLACE='sed -i '
+ case $(sed --help 2>&1) in
++ sed --help
+ SED_REPLACE='sed -i '
+ CONF_KEYS=('node.data_dir' 'node.name' 'node.cookie' 'node.db_backend' 'cluster.proto_dist')
+ '[' no = yes ']'
+ '[' yes = yes ']'
+ '[' '' = '' ']'
++ call_hocon -s emqx_conf_schema -c /etc/emqx/emqx.conf multi_get node.data_dir node.name node.cookie node.db_backend cluster.proto_dist
++ call_nodetool hocon -s emqx_conf_schema -c /etc/emqx/emqx.conf multi_get node.data_dir node.name node.cookie node.db_backend cluster.proto_dist
++ /usr/lib/emqx/erts-12.2.1/bin/escript /usr/lib/emqx/bin/nodetool hocon -s emqx_conf_schema -c /etc/emqx/emqx.conf multi_get node.data_dir node.name node.cookie node.db_backend cluster.proto_dist
[debug]: ConfFiles: {["/etc/emqx/emqx.conf"],[]}
+ EMQX_BOOT_CONFIGS='node.data_dir="/var/lib/emqx"
node.name="emqx@127.0.0.1"
node.cookie="emqxsecretcookie"
node.db_backend=rlog
cluster.proto_dist=inet_tcp'
+ export EMQX_BOOT_CONFIGS
+ EPMD_ARGS='-start_epmd false -epmd_module ekka_epmd -proto_dist ekka'
++ get_boot_config cluster.proto_dist
++ path_to_value=cluster.proto_dist
++ echo -e 'node.data_dir="/var/lib/emqx"
node.name="emqx@127.0.0.1"
node.cookie="emqxsecretcookie"
node.db_backend=rlog
cluster.proto_dist=inet_tcp'
++ grep cluster.proto_dist=
++ tr -d '"'
++ sed -e s/cluster.proto_dist=//g
+ PROTO_DIST=inet_tcp
+ export EKKA_PROTO_DIST_MOD=inet_tcp
+ EKKA_PROTO_DIST_MOD=inet_tcp
+ '[' inet_tcp = inet_tls ']'
++ get_boot_config node.data_dir
++ path_to_value=node.data_dir
++ echo -e 'node.data_dir="/var/lib/emqx"
node.name="emqx@127.0.0.1"
node.cookie="emqxsecretcookie"
node.db_backend=rlog
cluster.proto_dist=inet_tcp'
++ grep node.data_dir=
++ tr -d '"'
++ sed -e s/node.data_dir=//g
+ DATA_DIR=/var/lib/emqx
+ DATA_DIR=/var/lib/emqx
+ [[ /var/lib/emqx != /* ]]
+ CONFIGS_DIR=/var/lib/emqx/configs
+ mkdir -p /var/lib/emqx/configs
+ '[' -n '' ']'
+ NAME=
+ '[' -z '' ']'
+ '[' yes = yes ']'
++ get_boot_config node.name
++ path_to_value=node.name
++ echo -e 'node.data_dir="/var/lib/emqx"
node.name="emqx@127.0.0.1"
node.cookie="emqxsecretcookie"
node.db_backend=rlog
cluster.proto_dist=inet_tcp'
++ grep node.name=
++ tr -d '"'
++ sed -e s/node.name=//g
+ NAME=emqx@127.0.0.1
+ '[' -z emqx@127.0.0.1 ']'
+ case "$NAME" in
+ NAME_TYPE=-name
++ echo emqx@127.0.0.1
++ awk -F@ '{print $1}'
+ SHORT_NAME=emqx
+ echo emqx
+ grep -q '^[0-9A-Za-z_\-]\+
`
+ export ESCRIPT_NAME=emqx
+ ESCRIPT_NAME=emqx
+ PIPE_DIR=//var/lib/emqx/emqx_erl_pipes/emqx@127.0.0.1/
+ '[' -n '' ']'
+ COOKIE=
+ '[' -z '' ']'
+ '[' yes = yes ']'
++ get_boot_config node.cookie
++ path_to_value=node.cookie
++ grep node.cookie=
++ echo -e 'node.data_dir="/var/lib/emqx"
node.name="emqx@127.0.0.1"
node.cookie="emqxsecretcookie"
node.db_backend=rlog
cluster.proto_dist=inet_tcp'
++ sed -e s/node.cookie=//g
++ tr -d '"'
+ COOKIE=emqxsecretcookie
+ '[' -z emqxsecretcookie ']'
+ [[ yes == \y\e\s ]]
++ get_boot_config node.db_backend
++ path_to_value=node.db_backend
++ echo -e 'node.data_dir="/var/lib/emqx"
node.name="emqx@127.0.0.1"
node.cookie="emqxsecretcookie"
node.db_backend=rlog
cluster.proto_dist=inet_tcp'
++ grep node.db_backend=
++ sed -e s/node.db_backend=//g
++ tr -d '"'
+ [[ rlog == \r\l\o\g ]]
+ echo -e 'BEAM_OK
CRYPTO_OK [{<<"OpenSSL">>,269488319,<<"OpenSSL 1.1.1k FIPS 25 Mar 2021">>}]
MNESIA_OK'
+ grep -q MNESIA_OK
+ cd /usr/lib/emqx
+ case "${COMMAND}" in
+ case "$COMMAND" in
+ '[' -f /usr/lib/emqx/releases/5.0.4/emqx.boot ']'
+ BOOTFILE=/usr/lib/emqx/releases/5.0.4/start
+ case "$COMMAND" in
+ FOREGROUNDOPTIONS=
+ '[' '' = 1 ']'
+ maybe_log_to_console
+ '[' '' = default ']'
+ tr_log_to_env
+ local log_to=undefined
+ unset EMQX_LOG__TO
+ case "${log_to}" in
+ export EMQX_LOG__CONSOLE_HANDLER__ENABLE=true
+ EMQX_LOG__CONSOLE_HANDLER__ENABLE=true
+ export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE=false
+ EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE=false
+ generate_config -name emqx@127.0.0.1
+ local name_type=-name
+ local node_name=emqx@127.0.0.1
+ rm -rf '/var/log/emqx/*.siz'
+ local NOW_TIME
++ date +%Y.%m.%d.%H.%M.%S
+ NOW_TIME=2024.03.18.17.10.35
+ call_hocon -v -t 2024.03.18.17.10.35 -s emqx_conf_schema -c /etc/emqx/emqx.conf -d /var/lib/emqx/configs generate
+ call_nodetool hocon -v -t 2024.03.18.17.10.35 -s emqx_conf_schema -c /etc/emqx/emqx.conf -d /var/lib/emqx/configs generate
+ /usr/lib/emqx/erts-12.2.1/bin/escript /usr/lib/emqx/bin/nodetool hocon -v -t 2024.03.18.17.10.35 -s emqx_conf_schema -c /etc/emqx/emqx.conf -d /var/lib/emqx/configs generate
log.file_handlers.default.enable = EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE = false
log.console_handler.enable = EMQX_LOG__CONSOLE_HANDLER__ENABLE = true
+ CONF_FILE=/var/lib/emqx/configs/app.2024.03.18.17.10.35.config
+ ARGS_FILE=/var/lib/emqx/configs/vm.2024.03.18.17.10.35.args
+ TMP_ARG_FILE=/var/lib/emqx/configs/vm.args.tmp
+ cp /etc/emqx/vm.args /var/lib/emqx/configs/vm.args.tmp
+ echo ''
+ echo '-pa "/usr/lib/emqx/releases/5.0.4/consolidated"'
+ sed '/^$/d'
+ sed '/^#/d' /var/lib/emqx/configs/vm.2024.03.18.17.10.35.args
+ IFS=
+ read -r ARG_LINE
++ echo '-kernel net_ticktime 120'
++ awk '{$NF="";print}'
+ ARG_KEY='-kernel net_ticktime '
++ echo '-kernel net_ticktime 120'
++ awk '{print $NF}'
+ ARG_VALUE=120
++ grep '^-kernel net_ticktime ' /var/lib/emqx/configs/vm.args.tmp
+ TMP_ARG_VALUE='-kernel net_ticktime 120'
+ '[' 120 '!=' '-kernel net_ticktime 120' ']'
+ '[' -n '-kernel net_ticktime 120' ']'
+ sh -c 'sed -i '\''s|^-kernel net_ticktime .*$|-kernel net_ticktime 120|'\'' "/var/lib/emqx/configs/vm.args.tmp"'
+ IFS=
+ read -r ARG_LINE
++ echo '-env ERL_CRASH_DUMP_BYTES 104857600'
++ awk '{$NF="";print}'
+ ARG_KEY='-env ERL_CRASH_DUMP_BYTES '
++ echo '-env ERL_CRASH_DUMP_BYTES 104857600'
++ awk '{print $NF}'
+ ARG_VALUE=104857600
++ grep '^-env ERL_CRASH_DUMP_BYTES ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' 104857600 '!=' '' ']'
+ '[' -n '' ']'
+ echo '-env ERL_CRASH_DUMP_BYTES 104857600'
+ IFS=
+ read -r ARG_LINE
++ echo '-env ERL_CRASH_DUMP_SECONDS 30'
++ awk '{$NF="";print}'
+ ARG_KEY='-env ERL_CRASH_DUMP_SECONDS '
++ echo '-env ERL_CRASH_DUMP_SECONDS 30'
++ awk '{print $NF}'
+ ARG_VALUE=30
++ grep '^-env ERL_CRASH_DUMP_SECONDS ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' 30 '!=' '' ']'
+ '[' -n '' ']'
+ echo '-env ERL_CRASH_DUMP_SECONDS 30'
+ IFS=
+ read -r ARG_LINE
++ echo '-env ERL_CRASH_DUMP log/erl_crash.dump'
++ awk '{$NF="";print}'
+ ARG_KEY='-env ERL_CRASH_DUMP '
++ echo '-env ERL_CRASH_DUMP log/erl_crash.dump'
++ awk '{print $NF}'
+ ARG_VALUE=log/erl_crash.dump
++ grep '^-env ERL_CRASH_DUMP ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' log/erl_crash.dump '!=' '' ']'
+ '[' -n '' ']'
+ echo '-env ERL_CRASH_DUMP log/erl_crash.dump'
+ IFS=
+ read -r ARG_LINE
++ echo '+e 262144'
++ awk '{$NF="";print}'
+ ARG_KEY='+e '
++ echo '+e 262144'
++ awk '{print $NF}'
+ ARG_VALUE=262144
++ grep '^+e ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' 262144 '!=' '' ']'
+ '[' -n '' ']'
+ echo '+e 262144'
+ IFS=
+ read -r ARG_LINE
++ echo '+zdbbl 8192'
++ awk '{$NF="";print}'
+ ARG_KEY='+zdbbl '
++ echo '+zdbbl 8192'
++ awk '{print $NF}'
+ ARG_VALUE=8192
++ grep '^+zdbbl ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' 8192 '!=' '' ']'
+ '[' -n '' ']'
+ echo '+zdbbl 8192'
+ IFS=
+ read -r ARG_LINE
++ echo '+Q 1048576'
++ awk '{$NF="";print}'
+ ARG_KEY='+Q '
++ echo '+Q 1048576'
++ awk '{print $NF}'
+ ARG_VALUE=1048576
++ grep '^+Q ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' 1048576 '!=' '' ']'
+ '[' -n '' ']'
+ echo '+Q 1048576'
+ IFS=
+ read -r ARG_LINE
++ echo '+P 2097152'
++ awk '{$NF="";print}'
+ ARG_KEY='+P '
++ echo '+P 2097152'
++ awk '{print $NF}'
+ ARG_VALUE=2097152
++ grep '^+P ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' 2097152 '!=' '' ']'
+ '[' -n '' ']'
+ echo '+P 2097152'
+ IFS=
+ read -r ARG_LINE
+ '[' -n '-setcookie emqxsecretcookie' ']'
++ echo '-setcookie emqxsecretcookie'
++ awk '{$NF="";print}'
+ ARG_KEY='-setcookie '
++ echo '-setcookie emqxsecretcookie'
++ awk '{print $NF}'
+ ARG_VALUE=emqxsecretcookie
++ grep '^-setcookie ' /var/lib/emqx/configs/vm.args.tmp
++ true
++ awk '{print $NF}'
+ TMP_ARG_VALUE=
+ '[' emqxsecretcookie '!=' '' ']'
+ '[' -n '' ']'
+ echo '-setcookie emqxsecretcookie'
+ IFS=
+ read -r ARG_LINE
+ '[' -n '' ']'
+ echo '-name emqx@127.0.0.1'
+ echo '-mnesia dir '\''"/var/lib/emqx/mnesia/emqx@127.0.0.1"'\'''
+ mv -f /var/lib/emqx/configs/vm.args.tmp /var/lib/emqx/configs/vm.2024.03.18.17.10.35.args
+ check_license
+ '[' no == no ']'
+ return 0
+ EMU=beam
+ PROGNAME=/usr/bin/emqx
+ export EMU
+ export PROGNAME
+ ARGS=console
+ '[' no = no ']'
+ set -- /usr/lib/emqx/erts-12.2.1/bin/erlexec -boot /usr/lib/emqx/releases/5.0.4/start -boot_var RELEASE_LIB /usr/lib/emqx/lib -boot_var ERTS_LIB_DIR /usr/lib/emqx/lib -mode embedded -config /var/lib/emqx/configs/app.2024.03.18.17.10.35.config -args_file /var/lib/emqx/configs/vm.2024.03.18.17.10.35.args -start_epmd false -epmd_module ekka_epmd -proto_dist ekka
+ logger -t 'emqx[15963]' 'EXEC: /usr/lib/emqx/erts-12.2.1/bin/erlexec -boot /usr/lib/emqx/releases/5.0.4/start -boot_var RELEASE_LIB /usr/lib/emqx/lib -boot_var ERTS_LIB_DIR /usr/lib/emqx/lib -mode embedded -config /var/lib/emqx/configs/app.2024.03.18.17.10.35.config -args_file /var/lib/emqx/configs/vm.2024.03.18.17.10.35.args -start_epmd false -epmd_module ekka_epmd -proto_dist ekka -- console -emqx_data_dir /var/lib/emqx'
+ exec /usr/lib/emqx/erts-12.2.1/bin/erlexec -boot /usr/lib/emqx/releases/5.0.4/start -boot_var RELEASE_LIB /usr/lib/emqx/lib -boot_var ERTS_LIB_DIR /usr/lib/emqx/lib -mode embedded -config /var/lib/emqx/configs/app.2024.03.18.17.10.35.config -args_file /var/lib/emqx/configs/vm.2024.03.18.17.10.35.args -start_epmd false -epmd_module ekka_epmd -proto_dist ekka -- console -emqx_data_dir /var/lib/emqx
Erlang/OTP 24 [erts-12.2.1] [emqx] [64-bit] [smp:4:4] [ds:4:4:8] [async-threads:4] [jit]
Listener ssl:default on :8883 started.
Listener tcp:default on :1883 started.
Listener ws:default on :8083 started.
Listener wss:default on :8084 started.
Listener http:dashboard on :18083 started.
EMQX 5.0.4 is running now!
Restricted Eshell V12.2.1 (abort with ^G)
这日志也没看出来有才能异常,如果 console 能启动的话。
和你分析的差不多。
应该是systemd 启动失败了。
就用systemctl来启动,然后查看他的 log。
结合
journalctl -p err
或 dmesg | grep error