diff --git a/dracut-early-kdump-module-setup.sh b/dracut-early-kdump-module-setup.sh index f30bd67..3b23383 100755 --- a/dracut-early-kdump-module-setup.sh +++ b/dracut-early-kdump-module-setup.sh @@ -48,7 +48,10 @@ install() { inst_simple "/etc/sysconfig/kdump" inst_binary "/usr/sbin/kexec" inst_binary "/usr/bin/gawk" "/usr/bin/awk" + inst_binary "/usr/bin/logger" "/usr/bin/logger" + inst_binary "/usr/bin/printf" "/usr/bin/printf" inst_script "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh" + inst_script "/lib/kdump/kdump-logger.sh" "/lib/kdump-logger.sh" inst_hook cmdline 00 "$moddir/early-kdump.sh" inst_binary "$KDUMP_KERNEL" inst_binary "$KDUMP_INITRD" diff --git a/dracut-early-kdump.sh b/dracut-early-kdump.sh index 92913fb..23bb369 100755 --- a/dracut-early-kdump.sh +++ b/dracut-early-kdump.sh @@ -13,6 +13,13 @@ EARLY_KEXEC_ARGS="" . /lib/dracut-lib.sh . /lib/kdump-lib.sh +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + prepare_parameters() { EARLY_KDUMP_CMDLINE=$(prepare_cmdline "${KDUMP_COMMANDLINE}" "${KDUMP_COMMANDLINE_REMOVE}" "${KDUMP_COMMANDLINE_APPEND}") @@ -28,7 +35,7 @@ early_kdump_load() fi if is_fadump_capable; then - echo "WARNING: early kdump doesn't support fadump." + dwarn "WARNING: early kdump doesn't support fadump." return 1 fi @@ -42,18 +49,22 @@ early_kdump_load() EARLY_KEXEC_ARGS=$(prepare_kexec_args "${KEXEC_ARGS}") if is_secure_boot_enforced; then - echo "Secure Boot is enabled. Using kexec file based syscall." + dinfo "Secure Boot is enabled. Using kexec file based syscall." EARLY_KEXEC_ARGS="$EARLY_KEXEC_ARGS -s" fi + ddebug "earlykdump: $KEXEC ${EARLY_KEXEC_ARGS} $standard_kexec_args \ + --command-line=$EARLY_KDUMP_CMDLINE --initrd=$EARLY_KDUMP_INITRD \ + $EARLY_KDUMP_KERNEL" + $KEXEC ${EARLY_KEXEC_ARGS} $standard_kexec_args \ --command-line="$EARLY_KDUMP_CMDLINE" \ --initrd=$EARLY_KDUMP_INITRD $EARLY_KDUMP_KERNEL if [ $? == 0 ]; then - echo "kexec: loaded early-kdump kernel" + dinfo "kexec: loaded early-kdump kernel" return 0 else - echo "kexec: failed to load early-kdump kernel" + derror "kexec: failed to load early-kdump kernel" return 1 fi } @@ -61,10 +72,10 @@ early_kdump_load() set_early_kdump() { if getargbool 0 rd.earlykdump; then - echo "early-kdump is enabled." + dinfo "early-kdump is enabled." early_kdump_load else - echo "early-kdump is disabled." + dinfo "early-kdump is disabled." fi return 0 diff --git a/dracut-kdump.sh b/dracut-kdump.sh index 6f948fc..c2627c2 100755 --- a/dracut-kdump.sh +++ b/dracut-kdump.sh @@ -5,7 +5,6 @@ if [ -f /etc/fadump.initramfs ] && [ ! -f /proc/device-tree/rtas/ibm,kernel-dump exit 0 fi -exec &> /dev/console . /lib/dracut-lib.sh . /lib/kdump-lib-initramfs.sh @@ -22,7 +21,7 @@ do_dump() _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: saving vmcore failed" + derror "saving vmcore failed" fi return $_ret @@ -36,7 +35,7 @@ do_kdump_pre() "$KDUMP_PRE" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $KDUMP_PRE exited with $_ret status" + derror "$KDUMP_PRE exited with $_ret status" return $_ret fi fi @@ -47,7 +46,7 @@ do_kdump_pre() "$file" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $file exited with $_ret status" + derror "$file exited with $_ret status" fi done fi @@ -63,7 +62,7 @@ do_kdump_post() "$file" "$1" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $file exited with $_ret status" + derror "$file exited with $_ret status" fi done fi @@ -72,7 +71,7 @@ do_kdump_post() "$KDUMP_POST" "$1" _ret=$? if [ $_ret -ne 0 ]; then - echo "kdump: $KDUMP_POST exited with $_ret status" + derror "$KDUMP_POST exited with $_ret status" fi fi } @@ -88,7 +87,7 @@ dump_raw() [ -b "$_raw" ] || return 1 - echo "kdump: saving to raw disk $_raw" + dinfo "saving to raw disk $_raw" if ! $(echo -n $CORE_COLLECTOR|grep -q makedumpfile); then _src_size=`ls -l /proc/vmcore | cut -d' ' -f5` @@ -96,21 +95,22 @@ dump_raw() monitor_dd_progress $_src_size_mb & fi - echo "kdump: saving vmcore" + dinfo "saving vmcore" $CORE_COLLECTOR /proc/vmcore | dd of=$_raw bs=$DD_BLKSIZE >> /tmp/dd_progress_file 2>&1 || return 1 sync - echo "kdump: saving vmcore complete" + dinfo "saving vmcore complete" return 0 } dump_ssh() { + local ret local _opt="-i $1 -o BatchMode=yes -o StrictHostKeyChecking=yes" local _dir="$KDUMP_PATH/$HOST_IP-$DATEDIR" local _host=$2 - echo "kdump: saving to $_host:$_dir" + dinfo "saving to $_host:$_dir" cat /var/lib/random-seed > /dev/urandom ssh -q $_opt $_host mkdir -p $_dir || return 1 @@ -118,17 +118,29 @@ dump_ssh() save_vmcore_dmesg_ssh ${DMESG_COLLECTOR} ${_dir} "${_opt}" $_host save_opalcore_ssh ${_dir} "${_opt}" $_host - echo "kdump: saving vmcore" + dinfo "saving vmcore" if [ "${CORE_COLLECTOR%%[[:blank:]]*}" = "scp" ]; then - scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" || return 1 + scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" + ret=$? + save_log + scp -q $_opt $KDUMP_LOG_FILE "$_host:$_dir/" + if [ $ret -ne 0 ]; then + return 1 + fi ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore" || return 1 else - $CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" || return 1 + $CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" + ret=$? + save_log + scp -q $_opt $KDUMP_LOG_FILE "$_host:$_dir/" + if [ $ret -ne 0 ]; then + return 1 + fi ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore.flat" || return 1 fi - echo "kdump: saving vmcore complete" + dinfo "saving vmcore complete" return 0 } @@ -137,6 +149,8 @@ save_opalcore_ssh() { local _opts="$2" local _location=$3 + ddebug "_path=$_path _opts=$_opts _location=$_location" + if [ ! -f $OPALCORE ]; then # Check if we are on an old kernel that uses a different path if [ -f /sys/firmware/opal/core ]; then @@ -146,15 +160,15 @@ save_opalcore_ssh() { fi fi - echo "kdump: saving opalcore" + dinfo "saving opalcore:$OPALCORE to $_location:$_path" scp $_opts $OPALCORE $_location:$_path/opalcore-incomplete if [ $? -ne 0 ]; then - echo "kdump: saving opalcore failed" + derror "saving opalcore failed" return 1 fi ssh $_opts $_location mv $_path/opalcore-incomplete $_path/opalcore - echo "kdump: saving opalcore complete" + dinfo "saving opalcore complete" return 0 } @@ -164,15 +178,15 @@ save_vmcore_dmesg_ssh() { local _opts="$3" local _location=$4 - echo "kdump: saving vmcore-dmesg.txt" + dinfo "saving vmcore-dmesg.txt to $_location:$_path" $_dmesg_collector /proc/vmcore | ssh $_opts $_location "dd of=$_path/vmcore-dmesg-incomplete.txt" _exitcode=$? if [ $_exitcode -eq 0 ]; then ssh -q $_opts $_location mv $_path/vmcore-dmesg-incomplete.txt $_path/vmcore-dmesg.txt - echo "kdump: saving vmcore-dmesg.txt complete" + dinfo "saving vmcore-dmesg.txt complete" else - echo "kdump: saving vmcore-dmesg.txt failed" + derror "saving vmcore-dmesg.txt failed" fi } @@ -182,12 +196,12 @@ get_host_ip() if is_nfs_dump_target || is_ssh_dump_target then kdumpnic=$(getarg kdumpnic=) - [ -z "$kdumpnic" ] && echo "kdump: failed to get kdumpnic!" && return 1 + [ -z "$kdumpnic" ] && derror "failed to get kdumpnic!" && return 1 _host=`ip addr show dev $kdumpnic|grep '[ ]*inet'` - [ $? -ne 0 ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1 + [ $? -ne 0 ] && derror "wrong kdumpnic: $kdumpnic" && return 1 _host=`echo $_host | head -n 1 | cut -d' ' -f2` _host="${_host%%/*}" - [ -z "$_host" ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1 + [ -z "$_host" ] && derror "wrong kdumpnic: $kdumpnic" && return 1 HOST_IP=$_host fi return 0 @@ -196,7 +210,7 @@ get_host_ip() read_kdump_conf() { if [ ! -f "$KDUMP_CONF" ]; then - echo "kdump: $KDUMP_CONF not found" + derror "$KDUMP_CONF not found" return fi @@ -240,7 +254,7 @@ fence_kdump_notify get_host_ip if [ $? -ne 0 ]; then - echo "kdump: get_host_ip exited with non-zero status!" + derror "get_host_ip exited with non-zero status!" exit 1 fi @@ -250,7 +264,7 @@ fi do_kdump_pre if [ $? -ne 0 ]; then - echo "kdump: kdump_pre script exited with non-zero status!" + derror "kdump_pre script exited with non-zero status!" do_final_action # During systemd service to reboot the machine, stop this shell script running exit 1 @@ -261,7 +275,7 @@ DUMP_RETVAL=$? do_kdump_post $DUMP_RETVAL if [ $? -ne 0 ]; then - echo "kdump: kdump_post script exited with non-zero status!" + derror "kdump_post script exited with non-zero status!" fi if [ $DUMP_RETVAL -ne 0 ]; then diff --git a/dracut-module-setup.sh b/dracut-module-setup.sh index a186570..911a8b8 100755 --- a/dracut-module-setup.sh +++ b/dracut-module-setup.sh @@ -799,13 +799,14 @@ kdump_install_systemd_conf() { echo "DefaultTimeoutStartSec=300s" >> ${initdir}/etc/systemd/system.conf.d/kdump.conf fi - # Forward logs to console directly, this avoids unneccessary memory - # consumption and make console output more useful. + # Forward logs to console directly, and don't read Kmsg, this avoids + # unneccessary memory consumption and make console output more useful. # Only do so for non fadump image. - if ! is_fadump_capable && [ "$failure_action" != "shell" ]; then + if ! is_fadump_capable; then mkdir -p ${initdir}/etc/systemd/journald.conf.d echo "[Journal]" > ${initdir}/etc/systemd/journald.conf.d/kdump.conf - echo "Storage=none" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "Storage=volatile" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf + echo "ReadKMsg=no" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf echo "ForwardToConsole=yes" >> ${initdir}/etc/systemd/journald.conf.d/kdump.conf fi } @@ -818,6 +819,7 @@ install() { kdump_install_random_seed fi dracut_install -o /etc/adjtime /etc/localtime + inst_simple "/etc/sysconfig/kdump" inst "$moddir/monitor_dd_progress" "/kdumpscripts/monitor_dd_progress" chmod +x ${initdir}/kdumpscripts/monitor_dd_progress inst "/bin/dd" "/bin/dd" @@ -830,8 +832,11 @@ install() { inst "/bin/sed" "/bin/sed" inst "/sbin/makedumpfile" "/sbin/makedumpfile" inst "/sbin/vmcore-dmesg" "/sbin/vmcore-dmesg" + inst "/usr/bin/printf" "/sbin/printf" + inst "/usr/bin/logger" "/sbin/logger" inst "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh" inst "/lib/kdump/kdump-lib-initramfs.sh" "/lib/kdump-lib-initramfs.sh" + inst "/lib/kdump/kdump-logger.sh" "/lib/kdump-logger.sh" inst "$moddir/kdump.sh" "/usr/bin/kdump.sh" inst "$moddir/kdump-capture.service" "$systemdsystemunitdir/kdump-capture.service" mkdir -p "$initdir/$systemdsystemunitdir/initrd.target.wants" diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh index c98a43b..14aac7b 100755 --- a/kdump-lib-initramfs.sh +++ b/kdump-lib-initramfs.sh @@ -1,10 +1,12 @@ # These variables and functions are useful in 2nd kernel +. /etc/sysconfig/kdump . /lib/kdump-lib.sh KDUMP_PATH="/var/crash" +KDUMP_LOG_FILE="/run/initramfs/kexec-dmesg.log" CORE_COLLECTOR="" -DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" +DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 7 -d 31" DMESG_COLLECTOR="/sbin/vmcore-dmesg" FAILURE_ACTION="systemctl reboot -f" DATEDIR=`date +%Y-%m-%d-%T` @@ -20,6 +22,13 @@ KDUMP_POST="" NEWROOT="/sysroot" OPALCORE="/sys/firmware/opal/mpipl/core" +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + get_kdump_confs() { local config_opt config_val @@ -94,27 +103,40 @@ get_kdump_confs() fi } +# store the kexec kernel log to a file. +save_log() +{ + dmesg -T > $KDUMP_LOG_FILE + + if command -v journalctl > /dev/null; then + journalctl -ab >> $KDUMP_LOG_FILE + fi +} + # dump_fs dump_fs() { + local ret local _mp=$1 local _dev=$(get_mount_info SOURCE target $_mp -f) local _op=$(get_mount_info OPTIONS target $_mp -f) + ddebug "_mp=$_mp _dev=$_dev _op=$_op" + # If dump path have a corresponding device entry but not mounted, mount it. if [ -n "$_dev" ]; then if ! is_mounted "$_mp"; then - echo "kdump: dump target $_dev is not mounted, trying to mount..." + dinfo "dump target $_dev is not mounted, trying to mount..." mkdir -p $_mp mount -o $_op $_dev $_mp if [ $? -ne 0 ]; then - echo "kdump: mounting failed (mount point: $_mp, option: $_op)" + derror "mounting failed (mount point: $_mp, option: $_op)" return 1 fi fi else - echo "kdump: failed to dump to \"$_mp\", it's not a mount point!" + derror "failed to dump to \"$_mp\", it's not a mount point!" return 1 fi @@ -123,11 +145,11 @@ dump_fs() local _dump_path=$(echo "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" | tr -s /) - echo "kdump: saving to $_dump_path" + dinfo "saving to $_dump_path" # Only remount to read-write mode if the dump target is mounted read-only. if [[ "$_op" = "ro"* ]]; then - echo "kdump: Mounting Dump target $_dev in rw mode." + dinfo "Mounting Dump target $_dev in rw mode." mount -o remount,rw $_dev $_mp || return 1 fi @@ -136,12 +158,18 @@ dump_fs() save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_dump_path" save_opalcore_fs "$_dump_path" - echo "kdump: saving vmcore" - $CORE_COLLECTOR /proc/vmcore $_dump_path/vmcore-incomplete || return 1 + dinfo "saving vmcore" + $CORE_COLLECTOR /proc/vmcore $_dump_path/vmcore-incomplete + ret=$? + save_log + mv $KDUMP_LOG_FILE $_dump_path/ + if [ $ret -ne 0 ]; then + return 1 + fi mv $_dump_path/vmcore-incomplete $_dump_path/vmcore sync - echo "kdump: saving vmcore complete" + dinfo "saving vmcore complete" # improper kernel cmdline can cause the failure of echo, we can ignore this kind of failure return 0 @@ -151,7 +179,7 @@ save_vmcore_dmesg_fs() { local _dmesg_collector=$1 local _path=$2 - echo "kdump: saving vmcore-dmesg.txt" + dinfo "saving vmcore-dmesg.txt to ${_path}" $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt _exitcode=$? if [ $_exitcode -eq 0 ]; then @@ -161,9 +189,9 @@ save_vmcore_dmesg_fs() { # saving vmcore failed and system rebooted without sync and there # was no vmcore-dmesg.txt available. sync - echo "kdump: saving vmcore-dmesg.txt complete" + dinfo "saving vmcore-dmesg.txt complete" else - echo "kdump: saving vmcore-dmesg.txt failed" + derror "saving vmcore-dmesg.txt failed" fi } @@ -179,43 +207,47 @@ save_opalcore_fs() { fi fi - echo "kdump: saving opalcore" + dinfo "saving opalcore:$OPALCORE to ${_path}/opalcore" cp $OPALCORE ${_path}/opalcore if [ $? -ne 0 ]; then - echo "kdump: saving opalcore failed" + derror "saving opalcore failed" return 1 fi sync - echo "kdump: saving opalcore complete" + dinfo "saving opalcore complete" return 0 } dump_to_rootfs() { - echo "Kdump: trying to bring up rootfs device" + dinfo "Trying to bring up rootfs device" systemctl start dracut-initqueue - echo "Kdump: waiting for rootfs mount, will timeout after 90 seconds" + dinfo "Waiting for rootfs mount, will timeout after 90 seconds" systemctl start sysroot.mount + ddebug "NEWROOT=$NEWROOT" + dump_fs $NEWROOT } kdump_emergency_shell() { echo "PS1=\"kdump:\\\${PWD}# \"" >/etc/profile + ddebug "Switching to dracut emergency..." /bin/dracut-emergency rm -f /etc/profile } do_failure_action() { - echo "Kdump: Executing failure action $FAILURE_ACTION" + dinfo "Executing failure action $FAILURE_ACTION" eval $FAILURE_ACTION } do_final_action() { + dinfo "Executing final action $FINAL_ACTION" eval $FINAL_ACTION } diff --git a/kdump-lib.sh b/kdump-lib.sh index e18c473..b55d987 100755 --- a/kdump-lib.sh +++ b/kdump-lib.sh @@ -8,6 +8,12 @@ FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump" FENCE_KDUMP_SEND="/usr/libexec/fence_kdump_send" FADUMP_ENABLED_SYS_NODE="/sys/kernel/fadump_enabled" +if [ -f /lib/kdump/kdump-logger.sh ]; then + . /lib/kdump/kdump-logger.sh +elif [ -f /lib/kdump-logger.sh ]; then + . /lib/kdump-logger.sh +fi + is_fadump_capable() { # Check if firmware-assisted dump is enabled @@ -20,14 +26,10 @@ is_fadump_capable() } perror_exit() { - echo $@ >&2 + derror "$@" exit 1 } -perror() { - echo $@ >&2 -} - is_fs_type_nfs() { [ "$1" = "nfs" ] || [ "$1" = "nfs4" ] @@ -503,7 +505,7 @@ check_crash_mem_reserved() mem_reserved=$(cat /sys/kernel/kexec_crash_size) if [ $mem_reserved -eq 0 ]; then - echo "No memory reserved for crash kernel" + derror "No memory reserved for crash kernel" return 1 fi @@ -513,7 +515,7 @@ check_crash_mem_reserved() check_kdump_feasibility() { if [ ! -e /sys/kernel/kexec_crash_loaded ]; then - echo "Kdump is not supported on this kernel" + derror "Kdump is not supported on this kernel" return 1 fi check_crash_mem_reserved @@ -523,7 +525,7 @@ check_kdump_feasibility() check_current_kdump_status() { if [ ! -f /sys/kernel/kexec_crash_loaded ];then - echo "Perhaps CONFIG_CRASH_DUMP is not enabled in kernel" + derror "Perhaps CONFIG_CRASH_DUMP is not enabled in kernel" return 1 fi @@ -612,6 +614,7 @@ is_secure_boot_enforced() return 0 fi + # Detect secure boot on x86 and arm64 secure_boot_file=$(find /sys/firmware/efi/efivars -name SecureBoot-* 2>/dev/null) setup_mode_file=$(find /sys/firmware/efi/efivars -name SetupMode-* 2>/dev/null) @@ -624,6 +627,11 @@ is_secure_boot_enforced() fi fi + # Detect secure boot on s390x + if [[ -e "/sys/firmware/ipl/secure" && "$(cat /sys/firmware/ipl/secure)" == "1" ]]; then + return 0 + fi + return 1 } @@ -645,8 +653,7 @@ prepare_kexec_args() found_elf_args=`echo $kexec_args | grep elf32-core-headers` if [ -n "$found_elf_args" ] then - echo -n "Warning: elf32-core-headers overrides correct elf64 setting" - echo + dwarn "Warning: elf32-core-headers overrides correct elf64 setting" else kexec_args="$kexec_args --elf64-core-headers" fi @@ -697,7 +704,7 @@ prepare_kdump_bootinfo() done if ! [ -e "$KDUMP_KERNEL" ]; then - echo "Failed to detect kdump kernel location" + derror "Failed to detect kdump kernel location" return 1 fi diff --git a/kdump-logger.sh b/kdump-logger.sh new file mode 100755 index 0000000..c7afa34 --- /dev/null +++ b/kdump-logger.sh @@ -0,0 +1,334 @@ +#!/bin/bash +# +# This comes from the dracut-logger.sh +# +# The logger defined 6 logging levels: +# - dtrace() (6) +# The TRACE Level designates finer-grained informational events than the +# DEBUG. +# - ddebug (5) +# The DEBUG Level designates fine-grained informational events that are most +# useful to debug an application. +# - dinfo (4) +# The INFO level designates informational messages that highlight the +# progress of the application at coarse-grained level. +# - dwarn (3) +# The WARN level designates potentially harmful situations. +# - derror (2) +# The ERROR level designates error events that might still allow the +# application to continue running. +# - dfatal (1) +# The FATAL level designates very severe error events that will presumably +# lead the application to abort. +# +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl = 4 (info) +# - @var kdump_sysloglvl = 4 (info) +# - @var kdump_kmsgloglvl = 0 (no logging) +# +# First of all you have to start with dlog_init() function which initializes +# required variables. Don't call any other logging function before that one! +# +# @brief Initializes Logger. +# @retval 1 if something has gone wrong +# @retval 0 on success. +# +dlog_init() { + local ret=0; local errmsg + [ -z "$kdump_stdloglvl" ] && kdump_stdloglvl=4 + [ -z "$kdump_sysloglvl" ] && kdump_sysloglvl=4 + [ -z "$kdump_kmsgloglvl" ] && kdump_kmsgloglvl=0 + # Skip initialization if it's already done. + [ -n "$kdump_maxloglvl" ] && return 0 + + if [[ $UID -ne 0 ]]; then + kdump_kmsgloglvl=0 + kdump_sysloglvl=0 + fi + + if [[ $kdump_sysloglvl -gt 0 ]]; then + if [[ -d /run/systemd/journal ]] \ + && type -P systemd-cat &>/dev/null \ + && systemctl --quiet is-active systemd-journald.socket &>/dev/null; then + readonly _systemdcatfile="/var/tmp/systemd-cat" + mkfifo "$_systemdcatfile" &>/dev/null + readonly _dlogfd=15 + systemd-cat -t 'kdump' --level-prefix=true <"$_systemdcatfile" & + exec 15>"$_systemdcatfile" + elif ! [ -S /dev/log -a -w /dev/log ] || ! command -v logger >/dev/null; then + # We cannot log to syslog, so turn this facility off. + kdump_kmsgloglvl=$kdump_sysloglvl + kdump_sysloglvl=0 + ret=1 + errmsg="No '/dev/log' or 'logger' included for syslog logging" + fi + fi + + local lvl; local maxloglvl_l=0 + for lvl in $kdump_stdloglvl $kdump_sysloglvl $kdump_kmsgloglvl; do + [[ $lvl -gt $maxloglvl_l ]] && maxloglvl_l=$lvl + done + readonly kdump_maxloglvl=$maxloglvl_l + export kdump_maxloglvl + + + if [[ $kdump_stdloglvl -lt 6 ]] && [[ $kdump_kmsgloglvl -lt 6 ]] && [[ $kdump_sysloglvl -lt 6 ]]; then + unset dtrace + dtrace() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 5 ]] && [[ $kdump_kmsgloglvl -lt 5 ]] && [[ $kdump_sysloglvl -lt 5 ]]; then + unset ddebug + ddebug() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 4 ]] && [[ $kdump_kmsgloglvl -lt 4 ]] && [[ $kdump_sysloglvl -lt 4 ]]; then + unset dinfo + dinfo() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 3 ]] && [[ $kdump_kmsgloglvl -lt 3 ]] && [[ $kdump_sysloglvl -lt 3 ]]; then + unset dwarn + dwarn() { :; }; + unset dwarning + dwarning() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 2 ]] && [[ $kdump_kmsgloglvl -lt 2 ]] && [[ $kdump_sysloglvl -lt 2 ]]; then + unset derror + derror() { :; }; + fi + + if [[ $kdump_stdloglvl -lt 1 ]] && [[ $kdump_kmsgloglvl -lt 1 ]] && [[ $kdump_sysloglvl -lt 1 ]]; then + unset dfatal + dfatal() { :; }; + fi + + [ -n "$errmsg" ] && derror "$errmsg" + + return $ret +} + +## @brief Converts numeric logging level to the first letter of level name. +# +# @param lvl Numeric logging level in range from 1 to 6. +# @retval 1 if @a lvl is out of range. +# @retval 0 if @a lvl is correct. +# @result Echoes first letter of level name. +_lvl2char() { + case "$1" in + 1) echo F;; + 2) echo E;; + 3) echo W;; + 4) echo I;; + 5) echo D;; + 6) echo T;; + *) return 1;; + esac +} + +## @brief Converts numeric level to logger priority defined by POSIX.2. +# +# @param lvl Numeric logging level in range from 1 to 6. +# @retval 1 if @a lvl is out of range. +# @retval 0 if @a lvl is correct. +# @result Echoes logger priority. +_lvl2syspri() { + case "$1" in + 1) echo crit;; + 2) echo error;; + 3) echo warning;; + 4) echo info;; + 5) echo debug;; + 6) echo debug;; + *) return 1;; + esac +} + +## @brief Converts logger numeric level to syslog log level +# +# @param lvl Numeric logging level in range from 1 to 6. +# @retval 1 if @a lvl is out of range. +# @retval 0 if @a lvl is correct. +# @result Echoes kernel console numeric log level +# +# Conversion is done as follows: +# +# +# none -> LOG_EMERG (0) +# none -> LOG_ALERT (1) +# FATAL(1) -> LOG_CRIT (2) +# ERROR(2) -> LOG_ERR (3) +# WARN(3) -> LOG_WARNING (4) +# none -> LOG_NOTICE (5) +# INFO(4) -> LOG_INFO (6) +# DEBUG(5) -> LOG_DEBUG (7) +# TRACE(6) / +# +# +# @see /usr/include/sys/syslog.h +_dlvl2syslvl() { + local lvl + + case "$1" in + 1) lvl=2;; + 2) lvl=3;; + 3) lvl=4;; + 4) lvl=6;; + 5) lvl=7;; + 6) lvl=7;; + *) return 1;; + esac + + [ -s /proc/vmcore ] && echo $((24+$lvl)) || echo $((8+$lvl)) +} + +## @brief Prints to stderr and/or writes to file, to syslog and/or /dev/kmsg +# given message with given level (priority). +# +# @param lvl Numeric logging level. +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +# +# @note This function is not supposed to be called manually. Please use +# dtrace(), ddebug(), or others instead which wrap this one. +# +# This is core logging function which logs given message to standard error, file +# and/or syslog (with POSIX shell command logger) and/or to /dev/kmsg. +# The format is following: +# +# X: some message +# +# where @c X is the first letter of logging level. See module description for +# details on that. +# +# Message to syslog is sent with tag @c kdump. Priorities are mapped as +# following: +# - @c FATAL to @c crit +# - @c ERROR to @c error +# - @c WARN to @c warning +# - @c INFO to @c info +# - @c DEBUG and @c TRACE both to @c debug +_do_dlog() { + local lvl="$1"; shift + local lvlc=$(_lvl2char "$lvl") || return 0 + local msg="$*" + local lmsg="$lvlc: $*" + + [[ $lvl -le $kdump_stdloglvl ]] && printf -- 'kdump: %s\n' "$msg" >&2 + + if [[ $lvl -le $kdump_sysloglvl ]]; then + if [[ "$_dlogfd" ]]; then + printf -- "<%s>%s\n" "$(($(_dlvl2syslvl $lvl) & 7))" "$msg" >&$_dlogfd + else + logger -t "kdump[$$]" -p $(_lvl2syspri $lvl) -- "$msg" + fi + fi + + [[ $lvl -le $kdump_kmsgloglvl ]] && \ + echo "<$(_dlvl2syslvl $lvl)>kdump[$$] $msg" >/dev/kmsg +} + +## @brief Internal helper function for _do_dlog() +# +# @param lvl Numeric logging level. +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +# +# @note This function is not supposed to be called manually. Please use +# dtrace(), ddebug(), or others instead which wrap this one. +# +# This function calls _do_dlog() either with parameter msg, or if +# none is given, it will read standard input and will use every line as +# a message. +# +# This enables: +# dwarn "This is a warning" +# echo "This is a warning" | dwarn +dlog() { + [ -z "$kdump_maxloglvl" ] && return 0 + [[ $1 -le $kdump_maxloglvl ]] || return 0 + + if [[ $# -gt 1 ]]; then + _do_dlog "$@" + else + while read line || [ -n "$line" ]; do + _do_dlog "$1" "$line" + done + fi +} + +## @brief Logs message at TRACE level (6) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dtrace() { + set +x + dlog 6 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at DEBUG level (5) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +ddebug() { + set +x + dlog 5 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at INFO level (4) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dinfo() { + set +x + dlog 4 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at WARN level (3) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dwarn() { + set +x + dlog 3 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief It's an alias to dwarn() function. +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dwarning() { + set +x + dwarn "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at ERROR level (2) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +derror() { + set +x + dlog 2 "$@" + [ -n "$debug" ] && set -x || : +} + +## @brief Logs message at FATAL level (1) +# +# @param msg Message. +# @retval 0 It's always returned, even if logging failed. +dfatal() { + set +x + dlog 1 "$@" + [ -n "$debug" ] && set -x || : +} diff --git a/kdump.conf b/kdump.conf index 9fc0a4f..e4db52e 100644 --- a/kdump.conf +++ b/kdump.conf @@ -62,9 +62,9 @@ # as the initrd will automatically be populated with a # config file appropriate for the running kernel. # The default core_collector for raw/ssh dump is: -# "makedumpfile -F -l --message-level 1 -d 31". +# "makedumpfile -F -l --message-level 7 -d 31". # The default core_collector for other targets is: -# "makedumpfile -l --message-level 1 -d 31". +# "makedumpfile -l --message-level 7 -d 31". # # "makedumpfile -F" will create a flattened vmcore. # You need to use "makedumpfile -R" to rearrange the dump data to @@ -169,7 +169,7 @@ #ssh user@my.server.com #sshkey /root/.ssh/kdump_id_rsa path /var/crash -core_collector makedumpfile -l --message-level 1 -d 31 +core_collector makedumpfile -l --message-level 7 -d 31 #core_collector scp #kdump_post /var/crash/scripts/kdump-post.sh #kdump_pre /var/crash/scripts/kdump-pre.sh diff --git a/kdump.conf.5 b/kdump.conf.5 index 34c996c..ae07466 100644 --- a/kdump.conf.5 +++ b/kdump.conf.5 @@ -41,7 +41,7 @@ mount point. .B ssh .RS -Will scp /proc/vmcore to :/%HOST-%DATE/, +Will save /proc/vmcore through ssh pipe to :/%HOST-%DATE/, supports DNS. NOTE: make sure user has necessary write permissions on server and that a fqdn is used as the server name. .RE @@ -85,9 +85,9 @@ for the running kernel. .PP Note 1: About default core collector: The default core_collector for raw/ssh dump is: -"makedumpfile -F -l --message-level 1 -d 31". +"makedumpfile -F -l --message-level 7 -d 31". The default core_collector for other targets is: -"makedumpfile -l --message-level 1 -d 31". +"makedumpfile -l --message-level 7 -d 31". Even if core_collector option is commented out in kdump.conf, makedumpfile is the default core collector and kdump uses it internally. If one does not want makedumpfile as default core_collector, then they @@ -307,11 +307,11 @@ Above will effectively be translated to: cp --sparse=always /proc/vmcore /vmcore .TP ex2. -core_collector "makedumpfile -l --message-level 1 -d 31" +core_collector "makedumpfile -l --message-level 7 -d 31" Above will effectively be translated to: -makedumpfile -l --message-level 1 -d 31 /proc/vmcore /vmcore +makedumpfile -l --message-level 7 -d 31 /proc/vmcore /vmcore .PP For dump targets like raw and ssh, in general, core collector should expect one argument (source file) and should output the processed core on standard @@ -328,11 +328,11 @@ Above will effectively be translated to. cat /proc/vmcore | dd of= .TP ex4. -core_collector "makedumpfile -F -l --message-level 1 -d 31" +core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. -makedumpfile -F -l --message-level 1 -d 31 | dd of= +makedumpfile -F -l --message-level 7 -d 31 | dd of= .PP ssh dumps examples .TP @@ -344,11 +344,11 @@ Above will effectively be translated to. cat /proc/vmcore | ssh "dd of=path/vmcore" .TP ex6. -core_collector "makedumpfile -F -l --message-level 1 -d 31" +core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. -makedumpfile -F -l --message-level 1 -d 31 | ssh "dd of=path/vmcore" +makedumpfile -F -l --message-level 7 -d 31 | ssh "dd of=path/vmcore" There is one exception to standard output rule for ssh dumps. And that is scp. As scp can handle ssh destinations for file transfers, one can diff --git a/kdump.service b/kdump.service index f888dd6..99feed8 100644 --- a/kdump.service +++ b/kdump.service @@ -2,6 +2,7 @@ Description=Crash recovery kernel arming After=network.target network-online.target remote-fs.target basic.target DefaultDependencies=no +ConditionKernelCommandLine=crashkernel [Service] Type=oneshot diff --git a/kdump.sysconfig b/kdump.sysconfig index ffc387a..52a0b92 100644 --- a/kdump.sysconfig +++ b/kdump.sysconfig @@ -35,3 +35,19 @@ KEXEC_ARGS="" #What is the image type used for kdump KDUMP_IMG="vmlinuz" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.aarch64 b/kdump.sysconfig.aarch64 index 6f55c03..1d75979 100644 --- a/kdump.sysconfig.aarch64 +++ b/kdump.sysconfig.aarch64 @@ -35,3 +35,19 @@ KEXEC_ARGS="" #What is the image type used for kdump KDUMP_IMG="vmlinuz" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.i386 b/kdump.sysconfig.i386 index 45f6d50..2eb9705 100644 --- a/kdump.sysconfig.i386 +++ b/kdump.sysconfig.i386 @@ -38,3 +38,19 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.ppc64 b/kdump.sysconfig.ppc64 index 8dcbe60..24279ff 100644 --- a/kdump.sysconfig.ppc64 +++ b/kdump.sysconfig.ppc64 @@ -40,3 +40,19 @@ KDUMP_IMG="vmlinuz" KDUMP_IMG_EXT="" #Specify the action after failure + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.ppc64le b/kdump.sysconfig.ppc64le index 8dcbe60..24279ff 100644 --- a/kdump.sysconfig.ppc64le +++ b/kdump.sysconfig.ppc64le @@ -40,3 +40,19 @@ KDUMP_IMG="vmlinuz" KDUMP_IMG_EXT="" #Specify the action after failure + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.s390x b/kdump.sysconfig.s390x index 2a24688..5e1cb5a 100644 --- a/kdump.sysconfig.s390x +++ b/kdump.sysconfig.s390x @@ -31,7 +31,7 @@ MKDUMPRD_ARGS="" # # Example: # KEXEC_ARGS="--elf32-core-headers" -KEXEC_ARGS="" +KEXEC_ARGS="-s" #Where to find the boot image #KDUMP_BOOTDIR="/boot" @@ -41,3 +41,19 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdump.sysconfig.x86_64 b/kdump.sysconfig.x86_64 index f67d999..eb1c538 100644 --- a/kdump.sysconfig.x86_64 +++ b/kdump.sysconfig.x86_64 @@ -38,3 +38,19 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" + +# Logging is controlled by following global variables: +# - @var kdump_stdloglvl - logging level to standard error (console output) +# - @var kdump_sysloglvl - logging level to syslog (by logger command) +# - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) +# If any of the variables is not set, this function set it to default: +# - @var kdump_stdloglvl=4 (info) +# - @var kdump_sysloglvl=4 (info) +# - @var kdump_kmsgloglvl=0 (no logging) +# +# Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) +# +# For example: the following configurations indicate that kdump messages will be printed +# to console and journald for debugging case. +# kdump_sysloglvl=5 +# kdump_stdloglvl=5 diff --git a/kdumpctl b/kdumpctl index 2248da4..e816719 100755 --- a/kdumpctl +++ b/kdumpctl @@ -6,6 +6,7 @@ KDUMP_KERNEL="" KDUMP_COMMANDLINE="" KEXEC_ARGS="" KDUMP_CONFIG_FILE="/etc/kdump.conf" +KDUMP_LOG_PATH="/var/log" MKDUMPRD="/sbin/mkdumprd -f" DRACUT_MODULES_FILE="/usr/lib/dracut/modules.txt" SAVE_PATH=/var/crash @@ -21,11 +22,7 @@ FADUMP_REGISTER_SYS_NODE="/sys/kernel/fadump_registered" DEFAULT_DUMP_MODE="kdump" image_time=0 -[[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut -. $dracutbasedir/dracut-functions.sh -. /lib/kdump/kdump-lib.sh - -standard_kexec_args="-p" +standard_kexec_args="-d -p" # Some default values in case /etc/sysconfig/kdump doesn't include KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug" @@ -34,13 +31,24 @@ if [ -f /etc/sysconfig/kdump ]; then . /etc/sysconfig/kdump fi +[[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut +. $dracutbasedir/dracut-functions.sh +. /lib/kdump/kdump-lib.sh + +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + single_instance_lock() { local rc timeout=5 exec 9>/var/lock/kdump if [ $? -ne 0 ]; then - echo "Create file lock failed" + derror "Create file lock failed" exit 1 fi @@ -48,7 +56,7 @@ single_instance_lock() rc=$? while [ $rc -ne 0 ]; do - echo "Another app is currently holding the kdump lock; waiting for it to exit..." + dinfo "Another app is currently holding the kdump lock; waiting for it to exit..." flock -w $timeout 9 rc=$? done @@ -59,9 +67,10 @@ determine_dump_mode() # Check if firmware-assisted dump is enabled # if yes, set the dump mode as fadump if is_fadump_capable; then - echo "Dump mode is fadump" + dinfo "Dump mode is fadump" DEFAULT_DUMP_MODE="fadump" fi + ddebug "DEFAULT_DUMP_MODE=$DEFAULT_DUMP_MODE" } save_core() @@ -69,22 +78,25 @@ save_core() coredir="/var/crash/`date +"%Y-%m-%d-%H:%M"`" mkdir -p $coredir + ddebug "cp --sparse=always /proc/vmcore $coredir/vmcore-incomplete" cp --sparse=always /proc/vmcore $coredir/vmcore-incomplete if [ $? == 0 ]; then mv $coredir/vmcore-incomplete $coredir/vmcore - echo "saved a vmcore to $coredir" + dinfo "saved a vmcore to $coredir" else - echo "failed to save a vmcore to $coredir" >&2 + derror "failed to save a vmcore to $coredir" fi # pass the dmesg to Abrt tool if exists, in order # to collect the kernel oops message. # https://fedorahosted.org/abrt/ if [ -x /usr/bin/dumpoops ]; then + ddebug "makedumpfile --dump-dmesg $coredir/vmcore $coredir/dmesg" makedumpfile --dump-dmesg $coredir/vmcore $coredir/dmesg >/dev/null 2>&1 + ddebug "dumpoops -d $coredir/dmesg" dumpoops -d $coredir/dmesg >/dev/null 2>&1 if [ $? == 0 ]; then - echo "kernel oops has been collected by abrt tool" + dinfo "kernel oops has been collected by abrt tool" fi fi } @@ -96,16 +108,18 @@ rebuild_fadump_initrd() # this file tells the initrd is fadump enabled touch /tmp/fadump.initramfs target_initrd_tmp="$TARGET_INITRD.tmp" + ddebug "rebuild fadump initrd: $target_initrd_tmp $DEFAULT_INITRD_BAK $KDUMP_KERNELVER" $MKDUMPRD $target_initrd_tmp --rebuild $DEFAULT_INITRD_BAK --kver $KDUMP_KERNELVER \ -i /tmp/fadump.initramfs /etc/fadump.initramfs if [ $? != 0 ]; then - echo "mkdumprd: failed to rebuild initrd with fadump support" >&2 + derror "mkdumprd: failed to rebuild initrd with fadump support" rm -f /tmp/fadump.initramfs return 1 fi rm -f /tmp/fadump.initramfs # updating fadump initrd + ddebug "updating fadump initrd: $target_initrd_tmp $TARGET_INITRD" mv $target_initrd_tmp $TARGET_INITRD sync @@ -120,14 +134,15 @@ check_earlykdump_is_enabled() rebuild_kdump_initrd() { + ddebug "rebuild kdump initrd: $MKDUMPRD $TARGET_INITRD $KDUMP_KERNELVER" $MKDUMPRD $TARGET_INITRD $KDUMP_KERNELVER if [ $? != 0 ]; then - echo "mkdumprd: failed to make kdump initrd" >&2 + derror "mkdumprd: failed to make kdump initrd" return 1 fi if check_earlykdump_is_enabled; then - echo "Tips: If early kdump is enabled, also require rebuilding the system initramfs to make the changes take effect for early kdump." + dwarn "Tips: If early kdump is enabled, also require rebuilding the system initramfs to make the changes take effect for early kdump." fi return 0 @@ -136,7 +151,7 @@ rebuild_kdump_initrd() rebuild_initrd() { if [[ ! -w "$KDUMP_BOOTDIR" ]];then - echo "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD" + derror "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD" return 1 fi @@ -154,7 +169,7 @@ check_exist() { for file in $1; do if [ ! -e "$file" ]; then - echo -n "Error: $file not found."; echo + derror "Error: $file not found." return 1 fi done @@ -165,7 +180,7 @@ check_executable() { for file in $1; do if [ ! -x "$file" ]; then - echo -n "Error: $file is not executable."; echo + derror "Error: $file is not executable." return 1 fi done @@ -173,17 +188,19 @@ check_executable() backup_default_initrd() { + ddebug "backup default initrd: $DEFAULT_INITRD" + if [ ! -f "$DEFAULT_INITRD" ]; then return fi if [ ! -e $DEFAULT_INITRD_BAK ]; then - echo "Backing up $DEFAULT_INITRD before rebuild." + dinfo "Backing up $DEFAULT_INITRD before rebuild." # save checksum to verify before restoring sha1sum $DEFAULT_INITRD > $INITRD_CHECKSUM_LOCATION cp $DEFAULT_INITRD $DEFAULT_INITRD_BAK if [ $? -ne 0 ]; then - echo "WARNING: failed to backup $DEFAULT_INITRD." + dwarn "WARNING: failed to backup $DEFAULT_INITRD." rm -f $DEFAULT_INITRD_BAK fi fi @@ -191,6 +208,8 @@ backup_default_initrd() restore_default_initrd() { + ddebug "restore default initrd: $DEFAULT_INITRD" + if [ ! -f "$DEFAULT_INITRD" ]; then return fi @@ -202,13 +221,12 @@ restore_default_initrd() backup_checksum=`sha1sum $DEFAULT_INITRD_BAK | awk '{ print $1 }'` default_checksum=`cat $INITRD_CHECKSUM_LOCATION | awk '{ print $1 }'` if [ "$default_checksum" != "$backup_checksum" ]; then - echo "WARNING: checksum mismatch! Can't restore original initrd.." + dwarn "WARNING: checksum mismatch! Can't restore original initrd.." else rm -f $INITRD_CHECKSUM_LOCATION mv $DEFAULT_INITRD_BAK $DEFAULT_INITRD if [[ $? -eq 0 ]]; then - echo -n "Restoring original initrd as fadump mode " - echo "is disabled." + derror "Restoring original initrd as fadump mode is disabled." sync fi fi @@ -217,62 +235,57 @@ restore_default_initrd() check_config() { - local nr - - nr=$(awk 'BEGIN{cnt=0} /^raw|^ssh[[:blank:]]|^nfs|^ext[234]|^xfs|^btrfs|^minix|^dracut_args .*\-\-mount/{cnt++} END{print cnt}' $KDUMP_CONFIG_FILE) - [ $nr -gt 1 ] && { - echo "More than one dump targets specified." - return 1 - } - - # Check if path option is set more than once. - nr=$(awk 'BEGIN{cnt=0} /^path /{cnt++} END{print cnt}' $KDUMP_CONFIG_FILE) - [ $nr -gt 1 ] && { - echo "Mutiple paths specifed in $KDUMP_CONFIG_FILE" - return 1 - } - - nr=$(grep "^dracut_args .*\-\-mount" $KDUMP_CONFIG_FILE | grep -o "\-\-mount" | wc -l) - [ $nr -gt 1 ] && { - echo "Multiple mount targets specified in one \"dracut_args\"." - return 1 - } - - # Check if we have any leading spaces (or tabs) before the - # variable name in the kdump conf file - if grep -E -q '^[[:blank:]]+[a-z]' $KDUMP_CONFIG_FILE; then - echo "No whitespaces are allowed before a kdump option name in $KDUMP_CONFIG_FILE" - return 1 - fi - + local -A _opt_rec while read config_opt config_val; do + if [ -z "$config_val" ]; then + derror "Invalid kdump config value for option $config_opt" + return 1 + fi + case "$config_opt" in - \#* | "") + dracut_args) + if [[ $config_val == *--mount* ]]; then + if [ $(echo $config_val | grep -o "\-\-mount" | wc -l) -ne 1 ]; then + derror "Multiple mount targets specified in one \"dracut_args\"." + return 1 + fi + config_opt=_target + fi ;; - raw|ext2|ext3|ext4|minix|btrfs|xfs|nfs|ssh|sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|failure_action|default|final_action|force_rebuild|force_no_rebuild|dracut_args|fence_kdump_args|fence_kdump_nodes) - # remove inline comments after the end of a directive. - [ -z "$config_val" ] && { - echo "Invalid kdump config value for option $config_opt." - return 1; - } - if [ -d "/proc/device-tree/ibm,opal/dump" ] && [ "$config_opt" == "raw" ]; then - echo "WARNING: Won't capture opalcore when 'raw' dump target is used." + raw) + if [ -d "/proc/device-tree/ibm,opal/dump" ]; then + dwarn "WARNING: Won't capture opalcore when 'raw' dump target is used." fi + config_opt=_target + ;; + ext[234]|minix|btrfs|xfs|nfs|ssh) + config_opt=_target + ;; + sshkey|path|core_collector|kdump_post|kdump_pre|extra_bins|extra_modules|failure_action|default|final_action|force_rebuild|force_no_rebuild|fence_kdump_args|fence_kdump_nodes) ;; net|options|link_delay|disk_timeout|debug_mem_level|blacklist) - echo "Deprecated kdump config option: $config_opt. Refer to kdump.conf manpage for alternatives." + derror "Deprecated kdump config option: $config_opt. Refer to kdump.conf manpage for alternatives." return 1 ;; *) - echo "Invalid kdump config option $config_opt" - return 1; + derror "Invalid kdump config option $config_opt" + return 1 ;; esac + + if [ -n "${_opt_rec[$config_opt]}" ]; then + if [ $config_opt == _target ]; then + derror "More than one dump targets specified" + else + derror "Duplicated kdump config value of option $config_opt" + fi + return 1 + fi + _opt_rec[$config_opt]="$config_val" done <<< "$(read_strip_comments $KDUMP_CONFIG_FILE)" check_failure_action_config || return 1 check_final_action_config || return 1 - check_fence_kdump_config || return 1 return 0 @@ -309,6 +322,7 @@ setup_initrd() { prepare_kdump_bootinfo if [ $? -ne 0 ]; then + derror "failed to prepare for kdump bootinfo." return 1 fi @@ -379,7 +393,7 @@ check_files_modified() else # If it's not a module nor builtin, give an error if ! ( modprobe --set-version "$KDUMP_KERNELVER" --dry-run "$_module" &>/dev/null ); then - echo "Module $_module not found" + dwarn "Module $_module not found" fi fi done @@ -404,13 +418,12 @@ check_files_modified() fi fi else - echo "$file doesn't exist" + dwarn "$file doesn't exist" fi done if [ -n "$modified_files" ]; then - echo "Detected change(s) in the following file(s):" - echo -n " "; echo "$modified_files" | sed 's/\s/\n /g' + dinfo "Detected change(s) in the following file(s): $modified_files" return 1 fi @@ -448,13 +461,16 @@ check_dump_fs_modified() _target=$(to_dev_name $_target) _new_fstype=$(get_fs_type_from_target $_target) if [[ -z "$_target" || -z "$_new_fstype" ]];then - echo "Dump path $_path does not exist" + derror "Dump path $_path does not exist" return 2 fi fi + ddebug "_target=$_target _path=$_path _new_fstype=$_new_fstype" + _record_block_drivers() { local _drivers + if [[ -b /dev/block/$1 ]]; then _drivers=$(udevadm info -a "/dev/block/$1" | sed -n 's/\s*DRIVERS=="\(\S\+\)"/\1/p') fi @@ -466,6 +482,7 @@ check_dump_fs_modified() _target_drivers="$_target_drivers $_driver" fi done + ddebug "MAJ:MIN=$1 _drivers=$_drivers _target_drivers=$_targer_drivers" return 1 } @@ -478,7 +495,7 @@ check_dump_fs_modified() continue fi if ! [[ " $_old_drivers " == *" $_module_name "* ]]; then - echo "Detected change in block device driver, new loaded module: $_module_name" + dinfo "Detected change in block device driver, new loaded module: $_module_name" return 1 fi done @@ -488,7 +505,7 @@ check_dump_fs_modified() else _new_dev=$(kdump_get_persistent_dev $_target) if [ -z "$_new_dev" ]; then - echo "Get persistent device name failed" + derror "Get persistent device name failed" return 2 fi fi @@ -496,7 +513,7 @@ check_dump_fs_modified() _new_mntpoint="$(get_kdump_mntpoint_from_target $_target)" _dracut_args=$(lsinitrd $TARGET_INITRD -f usr/lib/dracut/build-parameter.txt) if [[ -z "$_dracut_args" ]];then - echo "Warning: No dracut arguments found in initrd" + dwarn "Warning: No dracut arguments found in initrd" return 0 fi @@ -514,7 +531,7 @@ check_dump_fs_modified() [[ "$_target" = "$(get_root_fs_device)" ]] && return 0 fi - echo "Detected change in File System" + dinfo "Detected change in File System" return 1 } @@ -597,7 +614,7 @@ check_system_modified() check_wdt_modified if [ $? -ne 0 ]; then - echo "Detected change in watchdog state" + dinfo "Detected change in watchdog state" return 1 fi @@ -621,7 +638,7 @@ check_rebuild() if [ $? -eq 0 ]; then force_no_rebuild=`echo $_force_no_rebuild | cut -d' ' -f2` if [ "$force_no_rebuild" != "0" ] && [ "$force_no_rebuild" != "1" ];then - echo "Error: force_no_rebuild value is invalid" + derror "Error: force_no_rebuild value is invalid" return 1 fi fi @@ -630,13 +647,13 @@ check_rebuild() if [ $? -eq 0 ]; then force_rebuild=`echo $_force_rebuild | cut -d' ' -f2` if [ "$force_rebuild" != "0" ] && [ "$force_rebuild" != "1" ];then - echo "Error: force_rebuild value is invalid" + derror "Error: force_rebuild value is invalid" return 1 fi fi if [[ "$force_no_rebuild" == "1" && "$force_rebuild" == "1" ]]; then - echo "Error: force_rebuild and force_no_rebuild are enabled simultaneously in kdump.conf" + derror "Error: force_rebuild and force_no_rebuild are enabled simultaneously in kdump.conf" return 1 fi @@ -666,18 +683,18 @@ check_rebuild() fi if [ $image_time -eq 0 ]; then - echo -n "No kdump initial ramdisk found."; echo + dinfo "No kdump initial ramdisk found." elif [ "$capture_capable_initrd" == "0" ]; then - echo -n "Rebuild $TARGET_INITRD with dump capture support"; echo + dinfo "Rebuild $TARGET_INITRD with dump capture support" elif [ "$force_rebuild" != "0" ]; then - echo -n "Force rebuild $TARGET_INITRD"; echo + dinfo "Force rebuild $TARGET_INITRD" elif [ "$system_modified" != "0" ]; then : else return 0 fi - echo "Rebuilding $TARGET_INITRD" + dinfo "Rebuilding $TARGET_INITRD" rebuild_initrd return $? } @@ -687,6 +704,8 @@ check_rebuild() # as the currently running kernel. load_kdump() { + local ret + KEXEC_ARGS=$(prepare_kexec_args "${KEXEC_ARGS}") KDUMP_COMMANDLINE=$(prepare_cmdline "${KDUMP_COMMANDLINE}" "${KDUMP_COMMANDLINE_REMOVE}" "${KDUMP_COMMANDLINE_APPEND}") @@ -694,18 +713,30 @@ load_kdump() # Old syscall will always fail as it does not have capability to # to kernel signature verification. if is_secure_boot_enforced; then - echo "Secure Boot is enabled. Using kexec file based syscall." + dinfo "Secure Boot is enabled. Using kexec file based syscall." KEXEC_ARGS="$KEXEC_ARGS -s" fi + ddebug "$KEXEC $KEXEC_ARGS $standard_kexec_args --command-line=$KDUMP_COMMANDLINE --initrd=$TARGET_INITRD $KDUMP_KERNEL" + + exec 12>&2 + exec 2>> $KDUMP_LOG_PATH/kdump.log + PS4='+ $(date "+%Y-%m-%d %H:%M:%S") ${BASH_SOURCE}@${LINENO}: ' + set -x + $KEXEC $KEXEC_ARGS $standard_kexec_args \ --command-line="$KDUMP_COMMANDLINE" \ --initrd=$TARGET_INITRD $KDUMP_KERNEL - if [ $? == 0 ]; then - echo "kexec: loaded kdump kernel" + + ret=$? + set +x + exec 2>&12 12>&- + + if [ $ret == 0 ]; then + dinfo "kexec: loaded kdump kernel" return 0 else - echo "kexec: failed to load kdump kernel" >&2 + derror "kexec: failed to load kdump kernel" return 1 fi } @@ -720,7 +751,7 @@ check_ssh_config() # canonicalize the path SSH_KEY_LOCATION=$(/usr/bin/readlink -m $config_val) else - echo "WARNING: '$config_val' doesn't exist, using default value '$SSH_KEY_LOCATION'" + dwarn "WARNING: '$config_val' doesn't exist, using default value '$SSH_KEY_LOCATION'" fi ;; path) @@ -762,22 +793,22 @@ check_and_wait_network_ready() if [ $retval -eq 0 ]; then return 0 elif [ $retval -ne 255 ]; then - echo "Could not create $DUMP_TARGET:$SAVE_PATH, you should check the privilege on server side" >&2 + derror "Could not create $DUMP_TARGET:$SAVE_PATH, you should check the privilege on server side" return 1 fi # if server removes the authorized_keys or, no /root/.ssh/kdump_id_rsa - echo $errmsg | grep -q "Permission denied\|No such file or directory\|Host key verification failed" + ddebug "$errmsg" + echo $errmsg | grep -q "Permission denied\|No such file or directory\|Host key verification failed" &> /dev/null if [ $? -eq 0 ]; then - echo "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" >&2 + derror "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" return 1 fi if [ $warn_once -eq 1 ]; then - echo "Network dump target is not usable, waiting for it to be ready" + dwarn "Network dump target is not usable, waiting for it to be ready..." warn_once=0 fi - echo -n . cur=$(date +%s) let "diff = $cur - $start_time" @@ -788,7 +819,7 @@ check_and_wait_network_ready() sleep 1 done - echo "Could not create $DUMP_TARGET:$SAVE_PATH, ipaddr is not ready yet. You should check network connection" >&2 + dinfo "Could not create $DUMP_TARGET:$SAVE_PATH, ipaddr is not ready yet. You should check network connection" return 1 } @@ -805,7 +836,7 @@ propagate_ssh_key() { check_ssh_config if [ $? -ne 0 ]; then - echo "No ssh config specified in $KDUMP_CONFIG_FILE. Can't propagate" >&2 + derror "No ssh config specified in $KDUMP_CONFIG_FILE. Can't propagate" exit 1 fi @@ -814,11 +845,11 @@ propagate_ssh_key() #Check to see if we already created key, if not, create it. if [ -f $KEYFILE ]; then - echo "Using existing keys..." + dinfo "Using existing keys..." else - echo -n "Generating new ssh keys... " + dinfo "Generating new ssh keys... " /usr/bin/ssh-keygen -t rsa -f $KEYFILE -N "" 2>&1 > /dev/null - echo "done." + dinfo "done." fi #now find the target ssh user and server to contact. @@ -829,10 +860,10 @@ propagate_ssh_key() ssh-copy-id -i $KEYFILE $SSH_USER@$SSH_SERVER RET=$? if [ $RET == 0 ]; then - echo $KEYFILE has been added to ~$SSH_USER/.ssh/authorized_keys on $SSH_SERVER + dinfo "$KEYFILE has been added to ~$SSH_USER/.ssh/authorized_keys on $SSH_SERVER" return 0 else - echo $errmsg, $KEYFILE failed in transfer to $SSH_SERVER >&2 + derror "$errmsg, $KEYFILE failed in transfer to $SSH_SERVER" exit 1 fi } @@ -842,7 +873,7 @@ show_reserved_mem() local mem=$(cat /sys/kernel/kexec_crash_size) local mem_mb=$(expr $mem / 1024 / 1024) - echo "Reserved "$mem_mb"MB memory for crash kernel" + dinfo "Reserved "$mem_mb"MB memory for crash kernel" } check_current_fadump_status() @@ -872,12 +903,12 @@ save_raw() raw_target=$(awk '$1 ~ /^raw$/ { print $2; }' $KDUMP_CONFIG_FILE) [ -z "$raw_target" ] && return 0 [ -b "$raw_target" ] || { - echo "raw partition $raw_target not found" + derror "raw partition $raw_target not found" return 1 } check_fs=$(lsblk --nodeps -npo FSTYPE $raw_target) if [[ $(echo $check_fs | wc -w) -ne 0 ]]; then - echo "Warning: Detected '$check_fs' signature on $raw_target, data loss is expected." + dwarn "Warning: Detected '$check_fs' signature on $raw_target, data loss is expected." return 0 fi kdump_dir=`grep ^path $KDUMP_CONFIG_FILE | cut -d' ' -f2-` @@ -889,12 +920,12 @@ save_raw() mkdir -p "$coredir" [ -d "$coredir" ] || { - echo "failed to create $coredir" + derror "failed to create $coredir" return 1 } if makedumpfile -R $coredir/vmcore <$raw_target >/dev/null 2>&1; then # dump found - echo "Dump saved to $coredir/vmcore" + dinfo "Dump saved to $coredir/vmcore" # wipe makedumpfile header dd if=/dev/zero of=$raw_target bs=1b count=1 2>/dev/null else @@ -967,13 +998,13 @@ check_fence_kdump_config() for node in $nodes; do if [ "$node" = "$hostname" ]; then - echo "Option fence_kdump_nodes cannot contain $hostname" + derror "Option fence_kdump_nodes cannot contain $hostname" return 1 fi # node can be ipaddr echo $ipaddrs | grep $node > /dev/null if [ $? -eq 0 ]; then - echo "Option fence_kdump_nodes cannot contain $node" + derror "Option fence_kdump_nodes cannot contain $node" return 1 fi done @@ -995,11 +1026,11 @@ start_fadump() { echo 1 > $FADUMP_REGISTER_SYS_NODE if ! check_current_fadump_status; then - echo "fadump: failed to register" + derror "fadump: failed to register" return 1 fi - echo "fadump: registered successfully" + dinfo "fadump: registered successfully" return 0 } @@ -1026,7 +1057,7 @@ check_failure_action_config() if [ -z "$failure_action" -a -z "$default_option" ]; then return 0 elif [ -n "$failure_action" -a -n "$default_option" ]; then - echo "Cannot specify 'failure_action' and 'default' option together" + derror "Cannot specify 'failure_action' and 'default' option together" return 1 fi @@ -1040,7 +1071,7 @@ check_failure_action_config() return 0 ;; *) - echo $"Usage kdump.conf: $option {reboot|halt|poweroff|shell|dump_to_rootfs}" + dinfo $"Usage kdump.conf: $option {reboot|halt|poweroff|shell|dump_to_rootfs}" return 1 esac } @@ -1058,7 +1089,7 @@ check_final_action_config() return 0 ;; *) - echo $"Usage kdump.conf: final_action {reboot|halt|poweroff}" + dinfo $"Usage kdump.conf: final_action {reboot|halt|poweroff}" return 1 esac fi @@ -1068,13 +1099,13 @@ start() { check_dump_feasibility if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi check_config if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi @@ -1084,43 +1115,43 @@ start() save_raw if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi check_current_status if [ $? == 0 ]; then - echo "Kdump already running: [WARNING]" + dwarn "Kdump already running: [WARNING]" return 0 fi if check_ssh_config; then if ! check_ssh_target; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi fi check_rebuild if [ $? != 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi start_dump if [ $? != 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi - echo "Starting kdump: [OK]" + dinfo "Starting kdump: [OK]" } reload() { check_current_status if [ $? -ne 0 ]; then - echo "Kdump was not running: [WARNING]" + dwarn "Kdump was not running: [WARNING]" fi if [ $DEFAULT_DUMP_MODE == "fadump" ]; then @@ -1131,36 +1162,36 @@ reload() fi if [ $? -ne 0 ]; then - echo "Stopping kdump: [FAILED]" + derror "Stopping kdump: [FAILED]" return 1 fi - echo "Stopping kdump: [OK]" + dinfo "Stopping kdump: [OK]" setup_initrd if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi start_dump if [ $? -ne 0 ]; then - echo "Starting kdump: [FAILED]" + derror "Starting kdump: [FAILED]" return 1 fi - echo "Starting kdump: [OK]" + dinfo "Starting kdump: [OK]" } stop_fadump() { echo 0 > $FADUMP_REGISTER_SYS_NODE if check_current_fadump_status; then - echo "fadump: failed to unregister" + derror "fadump: failed to unregister" return 1 fi - echo "fadump: unregistered successfully" + dinfo "fadump: unregistered successfully" return 0 } @@ -1173,11 +1204,11 @@ stop_kdump() fi if [ $? != 0 ]; then - echo "kexec: failed to unload kdump kernel" + derror "kexec: failed to unload kdump kernel" return 1 fi - echo "kexec: unloaded kdump kernel" + dinfo "kexec: unloaded kdump kernel" return 0 } @@ -1185,7 +1216,7 @@ reload_fadump() { echo 1 > $FADUMP_REGISTER_SYS_NODE if [ $? == 0 ]; then - echo "fadump: re-registered successfully" + dinfo "fadump: re-registered successfully" return 0 else # FADump could fail on older kernel where re-register @@ -1210,11 +1241,11 @@ stop() fi if [ $? != 0 ]; then - echo "Stopping kdump: [FAILED]" + derror "Stopping kdump: [FAILED]" return 1 fi - echo "Stopping kdump: [OK]" + dinfo "Stopping kdump: [OK]" return 0 } @@ -1235,13 +1266,13 @@ rebuild() { return 1 fi - echo "Rebuilding $TARGET_INITRD" + dinfo "Rebuilding $TARGET_INITRD" rebuild_initrd return $? } if [ ! -f "$KDUMP_CONFIG_FILE" ]; then - echo "Error: No kdump config file found!" >&2 + derror "Error: No kdump config file found!" exit 1 fi @@ -1267,11 +1298,11 @@ main () check_current_status case "$?" in 0) - echo "Kdump is operational" + dinfo "Kdump is operational" EXIT_CODE=0 ;; 1) - echo "Kdump is not operational" + dinfo "Kdump is not operational" EXIT_CODE=3 ;; esac @@ -1296,7 +1327,7 @@ main () show_reserved_mem ;; *) - echo $"Usage: $0 {start|stop|status|restart|reload|rebuild|propagate|showmem}" + dinfo $"Usage: $0 {start|stop|status|restart|reload|rebuild|propagate|showmem}" exit 1 esac } diff --git a/kexec-kdump-howto.txt b/kexec-kdump-howto.txt index 2209e5a..5f57a84 100644 --- a/kexec-kdump-howto.txt +++ b/kexec-kdump-howto.txt @@ -613,7 +613,7 @@ is a dump filtering and compression utility provided with kexec-tools. On some architectures, it can drastically reduce the size of your vmcore files, which becomes very useful on systems with large amounts of memory. -A typical setup is 'core_collector makedumpfile -F -l --message-level 1 -d 31', +A typical setup is 'core_collector makedumpfile -F -l --message-level 7 -d 31', but check the output of '/sbin/makedumpfile --help' for a list of all available options (-i and -g don't need to be specified, they're automatically taken care of). Note that use of makedumpfile requires that the kernel-debuginfo package @@ -633,11 +633,11 @@ First one is source file and second one is target file. For ex. - ex2. - core_collector "makedumpfile -l --message-level 1 -d 31" + core_collector "makedumpfile -l --message-level 7 -d 31" Above will effectively be translated to: - makedumpfile -l --message-level 1 -d 31 /proc/vmcore /vmcore + makedumpfile -l --message-level 7 -d 31 /proc/vmcore /vmcore For dump targets like raw and ssh, in general, core collector should expect one argument (source file) and should output the processed core on standard @@ -656,11 +656,11 @@ raw dumps core_collector examples: - ex4. - core_collector "makedumpfile -F -l --message-level 1 -d 31" + core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. - makedumpfile -F -l --message-level 1 -d 31 | dd of= + makedumpfile -F -l --message-level 7 -d 31 | dd of= ssh dumps core_collector examples: @@ -674,11 +674,11 @@ ssh dumps core_collector examples: - ex6. - core_collector "makedumpfile -F -l --message-level 1 -d 31" + core_collector "makedumpfile -F -l --message-level 7 -d 31" Above will effectively be translated to. - makedumpfile -F -l --message-level 1 -d 31 | ssh "dd of=path/vmcore" + makedumpfile -F -l --message-level 7 -d 31 | ssh "dd of=path/vmcore" There is one exception to standard output rule for ssh dumps. And that is scp. As scp can handle ssh destinations for file transfers, one can @@ -696,9 +696,9 @@ About default core collector ---------------------------- Default core_collector for ssh/raw dump is: -"makedumpfile -F -l --message-level 1 -d 31". +"makedumpfile -F -l --message-level 7 -d 31". Default core_collector for other targets is: -"makedumpfile -l --message-level 1 -d 31". +"makedumpfile -l --message-level 7 -d 31". Even if core_collector option is commented out in kdump.conf, makedumpfile is default core collector and kdump uses it internally. @@ -885,3 +885,38 @@ Debugging Tips minicom -C /tmp/console-logs Now minicom should be logging serial console in file console-logs. + +- Using the logger to output kdump log messages + + Currently, kdump messages are printed with the 'echo' command or redirect + to console, and which does not support to output kdump messages according + to the log level. + + That is not convenient to debug kdump issues, we usually need to capture + additional debugging information via the modification of the options or the + scripts like kdumpctl, mkdumprd, etc. Because there is no complete debugging + messages, which could waste valuable time. + + To cope with this challenging, we introduce the logger to output the kdump + messages according to the log level, and provide a chance to save logs to + the journald if the journald service is available, and then dump all logs + to a file, otherwise dump the logs with the dmesg to a file. + + Logging is controlled by following global variables: + - @var kdump_stdloglvl - logging level to standard error (console output) + - @var kdump_sysloglvl - logging level to syslog (by logger command) + - @var kdump_kmsgloglvl - logging level to /dev/kmsg (only for boot-time) + If any of the variables is not set, this function set it to default: + - @var kdump_stdloglvl=4 (info) + - @var kdump_sysloglvl=4 (info) + - @var kdump_kmsgloglvl=0 (no logging) + + Logging levels: fatal(1),error(2),warn(3),info(4),debug(5),trace(6) + + We can easily configure the above variables in the /etc/sysconfig/kdump. For + example: + kdump_sysloglvl=5 + kdump_stdloglvl=5 + + The above configurations indicate that kdump messages will be printed to the + console and journald if the journald service is enabled. diff --git a/kexec-tools.spec b/kexec-tools.spec index 669fd6c..623274c 100644 --- a/kexec-tools.spec +++ b/kexec-tools.spec @@ -4,7 +4,7 @@ Name: kexec-tools Version: 2.0.20 -Release: 18%{?dist} +Release: 19%{?dist} License: GPLv2 Summary: The kexec/kdump userspace component @@ -37,6 +37,7 @@ Source27: early-kdump-howto.txt Source28: kdump-udev-throttler Source29: kdump.sysconfig.aarch64 Source30: 60-kdump.install +Source31: kdump-logger.sh ####################################### # These are sources for mkdumpramfs @@ -190,6 +191,7 @@ install -m 644 %{SOURCE12} $RPM_BUILD_ROOT%{_mandir}/man8/mkdumprd.8 install -m 644 %{SOURCE25} $RPM_BUILD_ROOT%{_mandir}/man8/kdumpctl.8 install -m 755 %{SOURCE20} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib.sh install -m 755 %{SOURCE23} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-lib-initramfs.sh +install -m 755 %{SOURCE31} $RPM_BUILD_ROOT%{_prefix}/lib/kdump/kdump-logger.sh %ifnarch s390x install -m 755 %{SOURCE28} $RPM_BUILD_ROOT%{_udevrulesdir}/../kdump-udev-throttler %endif @@ -360,6 +362,28 @@ done %endif %changelog +- Fix comment about ssh dump target +- mkdumprd: Ensure kdumpbase is added +- kdump.service: use ConditionKernelCommandLine=crashkernel +- Revert "Revert "s390x: enable the kexec file load by def +- increase makdumpfile default message level to 7 +- Fix error when using raw target with opalcore +- module-setup.sh: Instead of drop journalctl log, just don't read kmsg +- Doc: add a documentation for the usage of logger +- Improve debugging in the kdump kernel +- kdumpctl: add the '-d' option to enable the kexec loading debugging messages +- kdump.sysconfig: add the kdump logger configurations +- enable the logger for kdump +- introduce the kdump logger from the dracut +- Rework check_config and warn on any duplicated option +- (watchdog) kdump-lib.sh: detect secure boot on s390 +- Don't drop journalctl content if failure action is "shell" +- dracut-module-install: Move systemd conf install code to a function +- selftest: Show the path of dumped vmcore on test end +- selftest: Add document for selftests +- selftest: Add basic test framework +- selftest: Add basic infrastructure to build test image + * Thu Aug 27 2020 Kairui Song - 2.0.20-18 - mkdumprd: Improve the warning message when using encrypted target - kdump-lib.sh: Remove is_atomic diff --git a/mkdumprd b/mkdumprd index 36f71b7..1c361d6 100644 --- a/mkdumprd +++ b/mkdumprd @@ -6,18 +6,29 @@ # Written by Cong Wang # +if [ -f /etc/sysconfig/kdump ]; then + . /etc/sysconfig/kdump +fi + [[ $dracutbasedir ]] || dracutbasedir=/usr/lib/dracut . $dracutbasedir/dracut-functions.sh . /lib/kdump/kdump-lib.sh export IN_KDUMP=1 +#initiate the kdump logger +dlog_init +if [ $? -ne 0 ]; then + echo "failed to initiate the kdump logger." + exit 1 +fi + conf_file="/etc/kdump.conf" SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" SAVE_PATH=$(get_save_path) OVERRIDE_RESETTABLE=0 extra_modules="" -dracut_args="--quiet --hostonly --hostonly-cmdline --hostonly-i18n --hostonly-mode strict -o \"plymouth dash resume ifcfg earlykdump\"" +dracut_args="--add kdumpbase --quiet --hostonly --hostonly-cmdline --hostonly-i18n --hostonly-mode strict -o \"plymouth dash resume ifcfg earlykdump\"" readonly MKDUMPRD_TMPDIR="$(mktemp -d -t mkdumprd.XXXXXX)" [ -d "$MKDUMPRD_TMPDIR" ] || perror_exit "dracut: mktemp -p -d -t dracut.XXXXXX failed." @@ -180,8 +191,8 @@ check_size() { fi if [ $avail -lt $memtotal ]; then - echo "Warning: There might not be enough space to save a vmcore." - echo " The size of $2 should be greater than $memtotal kilo bytes." + dwarn "Warning: There might not be enough space to save a vmcore." + dwarn " The size of $2 should be greater than $memtotal kilo bytes." fi } @@ -253,7 +264,7 @@ verify_core_collector() { if [ "$_cmd" != "makedumpfile" ]; then if is_raw_dump_target; then - echo "Warning: specifying a non-makedumpfile core collector, you will have to recover the vmcore manually." + dwarn "Warning: specifying a non-makedumpfile core collector, you will have to recover the vmcore manually." fi return fi @@ -343,7 +354,7 @@ is_unresettable() resettable="$(cat $path)" [ $resettable -eq 0 -a "$OVERRIDE_RESETTABLE" -eq 0 ] && { local device=$(udevadm info --query=all --path=/sys/dev/block/$1 | awk -F= '/DEVNAME/{print $2}') - echo "Error: Can not save vmcore because device $device is unresettable" + derror "Error: Can not save vmcore because device $device is unresettable" return 0 } fi @@ -377,7 +388,7 @@ is_crypt() eval "$line" [[ "$ID_FS_TYPE" = "crypto_LUKS" ]] && { dev=$(udevadm info --query=all --path=/sys/dev/block/$majmin | awk -F= '/DEVNAME/{print $2}') - echo "Device $dev is encrypted." + derror "Device $dev is encrypted." return 0 } return 1 @@ -400,7 +411,7 @@ if ! check_resettable; then fi if ! check_crypt; then - echo "Warning: Encrypted device is in dump path, which is not recommended, see kexec-kdump-howto.txt for more details." + dwarn "Warning: Encrypted device is in dump path, which is not recommended, see kexec-kdump-howto.txt for more details." fi # firstly get right SSH_KEY_LOCATION diff --git a/tests/scripts/testcases/local-kdump/0-local.sh b/tests/scripts/testcases/local-kdump/0-local.sh index d09c6f0..948cfa1 100755 --- a/tests/scripts/testcases/local-kdump/0-local.sh +++ b/tests/scripts/testcases/local-kdump/0-local.sh @@ -8,7 +8,7 @@ on_test() { if [ $boot_count -eq 1 ]; then cat << EOF > /etc/kdump.conf path /var/crash -core_collector makedumpfile -l --message-level 1 -d 31 +core_collector makedumpfile -l --message-level 7 -d 31 EOF kdumpctl start || test_failed "Failed to start kdump" diff --git a/tests/scripts/testcases/nfs-kdump/1-client.sh b/tests/scripts/testcases/nfs-kdump/1-client.sh index df62463..4122bb8 100755 --- a/tests/scripts/testcases/nfs-kdump/1-client.sh +++ b/tests/scripts/testcases/nfs-kdump/1-client.sh @@ -11,7 +11,7 @@ on_test() { if [ "$boot_count" -eq 1 ]; then cat << EOF > /etc/kdump.conf nfs $nfs_server:/srv/nfs -core_collector makedumpfile -l --message-level 1 -d 31 +core_collector makedumpfile -l --message-level 7 -d 31 EOF while ! ping -c 1 $nfs_server -W 1; do diff --git a/tests/scripts/testcases/ssh-kdump/1-client.sh b/tests/scripts/testcases/ssh-kdump/1-client.sh index d79f00f..bd03eff 100755 --- a/tests/scripts/testcases/ssh-kdump/1-client.sh +++ b/tests/scripts/testcases/ssh-kdump/1-client.sh @@ -11,7 +11,7 @@ on_test() { if [ "$boot_count" -eq 1 ]; then cat << EOF > /etc/kdump.conf ssh root@192.168.77.1 -core_collector makedumpfile -l --message-level 1 -d 31 -F +core_collector makedumpfile -l --message-level 7 -d 31 -F EOF ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa <<< y &>/dev/ttyS1