diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3e4cfae --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +SOURCES/eppic_050615.tar.gz +SOURCES/kdump-anaconda-addon-003-23-g80e78fb.tar.gz +SOURCES/kexec-tools-2.0.14.tar.xz +SOURCES/makedumpfile-1.6.1.tar.gz diff --git a/.kexec-tools.metadata b/.kexec-tools.metadata new file mode 100644 index 0000000..429335b --- /dev/null +++ b/.kexec-tools.metadata @@ -0,0 +1,4 @@ +a096c8e0892b559f40b01916aae240652f75b68a SOURCES/eppic_050615.tar.gz +dfa17c4a04bc3775ede56c1fd3a8904e938f0fe1 SOURCES/kdump-anaconda-addon-003-23-g80e78fb.tar.gz +cc3d6155ffb2aac5cf943312b41c0f41141dc9c6 SOURCES/kexec-tools-2.0.14.tar.xz +74ec93ff2709542772175ae7761a2852adce8c35 SOURCES/makedumpfile-1.6.1.tar.gz diff --git a/README.md b/README.md deleted file mode 100644 index 0e7897f..0000000 --- a/README.md +++ /dev/null @@ -1,5 +0,0 @@ -The master branch has no content - -Look at the c7 branch if you are working with CentOS-7, or the c4/c5/c6 branch for CentOS-4, 5 or 6 - -If you find this file in a distro specific branch, it means that no content has been checked in yet diff --git a/SOURCES/98-kexec.rules b/SOURCES/98-kexec.rules new file mode 100644 index 0000000..e32ee13 --- /dev/null +++ b/SOURCES/98-kexec.rules @@ -0,0 +1,4 @@ +SUBSYSTEM=="cpu", ACTION=="add", PROGRAM="/bin/systemctl try-restart kdump.service" +SUBSYSTEM=="cpu", ACTION=="remove", PROGRAM="/bin/systemctl try-restart kdump.service" +SUBSYSTEM=="memory", ACTION=="online", PROGRAM="/bin/systemctl try-restart kdump.service" +SUBSYSTEM=="memory", ACTION=="offline", PROGRAM="/bin/systemctl try-restart kdump.service" diff --git a/SOURCES/dracut-kdump-capture.service b/SOURCES/dracut-kdump-capture.service new file mode 100644 index 0000000..57139c9 --- /dev/null +++ b/SOURCES/dracut-kdump-capture.service @@ -0,0 +1,30 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Kdump Vmcore Save Service +After=initrd.target initrd-parse-etc.service sysroot.mount +After=dracut-initqueue.service dracut-pre-mount.service dracut-mount.service dracut-pre-pivot.service +Before=initrd-cleanup.service +ConditionPathExists=/etc/initrd-release +OnFailure=emergency.target +OnFailureIsolate=yes + +[Service] +Environment=DRACUT_SYSTEMD=1 +Environment=NEWROOT=/sysroot +Type=oneshot +ExecStart=/bin/kdump.sh +StandardInput=null +StandardOutput=syslog +StandardError=syslog+console +KillMode=process +RemainAfterExit=yes + +# Bash ignores SIGTERM, so we send SIGHUP instead, to ensure that bash +# terminates cleanly. +KillSignal=SIGHUP diff --git a/SOURCES/dracut-kdump-emergency.service b/SOURCES/dracut-kdump-emergency.service new file mode 100644 index 0000000..e023284 --- /dev/null +++ b/SOURCES/dracut-kdump-emergency.service @@ -0,0 +1,28 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +# This service will be placed in kdump initramfs and replace both the systemd +# emergency service and dracut emergency shell. IOW, any emergency will be +# kick this service and in turn isolating to kdump error handler. + +[Unit] +Description=Kdump Emergency +DefaultDependencies=no +IgnoreOnIsolate=yes + +[Service] +ExecStart=/usr/bin/systemctl --no-block isolate kdump-error-handler.service +Type=oneshot +StandardInput=tty-force +StandardOutput=inherit +StandardError=inherit +KillMode=process +IgnoreSIGPIPE=no + +# Bash ignores SIGTERM, so we send SIGHUP instead, to ensure that bash +# terminates cleanly. +KillSignal=SIGHUP diff --git a/SOURCES/dracut-kdump-emergency.target b/SOURCES/dracut-kdump-emergency.target new file mode 100644 index 0000000..a1bb493 --- /dev/null +++ b/SOURCES/dracut-kdump-emergency.target @@ -0,0 +1,14 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Emergency Mode +Documentation=man:systemd.special(7) +Requires=emergency.service +After=emergency.service +AllowIsolate=yes +IgnoreOnIsolate=yes diff --git a/SOURCES/dracut-kdump-error-handler.service b/SOURCES/dracut-kdump-error-handler.service new file mode 100644 index 0000000..13090be --- /dev/null +++ b/SOURCES/dracut-kdump-error-handler.service @@ -0,0 +1,34 @@ +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +# This service will run the real kdump error handler code. Executing the +# default action configured in kdump.conf + +[Unit] +Description=Kdump Error Handler +DefaultDependencies=no +After=systemd-vconsole-setup.service +Wants=systemd-vconsole-setup.service +AllowIsolate=yes + +[Service] +Environment=HOME=/ +Environment=DRACUT_SYSTEMD=1 +Environment=NEWROOT=/sysroot +WorkingDirectory=/ +ExecStart=/bin/kdump-error-handler.sh +ExecStopPost=-/usr/bin/systemctl --fail --no-block default +Type=oneshot +StandardInput=tty-force +StandardOutput=inherit +StandardError=inherit +KillMode=process +IgnoreSIGPIPE=no + +# Bash ignores SIGTERM, so we send SIGHUP instead, to ensure that bash +# terminates cleanly. +KillSignal=SIGHUP diff --git a/SOURCES/dracut-kdump-error-handler.sh b/SOURCES/dracut-kdump-error-handler.sh new file mode 100755 index 0000000..2f0f1d1 --- /dev/null +++ b/SOURCES/dracut-kdump-error-handler.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +. /lib/kdump-lib-initramfs.sh + +set -o pipefail +export PATH=$PATH:$KDUMP_SCRIPT_DIR + +get_kdump_confs +do_default_action +do_final_action diff --git a/SOURCES/dracut-kdump.sh b/SOURCES/dracut-kdump.sh new file mode 100755 index 0000000..42ba37f --- /dev/null +++ b/SOURCES/dracut-kdump.sh @@ -0,0 +1,204 @@ +#!/bin/sh + +# continue here only if we have to save dump. +if [ -f /etc/fadump.initramfs ] && [ ! -f /proc/device-tree/rtas/ibm,kernel-dump ]; then + exit 0 +fi + +exec &> /dev/console +. /lib/dracut-lib.sh +. /lib/kdump-lib-initramfs.sh + +set -o pipefail +DUMP_RETVAL=0 + +export PATH=$PATH:$KDUMP_SCRIPT_DIR + +do_dump() +{ + local _ret + + eval $DUMP_INSTRUCTION + _ret=$? + + if [ $_ret -ne 0 ]; then + echo "kdump: saving vmcore failed" + fi + + return $_ret +} + +do_kdump_pre() +{ + if [ -n "$KDUMP_PRE" ]; then + "$KDUMP_PRE" + fi +} + +do_kdump_post() +{ + if [ -n "$KDUMP_POST" ]; then + "$KDUMP_POST" "$1" + fi +} + +add_dump_code() +{ + DUMP_INSTRUCTION=$1 +} + +dump_raw() +{ + local _raw=$1 + + [ -b "$_raw" ] || return 1 + + echo "kdump: saving to raw disk $_raw" + + if ! $(echo -n $CORE_COLLECTOR|grep -q makedumpfile); then + _src_size=`ls -l /proc/vmcore | cut -d' ' -f5` + _src_size_mb=$(($_src_size / 1048576)) + monitor_dd_progress $_src_size_mb & + fi + + echo "kdump: saving vmcore" + $CORE_COLLECTOR /proc/vmcore | dd of=$_raw bs=$DD_BLKSIZE >> /tmp/dd_progress_file 2>&1 || return 1 + sync + + echo "kdump: saving vmcore complete" + return 0 +} + +dump_ssh() +{ + local _opt="-i $1 -o BatchMode=yes -o StrictHostKeyChecking=yes" + local _dir="$KDUMP_PATH/$HOST_IP-$DATEDIR" + local _host=$2 + + echo "kdump: saving to $_host:$_dir" + + cat /var/lib/random-seed > /dev/urandom + ssh -q $_opt $_host mkdir -p $_dir || return 1 + + save_vmcore_dmesg_ssh ${DMESG_COLLECTOR} ${_dir} "${_opt}" $_host + + echo "kdump: saving vmcore" + + if [ "${CORE_COLLECTOR%%[[:blank:]]*}" = "scp" ]; then + scp -q $_opt /proc/vmcore "$_host:$_dir/vmcore-incomplete" || return 1 + ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore" || return 1 + else + $CORE_COLLECTOR /proc/vmcore | ssh $_opt $_host "dd bs=512 of=$_dir/vmcore-incomplete" || return 1 + ssh $_opt $_host "mv $_dir/vmcore-incomplete $_dir/vmcore.flat" || return 1 + fi + + echo "kdump: saving vmcore complete" + return 0 +} + +save_vmcore_dmesg_ssh() { + local _dmesg_collector=$1 + local _path=$2 + local _opts="$3" + local _location=$4 + + echo "kdump: saving vmcore-dmesg.txt" + $_dmesg_collector /proc/vmcore | ssh $_opts $_location "dd of=$_path/vmcore-dmesg-incomplete.txt" + _exitcode=$? + + if [ $_exitcode -eq 0 ]; then + ssh -q $_opts $_location mv $_path/vmcore-dmesg-incomplete.txt $_path/vmcore-dmesg.txt + echo "kdump: saving vmcore-dmesg.txt complete" + else + echo "kdump: saving vmcore-dmesg.txt failed" + fi +} + +get_host_ip() +{ + local _host + if is_nfs_dump_target || is_ssh_dump_target + then + kdumpnic=$(getarg kdumpnic=) + [ -z "$kdumpnic" ] && echo "kdump: failed to get kdumpnic!" && return 1 + _host=`ip addr show dev $kdumpnic|grep '[ ]*inet'` + [ $? -ne 0 ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1 + _host=`echo $_host | head -n 1 | cut -d' ' -f2` + _host="${_host%%/*}" + [ -z "$_host" ] && echo "kdump: wrong kdumpnic: $kdumpnic" && return 1 + HOST_IP=$_host + fi + return 0 +} + +read_kdump_conf() +{ + if [ ! -f "$KDUMP_CONF" ]; then + echo "kdump: $KDUMP_CONF not found" + return + fi + + get_kdump_confs + + # rescan for add code for dump target + while read config_opt config_val; + do + # remove inline comments after the end of a directive. + config_val=$(strip_comments $config_val) + case "$config_opt" in + dracut_args) + config_val=$(get_dracut_args_target "$config_val") + [[ -n "$config_val" ]] && add_dump_code "dump_fs $config_val" + ;; + ext[234]|xfs|btrfs|minix|nfs) + add_dump_code "dump_fs $config_val" + ;; + raw) + add_dump_code "dump_raw $config_val" + ;; + ssh) + add_dump_code "dump_ssh $SSH_KEY_LOCATION $config_val" + ;; + esac + done < $KDUMP_CONF +} + +fence_kdump_notify() +{ + if [ -n "$FENCE_KDUMP_NODES" ]; then + $FENCE_KDUMP_SEND $FENCE_KDUMP_ARGS $FENCE_KDUMP_NODES & + fi +} + +read_kdump_conf +fence_kdump_notify + +get_host_ip +if [ $? -ne 0 ]; then + echo "kdump: get_host_ip exited with non-zero status!" + exit 1 +fi + +if [ -z "$DUMP_INSTRUCTION" ]; then + add_dump_code "dump_fs $NEWROOT" +fi + +do_kdump_pre +if [ $? -ne 0 ]; then + echo "kdump: kdump_pre script exited with non-zero status!" + do_final_action +fi +make_trace_mem "kdump saving vmcore" '1:shortmem' '2+:mem' '3+:slab' +do_dump +DUMP_RETVAL=$? + +do_kdump_post $DUMP_RETVAL +if [ $? -ne 0 ]; then + echo "kdump: kdump_post script exited with non-zero status!" +fi + +if [ $DUMP_RETVAL -ne 0 ]; then + exit 1 +fi + +do_final_action diff --git a/SOURCES/dracut-module-setup.sh b/SOURCES/dracut-module-setup.sh new file mode 100755 index 0000000..3dea978 --- /dev/null +++ b/SOURCES/dracut-module-setup.sh @@ -0,0 +1,759 @@ +#!/bin/bash + +. $dracutfunctions +. /lib/kdump/kdump-lib.sh + +if ! [[ -d "${initdir}/tmp" ]]; then + mkdir -p "${initdir}/tmp" +fi + +check() { + [[ $debug ]] && set -x + #kdumpctl sets this explicitly + if [ -z "$IN_KDUMP" ] || [ ! -f /etc/kdump.conf ] + then + return 1 + fi + return 0 +} + +depends() { + local _dep="base shutdown" + + if [ -d /sys/module/drm/drivers ]; then + _dep="$_dep drm" + fi + + if [ is_generic_fence_kdump -o is_pcs_fence_kdump ]; then + _dep="$_dep network" + fi + + echo $_dep + return 0 +} + +kdump_to_udev_name() { + local dev="${1//\"/}" + + case "$dev" in + UUID=*) + dev=`blkid -U "${dev#UUID=}"` + ;; + LABEL=*) + dev=`blkid -L "${dev#LABEL=}"` + ;; + esac + echo $(get_persistent_dev "$dev") +} + +kdump_is_bridge() { + [ -d /sys/class/net/"$1"/bridge ] +} + +kdump_is_bond() { + [ -d /sys/class/net/"$1"/bonding ] +} + +kdump_is_team() { + [ -f /usr/bin/teamnl ] && teamnl $1 ports &> /dev/null +} + +kdump_is_vlan() { + [ -f /proc/net/vlan/"$1" ] +} + +# $1: netdev name +source_ifcfg_file() { + local ifcfg_file + + ifcfg_file=$(get_ifcfg_filename $1) + if [ -f "${ifcfg_file}" ]; then + . ${ifcfg_file} + else + dwarning "The ifcfg file of $1 is not found!" + fi +} + +# $1: netdev name +kdump_setup_dns() { + local _nameserver _dns + local _dnsfile=${initdir}/etc/cmdline.d/42dns.conf + + source_ifcfg_file $1 + + [ -n "$DNS1" ] && echo "nameserver=$DNS1" > "$_dnsfile" + [ -n "$DNS2" ] && echo "nameserver=$DNS2" >> "$_dnsfile" + + while read content; + do + _nameserver=$(echo $content | grep ^nameserver) + [ -z "$_nameserver" ] && continue + + _dns=$(echo $_nameserver | cut -d' ' -f2) + [ -z "$_dns" ] && continue + + if [ ! -f $_dnsfile ] || [ ! $(cat $_dnsfile | grep -q $_dns) ]; then + echo "nameserver=$_dns" >> "$_dnsfile" + fi + done < "/etc/resolv.conf" +} + +#$1: netdev name +#$2: srcaddr +#if it use static ip echo it, or echo null +kdump_static_ip() { + local _netdev="$1" _srcaddr="$2" _ipv6_flag + local _netmask _gateway _ipaddr _target _nexthop + + _ipaddr=$(ip addr show dev $_netdev permanent | awk "/ $_srcaddr\/.* /{print \$2}") + + if is_ipv6_address $_srcaddr; then + _ipv6_flag="-6" + fi + + if [ -n "$_ipaddr" ]; then + _gateway=$(ip $_ipv6_flag route list dev $_netdev | \ + awk '/^default /{print $3}' | head -n 1) + + if [ "x" != "x"$_ipv6_flag ]; then + # _ipaddr="2002::56ff:feb6:56d5/64", _netmask is the number after "/" + _netmask=${_ipaddr#*\/} + _srcaddr="[$_srcaddr]" + _gateway="[$_gateway]" + else + _netmask=$(ipcalc -m $_ipaddr | cut -d'=' -f2) + fi + echo -n "${_srcaddr}::${_gateway}:${_netmask}::" + fi + + /sbin/ip $_ipv6_flag route show | grep -v default | grep ".*via.* $_netdev " |\ + while read _route; do + _target=`echo $_route | cut -d ' ' -f1` + _nexthop=`echo $_route | cut -d ' ' -f3` + if [ "x" != "x"$_ipv6_flag ]; then + _target="[$_target]" + _nexthop="[$_nexthop]" + fi + echo "rd.route=$_target:$_nexthop:$_netdev" + done >> ${initdir}/etc/cmdline.d/45route-static.conf +} + +kdump_get_mac_addr() { + cat /sys/class/net/$1/address +} + +#Bonding or team master modifies the mac address +#of its slaves, we should use perm address +kdump_get_perm_addr() { + local addr=$(ethtool -P $1 | sed -e 's/Permanent address: //') + if [ -z "$addr" ] || [ "$addr" = "00:00:00:00:00:00" ] + then + derror "Can't get the permanent address of $1" + else + echo "$addr" + fi +} + +# Prefix kernel assigned names with "kdump-". EX: eth0 -> kdump-eth0 +# Because kernel assigned names are not persistent between 1st and 2nd +# kernel. We could probably end up with eth0 being eth1, eth0 being +# eth1, and naming conflict happens. +kdump_setup_ifname() { + local _ifname + + # If ifname already has 'kdump-' prefix, we must be switching from + # fadump to kdump. Skip prefixing 'kdump-' in this case as adding + # another prefix may truncate the ifname. Since an ifname with + # 'kdump-' is already persistent, this should be fine. + if [[ $1 =~ eth* ]] && [[ ! $1 =~ ^kdump-* ]]; then + _ifname="kdump-$1" + else + _ifname="$1" + fi + + echo "$_ifname" +} + +kdump_setup_bridge() { + local _netdev=$1 + local _brif _dev _mac _kdumpdev + for _dev in `ls /sys/class/net/$_netdev/brif/`; do + _kdumpdev=$_dev + if kdump_is_bond "$_dev"; then + kdump_setup_bond "$_dev" + elif kdump_is_team "$_dev"; then + kdump_setup_team "$_dev" + elif kdump_is_vlan "$_dev"; then + kdump_setup_vlan "$_dev" + else + _mac=$(kdump_get_mac_addr $_dev) + _kdumpdev=$(kdump_setup_ifname $_dev) + echo -n " ifname=$_kdumpdev:$_mac" >> ${initdir}/etc/cmdline.d/41bridge.conf + fi + _brif+="$_kdumpdev," + done + echo " bridge=$_netdev:$(echo $_brif | sed -e 's/,$//')" >> ${initdir}/etc/cmdline.d/41bridge.conf +} + +kdump_setup_bond() { + local _netdev=$1 + local _dev _mac _slaves _kdumpdev + for _dev in `cat /sys/class/net/$_netdev/bonding/slaves`; do + _mac=$(kdump_get_perm_addr $_dev) + _kdumpdev=$(kdump_setup_ifname $_dev) + echo -n " ifname=$_kdumpdev:$_mac" >> ${initdir}/etc/cmdline.d/42bond.conf + _slaves+="$_kdumpdev," + done + echo -n " bond=$_netdev:$(echo $_slaves | sed 's/,$//')" >> ${initdir}/etc/cmdline.d/42bond.conf + # Get bond options specified in ifcfg + + source_ifcfg_file $_netdev + + bondoptions="$(echo :$BONDING_OPTS | sed 's/\s\+/,/')" + echo "$bondoptions" >> ${initdir}/etc/cmdline.d/42bond.conf +} + +kdump_setup_team() { + local _netdev=$1 + local _dev _mac _slaves _kdumpdev + for _dev in `teamnl $_netdev ports | awk -F':' '{print $2}'`; do + _mac=$(kdump_get_perm_addr $_dev) + _kdumpdev=$(kdump_setup_ifname $_dev) + echo -n " ifname=$_kdumpdev:$_mac" >> ${initdir}/etc/cmdline.d/44team.conf + _slaves+="$_kdumpdev," + done + echo " team=$_netdev:$(echo $_slaves | sed -e 's/,$//')" >> ${initdir}/etc/cmdline.d/44team.conf + #Buggy version teamdctl outputs to stderr! + #Try to use the latest version of teamd. + teamdctl "$_netdev" config dump > ${initdir}/tmp/$$-$_netdev.conf + if [ $? -ne 0 ] + then + derror "teamdctl failed." + exit 1 + fi + inst_dir /etc/teamd + inst_simple ${initdir}/tmp/$$-$_netdev.conf "/etc/teamd/$_netdev.conf" + rm -f ${initdir}/tmp/$$-$_netdev.conf +} + +kdump_setup_vlan() { + local _netdev=$1 + local _phydev="$(awk '/^Device:/{print $2}' /proc/net/vlan/"$_netdev")" + local _netmac="$(kdump_get_mac_addr $_phydev)" + local _kdumpdev + + #Just support vlan over bond, it is not easy + #to support all other complex setup + if kdump_is_bridge "$_phydev"; then + derror "Vlan over bridge is not supported!" + exit 1 + elif kdump_is_team "$_phydev"; then + derror "Vlan over team is not supported!" + exit 1 + elif kdump_is_bond "$_phydev"; then + kdump_setup_bond "$_phydev" + echo " vlan=$_netdev:$_phydev" > ${initdir}/etc/cmdline.d/43vlan.conf + else + _kdumpdev="$(kdump_setup_ifname $_phydev)" + echo " vlan=$_netdev:$_kdumpdev ifname=$_kdumpdev:$_netmac" > ${initdir}/etc/cmdline.d/43vlan.conf + fi +} + +# setup s390 znet cmdline +# $1: netdev name +kdump_setup_znet() { + local _options="" + + source_ifcfg_file $1 + + for i in $OPTIONS; do + _options=${_options},$i + done + echo rd.znet=${NETTYPE},${SUBCHANNELS}${_options} > ${initdir}/etc/cmdline.d/30znet.conf +} + +# Setup dracut to bringup a given network interface +kdump_setup_netdev() { + local _netdev=$1 _srcaddr=$2 + local _static _proto _ip_conf _ip_opts _ifname_opts + + if [ "$(uname -m)" = "s390x" ]; then + kdump_setup_znet $_netdev + fi + + _netmac=$(kdump_get_mac_addr $_netdev) + _static=$(kdump_static_ip $_netdev $_srcaddr) + if [ -n "$_static" ]; then + _proto=none + else + _proto=dhcp + fi + + _ip_conf="${initdir}/etc/cmdline.d/40ip.conf" + _ip_opts=" ip=${_static}$(kdump_setup_ifname $_netdev):${_proto}" + + # dracut doesn't allow duplicated configuration for same NIC, even they're exactly the same. + # so we have to avoid adding duplicates + # We should also check /proc/cmdline for existing ip=xx arg. + # For example, iscsi boot will specify ip=xxx arg in cmdline. + if [ ! -f $_ip_conf ] || ! grep -q $_ip_opts $_ip_conf &&\ + ! grep -q "ip=[^[:space:]]*$_netdev" /proc/cmdline; then + echo "$_ip_opts" >> $_ip_conf + fi + + if kdump_is_bridge "$_netdev"; then + kdump_setup_bridge "$_netdev" + elif kdump_is_bond "$_netdev"; then + kdump_setup_bond "$_netdev" + elif kdump_is_team "$_netdev"; then + kdump_setup_team "$_netdev" + elif kdump_is_vlan "$_netdev"; then + kdump_setup_vlan "$_netdev" + else + _ifname_opts=" ifname=$(kdump_setup_ifname $_netdev):$(kdump_get_mac_addr $_netdev)" + echo "$_ifname_opts" >> $_ip_conf + fi + + kdump_setup_dns "$_netdev" +} + +get_ip_route_field() +{ + if `echo $1 | grep -q $2`; then + echo ${1##*$2} | cut -d ' ' -f1 + fi +} + +#Function:kdump_install_net +#$1: config values of net line in kdump.conf +#$2: srcaddr of network device +kdump_install_net() { + local _server _netdev _srcaddr _route _serv_tmp + local config_val="$1" + + _server=$(get_remote_host $config_val) + + if is_hostname $_server; then + _serv_tmp=`getent ahosts $_server | grep -v : | head -n 1` + if [ -z "$_serv_tmp" ]; then + _serv_tmp=`getent ahosts $_server | head -n 1` + fi + _server=`echo $_serv_tmp | cut -d' ' -f1` + fi + + _route=`/sbin/ip -o route get to $_server 2>&1` + [ $? != 0 ] && echo "Bad kdump location: $config_val" && exit 1 + + #the field in the ip output changes if we go to another subnet + _srcaddr=$(get_ip_route_field "$_route" "src") + _netdev=$(get_ip_route_field "$_route" "dev") + + kdump_setup_netdev "${_netdev}" "${_srcaddr}" + + #save netdev used for kdump as cmdline + # Whoever calling kdump_install_net() is setting up the default gateway, + # ie. bootdev/kdumpnic. So don't override the setting if calling + # kdump_install_net() for another time. For example, after setting eth0 as + # the default gate way for network dump, eth1 in the fence kdump path will + # call kdump_install_net again and we don't want eth1 to be the default + # gateway. + if [ ! -f ${initdir}/etc/cmdline.d/60kdumpnic.conf ] && + [ ! -f ${initdir}/etc/cmdline.d/70bootdev.conf ]; then + echo "kdumpnic=$(kdump_setup_ifname $_netdev)" > ${initdir}/etc/cmdline.d/60kdumpnic.conf + echo "bootdev=$(kdump_setup_ifname $_netdev)" > ${initdir}/etc/cmdline.d/70bootdev.conf + fi +} + +default_dump_target_install_conf() +{ + local _target _fstype + local _mntpoint _save_path + + is_user_configured_dump_target && return + + _save_path=$(get_option_value "path") + [ -z "$_save_path" ] && _save_path=$DEFAULT_PATH + + # strip the duplicated "/" + _save_path=$(echo $_save_path | tr -s /) + + _mntpoint=$(get_mntpoint_from_path $_save_path) + _target=$(get_target_from_path $_save_path) + + if is_atomic && is_bind_mount $_mntpoint; then + _save_path=${_save_path##"$_mntpoint"} + # the real dump path in the 2nd kernel, if the mount point is bind mounted. + _save_path=$(get_bind_mount_directory $_mntpoint)/$_save_path + _mntpoint=$(get_mntpoint_from_target $_target) + + # the absolute path in the 1st kernel + _save_path=$_mntpoint/$_save_path + fi + + if [ "$_mntpoint" != "/" ]; then + _fstype=$(get_fs_type_from_target $_target) + + if $(is_fs_type_nfs $_fstype); then + kdump_install_net "$_target" + _fstype="nfs" + else + _target=$(kdump_to_udev_name $_target) + fi + + echo "$_fstype $_target" >> ${initdir}/tmp/$$-kdump.conf + + # strip the duplicated "/" + _save_path=$(echo $_save_path | tr -s /) + _save_path=${_save_path##"$_mntpoint"} + fi + + #erase the old path line, then insert the parsed path + sed -i "/^path/d" ${initdir}/tmp/$$-kdump.conf + echo "path $_save_path" >> ${initdir}/tmp/$$-kdump.conf +} + +adjust_bind_mount_path() +{ + local _target=$1 + local _save_path=$(get_option_value "path") + [ -z "$_save_path" ] && _save_path=$DEFAULT_PATH + + # strip the duplicated "/" + _save_path=$(echo $_save_path | tr -s /) + + local _absolute_save_path=$(get_mntpoint_from_target $_target)/$_save_path + _absolute_save_path=$(echo "$_absolute_save_path" | tr -s /) + local _mntpoint=$(get_mntpoint_from_path $_absolute_save_path) + + if is_bind_mount $_mntpoint; then + _save_path=${_absolute_save_path##"$_mntpoint"} + # the real dump path in the 2nd kernel, if the mount point is bind mounted. + _save_path=$(get_bind_mount_directory $_mntpoint)/$_save_path + + #erase the old path line, then insert the parsed path + sed -i "/^path/d" ${initdir}/tmp/$$-kdump.conf + echo "path $_save_path" >> ${initdir}/tmp/$$-kdump.conf + fi +} + +#install kdump.conf and what user specifies in kdump.conf +kdump_install_conf() { + sed -ne '/^#/!p' /etc/kdump.conf > ${initdir}/tmp/$$-kdump.conf + + while read config_opt config_val; + do + # remove inline comments after the end of a directive. + config_val=$(strip_comments $config_val) + case "$config_opt" in + ext[234]|xfs|btrfs|minix|raw) + sed -i -e "s#^$config_opt[[:space:]]\+$config_val#$config_opt $(kdump_to_udev_name $config_val)#" ${initdir}/tmp/$$-kdump.conf + if is_atomic; then + adjust_bind_mount_path "$config_val" + fi + ;; + ssh|nfs) + kdump_install_net "$config_val" + ;; + dracut_args) + if [[ $(get_dracut_args_fstype "$config_val") = nfs* ]] ; then + kdump_install_net "$(get_dracut_args_target "$config_val")" + fi + ;; + kdump_pre|kdump_post|extra_bins) + dracut_install $config_val + ;; + core_collector) + dracut_install "${config_val%%[[:blank:]]*}" + ;; + esac + done < /etc/kdump.conf + + default_dump_target_install_conf + + kdump_configure_fence_kdump "${initdir}/tmp/$$-kdump.conf" + inst "${initdir}/tmp/$$-kdump.conf" "/etc/kdump.conf" + rm -f ${initdir}/tmp/$$-kdump.conf +} + +# Default sysctl parameters should suffice for kdump kernel. +# Remove custom configurations sysctl.conf & sysctl.d/* +remove_sysctl_conf() { + + # As custom configurations like vm.min_free_kbytes can lead + # to OOM issues in kdump kernel, avoid them + rm -f "${initdir}/etc/sysctl.conf" + rm -rf "${initdir}/etc/sysctl.d" + rm -rf "${initdir}/run/sysctl.d" + rm -rf "${initdir}/usr/lib/sysctl.d" +} + +kdump_iscsi_get_rec_val() { + + local result + + # The open-iscsi 742 release changed to using flat files in + # /var/lib/iscsi. + + result=$(/sbin/iscsiadm --show -m session -r ${1} | grep "^${2} = ") + result=${result##* = } + echo $result +} + +kdump_get_iscsi_initiator() { + local _initiator + local initiator_conf="/etc/iscsi/initiatorname.iscsi" + + [ -f "$initiator_conf" ] || return 1 + + while read _initiator; do + [ -z "${_initiator%%#*}" ] && continue # Skip comment lines + + case $_initiator in + InitiatorName=*) + initiator=${_initiator#InitiatorName=} + echo "rd.iscsi.initiator=${initiator}" + return 0;; + *) ;; + esac + done < ${initiator_conf} + + return 1 +} + +# Figure out iBFT session according to session type +is_ibft() { + [ "$(kdump_iscsi_get_rec_val $1 "node.discovery_type")" = fw ] +} + +kdump_setup_iscsi_device() { + local path=$1 + local tgt_name; local tgt_ipaddr; + local username; local password; local userpwd_str; + local username_in; local password_in; local userpwd_in_str; + local netdev + local srcaddr + local idev + local netroot_str ; local initiator_str; + local netroot_conf="${initdir}/etc/cmdline.d/50iscsi.conf" + local initiator_conf="/etc/iscsi/initiatorname.iscsi" + + dinfo "Found iscsi component $1" + + # Check once before getting explicit values, so we can output a decent + # error message. + + if ! /sbin/iscsiadm -m session -r ${path} >/dev/null ; then + derror "Unable to find iscsi record for $path" + return 1 + fi + + if is_ibft ${path}; then + return + fi + + tgt_name=$(kdump_iscsi_get_rec_val ${path} "node.name") + tgt_ipaddr=$(kdump_iscsi_get_rec_val ${path} "node.conn\[0\].address") + + # get and set username and password details + username=$(kdump_iscsi_get_rec_val ${path} "node.session.auth.username") + [ "$username" == "" ] && username="" + password=$(kdump_iscsi_get_rec_val ${path} "node.session.auth.password") + [ "$password" == "" ] && password="" + username_in=$(kdump_iscsi_get_rec_val ${path} "node.session.auth.username_in") + [ -n "$username" ] && userpwd_str="$username:$password" + + # get and set incoming username and password details + [ "$username_in" == "" ] && username_in="" + password_in=$(kdump_iscsi_get_rec_val ${path} "node.session.auth.password_in") + [ "$password_in" == "" ] && password_in="" + + [ -n "$username_in" ] && userpwd_in_str=":$username_in:$password_in" + + netdev=$(/sbin/ip route get to ${tgt_ipaddr} | \ + sed 's|.*dev \(.*\).*|\1|g') + srcaddr=$(echo $netdev | awk '{ print $3; exit }') + netdev=$(echo $netdev | awk '{ print $1; exit }') + + kdump_setup_netdev $netdev $srcaddr + + # prepare netroot= command line + # FIXME: Do we need to parse and set other parameters like protocol, port + # iscsi_iface_name, netdev_name, LUN etc. + + if is_ipv6_address $tgt_ipaddr; then + tgt_ipaddr="[$tgt_ipaddr]" + fi + netroot_str="netroot=iscsi:${userpwd_str}${userpwd_in_str}@$tgt_ipaddr::::$tgt_name" + + [[ -f $netroot_conf ]] || touch $netroot_conf + + # If netroot target does not exist already, append. + if ! grep -q $netroot_str $netroot_conf; then + echo $netroot_str >> $netroot_conf + dinfo "Appended $netroot_str to $netroot_conf" + fi + + # Setup initator + initiator_str=$(kdump_get_iscsi_initiator) + [ $? -ne "0" ] && derror "Failed to get initiator name" && return 1 + + # If initiator details do not exist already, append. + if ! grep -q "$initiator_str" $netroot_conf; then + echo "$initiator_str" >> $netroot_conf + dinfo "Appended "$initiator_str" to $netroot_conf" + fi +} + +kdump_check_iscsi_targets () { + # If our prerequisites are not met, fail anyways. + type -P iscsistart >/dev/null || return 1 + + kdump_check_setup_iscsi() ( + local _dev + _dev=$1 + + [[ -L /sys/dev/block/$_dev ]] || return + cd "$(readlink -f /sys/dev/block/$_dev)" + until [[ -d sys || -d iscsi_session ]]; do + cd .. + done + [[ -d iscsi_session ]] && kdump_setup_iscsi_device "$PWD" + ) + + [[ $hostonly ]] || [[ $mount_needs ]] && { + for_each_host_dev_and_slaves_all kdump_check_setup_iscsi + } +} + +# retrieves fence_kdump nodes from Pacemaker cluster configuration +get_pcs_fence_kdump_nodes() { + local nodes + + # get cluster nodes from cluster cib, get interface and ip address + nodelist=`pcs cluster cib | xmllint --xpath "/cib/status/node_state/@uname" -` + + # nodelist is formed as 'uname="node1" uname="node2" ... uname="nodeX"' + # we need to convert each to node1, node2 ... nodeX in each iteration + for node in ${nodelist}; do + # convert $node from 'uname="nodeX"' to 'nodeX' + eval $node + nodename=$uname + # Skip its own node name + if [ "$nodename" = `hostname` -o "$nodename" = `hostname -s` ]; then + continue + fi + nodes="$nodes $nodename" + done + + echo $nodes +} + +# retrieves fence_kdump args from config file +get_pcs_fence_kdump_args() { + if [ -f $FENCE_KDUMP_CONFIG_FILE ]; then + . $FENCE_KDUMP_CONFIG_FILE + echo $FENCE_KDUMP_OPTS + fi +} + +# setup fence_kdump in cluster +# setup proper network and install needed files +kdump_configure_fence_kdump () { + local kdump_cfg_file=$1 + local nodes + local args + + if is_generic_fence_kdump; then + nodes=$(get_option_value "fence_kdump_nodes") + + elif is_pcs_fence_kdump; then + nodes=$(get_pcs_fence_kdump_nodes) + + # set appropriate options in kdump.conf + echo "fence_kdump_nodes $nodes" >> ${kdump_cfg_file} + + args=$(get_pcs_fence_kdump_args) + if [ -n "$args" ]; then + echo "fence_kdump_args $args" >> ${kdump_cfg_file} + fi + + else + # fence_kdump not configured + return 1 + fi + + # setup network for each node + for node in ${nodes}; do + kdump_install_net $node + done + + dracut_install $FENCE_KDUMP_SEND +} + +# Install a random seed used to feed /dev/urandom +# By the time kdump service starts, /dev/uramdom is already fed by systemd +kdump_install_random_seed() { + local poolsize=`cat /proc/sys/kernel/random/poolsize` + + if [ ! -d ${initdir}/var/lib/ ]; then + mkdir -p ${initdir}/var/lib/ + fi + + dd if=/dev/urandom of=${initdir}/var/lib/random-seed \ + bs=$poolsize count=1 2> /dev/null +} + +remove_cpu_online_rule() { + local file=${initdir}/usr/lib/udev/rules.d/40-redhat.rules + + sed -i '/SUBSYSTEM=="cpu"/d' $file +} + +install() { + local arch + + kdump_install_conf + remove_sysctl_conf + + # Onlining secondary cpus breaks kdump completely on KVM on Power hosts + # Though we use maxcpus=1 by default but 40-redhat.rules will bring up all + # possible cpus by default. (rhbz1270174 rhbz1266322) + # Thus before we get the kernel fix and the systemd rule fix let's remove + # the cpu online rule in kdump initramfs. + arch=$(uname -m) + if [[ "$arch" = "ppc64le" ]] || [[ "$arch" = "ppc64" ]]; then + remove_cpu_online_rule + fi + + if is_ssh_dump_target; then + kdump_install_random_seed + fi + dracut_install -o /etc/adjtime /etc/localtime + inst "$moddir/monitor_dd_progress" "/kdumpscripts/monitor_dd_progress" + chmod +x ${initdir}/kdumpscripts/monitor_dd_progress + inst "/bin/dd" "/bin/dd" + inst "/bin/tail" "/bin/tail" + inst "/bin/date" "/bin/date" + inst "/bin/sync" "/bin/sync" + inst "/bin/cut" "/bin/cut" + inst "/bin/head" "/bin/head" + inst "/sbin/makedumpfile" "/sbin/makedumpfile" + inst "/sbin/vmcore-dmesg" "/sbin/vmcore-dmesg" + inst "/lib/kdump/kdump-lib.sh" "/lib/kdump-lib.sh" + inst "/lib/kdump/kdump-lib-initramfs.sh" "/lib/kdump-lib-initramfs.sh" + inst "$moddir/kdump.sh" "/usr/bin/kdump.sh" + inst "$moddir/kdump-capture.service" "$systemdsystemunitdir/kdump-capture.service" + ln_r "$systemdsystemunitdir/kdump-capture.service" "$systemdsystemunitdir/initrd.target.wants/kdump-capture.service" + inst "$moddir/kdump-error-handler.sh" "/usr/bin/kdump-error-handler.sh" + inst "$moddir/kdump-error-handler.service" "$systemdsystemunitdir/kdump-error-handler.service" + # Replace existing emergency service and emergency target + cp "$moddir/kdump-emergency.service" "$initdir/$systemdsystemunitdir/emergency.service" + cp "$moddir/kdump-emergency.target" "$initdir/$systemdsystemunitdir/emergency.target" + # Also redirect dracut-emergency to kdump error handler + ln_r "$systemdsystemunitdir/emergency.service" "$systemdsystemunitdir/dracut-emergency.service" + + # Check for all the devices and if any device is iscsi, bring up iscsi + # target. Ideally all this should be pushed into dracut iscsi module + # at some point of time. + kdump_check_iscsi_targets +} diff --git a/SOURCES/dracut-monitor_dd_progress b/SOURCES/dracut-monitor_dd_progress new file mode 100644 index 0000000..e139d33 --- /dev/null +++ b/SOURCES/dracut-monitor_dd_progress @@ -0,0 +1,28 @@ +#!/bin/sh + +SRC_FILE_MB=$1 + +while true +do + DD_PID=`pidof dd` + if [ -n "$DD_PID" ]; then + break + fi +done + +while true +do + sleep 5 + if [ ! -d /proc/$DD_PID ]; then + break + fi + + kill -s USR1 $DD_PID + CURRENT_SIZE=`tail -n 1 /tmp/dd_progress_file | sed "s/[^0-9].*//g"` + [ -n "$CURRENT_SIZE" ] && { + CURRENT_MB=$(($CURRENT_SIZE / 1048576)) + echo -e "Copied $CURRENT_MB MB / $SRC_FILE_MB MB\r" + } +done + +rm -f /tmp/dd_progress_file diff --git a/SOURCES/fadump-howto.txt b/SOURCES/fadump-howto.txt new file mode 100644 index 0000000..1613d02 --- /dev/null +++ b/SOURCES/fadump-howto.txt @@ -0,0 +1,251 @@ +Firmware assisted dump (fadump) HOWTO + +Introduction + +Firmware assisted dump is a new feature in the 3.4 mainline kernel supported +only on powerpc architecture. The goal of firmware-assisted dump is to enable +the dump of a crashed system, and to do so from a fully-reset system, and to +minimize the total elapsed time until the system is back in production use. A +complete documentation on implementation can be found at +Documentation/powerpc/firmware-assisted-dump.txt in upstream linux kernel tree +from 3.4 version and above. + +Please note that the firmware-assisted dump feature is only available on Power6 +and above systems with recent firmware versions. + +Overview + +Fadump + +Fadump is a robust kernel crash dumping mechanism to get reliable kernel crash +dump with assistance from firmware. This approach does not use kexec, instead +firmware assists in booting the kdump kernel while preserving memory contents. +Unlike kdump, the system is fully reset, and loaded with a fresh copy of the +kernel. In particular, PCI and I/O devices are reinitialized and are in a +clean, consistent state. This second kernel, often called a capture kernel, +boots with very little memory and captures the dump image. + +The first kernel registers the sections of memory with the Power firmware for +dump preservation during OS initialization. These registered sections of memory +are reserved by the first kernel during early boot. When a system crashes, the +Power firmware fully resets the system, preserves all the system memory +contents, save the low memory (boot memory of size larger of 5% of system +RAM or 256MB) of RAM to the previous registered region. It will also save +system registers, and hardware PTE's. + +Fadump is supported only on ppc64 platform. The standard kernel and capture +kernel are one and the same on ppc64. + +If you're reading this document, you should already have kexec-tools +installed. If not, you install it via the following command: + + # yum install kexec-tools + +Fadump Operational Flow: + +Like kdump, fadump also exports the ELF formatted kernel crash dump through +/proc/vmcore. Hence existing kdump infrastructure can be used to capture fadump +vmcore. The idea is to keep the functionality transparent to end user. From +user perspective there is no change in the way kdump init script works. + +However, unlike kdump, fadump does not pre-load kdump kernel and initrd into +reserved memory, instead it always uses default OS initrd during second boot +after crash. Hence, for fadump, we rebuild the new kdump initrd and replace it +with default initrd. Before replacing existing default initrd we take a backup +of original default initrd for user's reference. The dracut package has been +enhanced to rebuild the default initrd with vmcore capture steps. The initrd +image is rebuilt as per the configuration in /etc/kdump.conf file. + +The control flow of fadump works as follows: +01. System panics. +02. At the crash, kernel informs power firmware that kernel has crashed. +03. Firmware takes the control and reboots the entire system preserving + only the memory (resets all other devices). +04. The reboot follows the normal booting process (non-kexec). +05. The boot loader loads the default kernel and initrd from /boot +06. The default initrd loads and runs /init +07. dracut-kdump.sh script present in fadump aware default initrd checks if + '/proc/device-tree/rtas/ibm,kernel-dump' file exists before executing + steps to capture vmcore. + (This check will help to bypass the vmcore capture steps during normal boot + process.) +09. Captures dump according to /etc/kdump.conf +10. Is dump capture successful (yes goto 12, no goto 11) +11. Perfom the default action specified in /etc/kdump.conf (Default action + is reboot, if unspecified) +12. Reboot + + +How to configure fadump: + +Again, we assume if you're reading this document, you should already have +kexec-tools installed. If not, you install it via the following command: + + # yum install kexec-tools + +To be able to do much of anything interesting in the way of debug analysis, +you'll also need to install the kernel-debuginfo package, of the same arch +as your running kernel, and the crash utility: + + # yum --enablerepo=\*debuginfo install kernel-debuginfo.$(uname -m) crash + +Next up, we need to modify some boot parameters to enable firmware assisted +dump. With the help of grubby, it's very easy to append "fadump=on" to the end +of your kernel boot parameters. Optionally, user can also append +'fadump_reserve_mem=X' kernel cmdline to specify size of the memory to reserve +for boot memory dump preservation. + + # grubby --args="fadump=on" --update-kernel=/boot/vmlinuz-`uname -r` + +The term 'boot memory' means size of the low memory chunk that is required for +a kernel to boot successfully when booted with restricted memory. By default, +the boot memory size will be the larger of 5% of system RAM or 256MB. +Alternatively, user can also specify boot memory size through boot parameter +'fadump_reserve_mem=' which will override the default calculated size. Use this +option if default boot memory size is not sufficient for second kernel to boot +successfully. + +After making said changes, reboot your system, so that the specified memory is +reserved and left untouched by the normal system. Take note that the output of +'free -m' will show X MB less memory than without this parameter, which is +expected. If you see OOM (Out Of Memory) error messages while loading capture +kernel, then you should bump up the memory reservation size. + +Now that you've got that reserved memory region set up, you want to turn on +the kdump init script: + + # systemctl enable kdump.service + +Then, start up kdump as well: + + # systemctl start kdump.service + +This should turn on the firmware assisted functionality in kernel by +echo'ing 1 to /sys/kernel/fadump_registered, leaving the system ready +to capture a vmcore upon crashing. To test this out, you can force-crash +your system by echo'ing a c into /proc/sysrq-trigger: + + # echo c > /proc/sysrq-trigger + +You should see some panic output, followed by the system reset and booting into +fresh copy of kernel. When default initrd loads and runs /init, vmcore should +be copied out to disk (by default, in /var/crash//vmcore), +then the system rebooted back into your normal kernel. + +Once back to your normal kernel, you can use the previously installed crash +kernel in conjunction with the previously installed kernel-debuginfo to +perform postmortem analysis: + + # crash /usr/lib/debug/lib/modules/2.6.17-1.2621.el5/vmlinux + /var/crash/2006-08-23-15:34/vmcore + + crash> bt + +and so on... + +Saving vmcore-dmesg.txt +---------------------- +Kernel log bufferes are one of the most important information available +in vmcore. Now before saving vmcore, kernel log bufferes are extracted +from /proc/vmcore and saved into a file vmcore-dmesg.txt. After +vmcore-dmesg.txt, vmcore is saved. Destination disk and directory for +vmcore-dmesg.txt is same as vmcore. Note that kernel log buffers will +not be available if dump target is raw device. + +Dump Triggering methods: + +This section talks about the various ways, other than a Kernel Panic, in which +fadump can be triggered. The following methods assume that fadump is configured +on your system, with the scripts enabled as described in the section above. + +1) AltSysRq C + +FAdump can be triggered with the combination of the 'Alt','SysRq' and 'C' +keyboard keys. Please refer to the following link for more details: + +https://access.redhat.com/articles/231663 + +In addition, on PowerPC boxes, fadump can also be triggered via Hardware +Management Console(HMC) using 'Ctrl', 'O' and 'C' keyboard keys. + +2) Kernel OOPs + +If we want to generate a dump everytime the Kernel OOPses, we can achieve this +by setting the 'Panic On OOPs' option as follows: + + # echo 1 > /proc/sys/kernel/panic_on_oops + +3) PowerPC specific methods: + +On IBM PowerPC machines, issuing a soft reset invokes the XMON debugger(if +XMON is configured). To configure XMON one needs to compile the kernel with +the CONFIG_XMON and CONFIG_XMON_DEFAULT options, or by compiling with +CONFIG_XMON and booting the kernel with xmon=on option. + +Following are the ways to remotely issue a soft reset on PowerPC boxes, which +would drop you to XMON. Pressing a 'X' (capital alphabet X) followed by an +'Enter' here will trigger the dump. + +3.1) HMC + +Hardware Management Console(HMC) available on Power4 and Power5 machines allow +partitions to be reset remotely. This is specially useful in hang situations +where the system is not accepting any keyboard inputs. + +Once you have HMC configured, the following steps will enable you to trigger +fadump via a soft reset: + +On Power4 + Using GUI + + * In the right pane, right click on the partition you wish to dump. + * Select "Operating System->Reset". + * Select "Soft Reset". + * Select "Yes". + + Using HMC Commandline + + # reset_partition -m -p -t soft + +On Power5 + Using GUI + + * In the right pane, right click on the partition you wish to dump. + * Select "Restart Partition". + * Select "Dump". + * Select "OK". + + Using HMC Commandline + + # chsysstate -m -n -o dumprestart -r lpar + +3.2) Blade Management Console for Blade Center + +To initiate a dump operation, go to Power/Restart option under "Blade Tasks" in +the Blade Management Console. Select the corresponding blade for which you want +to initate the dump and then click "Restart blade with NMI". This issues a +system reset and invokes xmon debugger. + + +Advanced Setups & Default action: + +Kdump and fadump exhibit similar behavior in terms of setup & default action. +For fadump advanced setup related information see section "Advanced Setups" in +"kexec-kdump-howto.txt" document. Refer to "Default action" section in "kexec- +kdump-howto.txt" document for fadump default action related information. + +Compression and filtering + +Refer "Compression and filtering" section in "kexec-kdump-howto.txt" document. +Compression and filtering are same for kdump & fadump. + + +Notes on rootfs mount: +Dracut is designed to mount rootfs by default. If rootfs mounting fails it +will refuse to go on. So fadump leaves rootfs mounting to dracut currently. +We make the assumtion that proper root= cmdline is being passed to dracut +initramfs for the time being. If you need modify "KDUMP_COMMANDLINE=" in +/etc/sysconfig/kdump, you will need to make sure that appropriate root= +options are copied from /proc/cmdline. In general it is best to append +command line options using "KDUMP_COMMANDLINE_APPEND=" instead of replacing +the original command line completely. diff --git a/SOURCES/kdump-dep-generator.sh b/SOURCES/kdump-dep-generator.sh new file mode 100644 index 0000000..b6fab2d --- /dev/null +++ b/SOURCES/kdump-dep-generator.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# More details about systemd generator: +# http://www.freedesktop.org/wiki/Software/systemd/Generators/ + +. /usr/lib/kdump/kdump-lib.sh + +# If invokded with no arguments for testing purpose, output to /tmp to +# avoid overriding the existing. +dest_dir="/tmp" + +if [ -n "$1" ]; then + dest_dir=$1 +fi + +systemd_dir=/usr/lib/systemd/system +kdump_wants=$dest_dir/kdump.service.wants + +if is_ssh_dump_target; then + mkdir -p $kdump_wants + ln -sf $systemd_dir/network-online.target $kdump_wants/ +fi diff --git a/SOURCES/kdump-in-cluster-environment.txt b/SOURCES/kdump-in-cluster-environment.txt new file mode 100644 index 0000000..de1eb5e --- /dev/null +++ b/SOURCES/kdump-in-cluster-environment.txt @@ -0,0 +1,91 @@ +Kdump-in-cluster-environment HOWTO + +Introduction + +Kdump is a kexec based crash dumping mechansim for Linux. This docuement +illustrate how to configure kdump in cluster environment to allow the kdump +crash recovery service complete without being preempted by traditional power +fencing methods. + +Overview + +Kexec/Kdump + +Details about Kexec/Kdump are available in Kexec-Kdump-howto file and will not +be described here. + +fence_kdump + +fence_kdump is an I/O fencing agent to be used with the kdump crash recovery +service. When the fence_kdump agent is invoked, it will listen for a message +from the failed node that acknowledges that the failed node is executing the +kdump crash kernel. Note that fence_kdump is not a replacement for traditional +fencing methods. The fence_kdump agent can only detect that a node has entered +the kdump crash recovery service. This allows the kdump crash recovery service +complete without being preempted by traditional power fencing methods. + +fence_kdump_send + +fence_kdump_send is a utility used to send messages that acknowledge that the +node itself has entered the kdump crash recovery service. The fence_kdump_send +utility is typically run in the kdump kernel after a cluster node has +encountered a kernel panic. Once the cluster node has entered the kdump crash +recovery service, fence_kdump_send will periodically send messages to all +cluster nodes. When the fence_kdump agent receives a valid message from the +failed nodes, fencing is complete. + +How to configure Pacemaker cluster environment: + +If we want to use kdump in Pacemaker cluster environment, fence-agents-kdump +should be installed in every nodes in the cluster. You can achieve this via +the following command: + + # yum install -y fence-agents-kdump + +Next is to add kdump_fence to the cluster. Assuming that the cluster consists +of three nodes, they are node1, node2 and node3, and use Pacemaker to perform +resource management and pcs as cli configuration tool. + +With pcs it is easy to add a stonith resource to the cluster. For example, add +a stonith resource named mykdumpfence with fence type of fence_kdump via the +following commands: + + # pcs stonith create mykdumpfence fence_kdump \ + pcmk_host_check=static-list pcmk_host_list="node1 node2 node3" + # pcs stonith update mykdumpfence pcmk_monitor_action=metadata --force + # pcs stonith update mykdumpfence pcmk_status_action=metadata --force + # pcs stonith update mykdumpfence pcmk_reboot_action=off --force + +Then enable stonith + # pcs property set stonith-enabled=true + +How to configure kdump: + +Actually there are two ways how to configure fence_kdump support: + +1) Pacemaker based clusters + If you have successfully configured fence_kdump in Pacemaker, there is + no need to add some special configuration in kdump. So please refer to + Kexec-Kdump-howto file for more information. + +2) Generic clusters + For other types of clusters there are two configuration options in + kdump.conf which enables fence_kdump support: + + fence_kdump_nodes + Contains list of cluster node(s) separated by space to send + fence_kdump notification to (this option is mandatory to enable + fence_kdump) + + fence_kdump_args + Command line arguments for fence_kdump_send (it can contain + all valid arguments except hosts to send notification to) + + These options will most probably be configured by your cluster software, + so please refer to your cluster documentation how to enable fence_kdump + support. + +Please be aware that these two ways cannot be combined and 2) has precedence +over 1). It means that if fence_kdump is configured using fence_kdump_nodes +and fence_kdump_args options in kdump.conf, Pacemaker configuration is not +used even if it exists. diff --git a/SOURCES/kdump-lib-initramfs.sh b/SOURCES/kdump-lib-initramfs.sh new file mode 100755 index 0000000..2c18c87 --- /dev/null +++ b/SOURCES/kdump-lib-initramfs.sh @@ -0,0 +1,165 @@ +# These variables and functions are useful in 2nd kernel + +. /lib/kdump-lib.sh + +KDUMP_PATH="/var/crash" +CORE_COLLECTOR="" +DEFAULT_CORE_COLLECTOR="makedumpfile -l --message-level 1 -d 31" +DMESG_COLLECTOR="/sbin/vmcore-dmesg" +DEFAULT_ACTION="systemctl reboot -f" +DATEDIR=`date +%Y-%m-%d-%T` +HOST_IP='127.0.0.1' +DUMP_INSTRUCTION="" +SSH_KEY_LOCATION="/root/.ssh/kdump_id_rsa" +KDUMP_SCRIPT_DIR="/kdumpscripts" +DD_BLKSIZE=512 +FINAL_ACTION="systemctl reboot -f" +KDUMP_CONF="/etc/kdump.conf" +KDUMP_PRE="" +KDUMP_POST="" +NEWROOT="/sysroot" + +get_kdump_confs() +{ + local config_opt config_val + + while read config_opt config_val; + do + # remove inline comments after the end of a directive. + config_val=$(strip_comments $config_val) + case "$config_opt" in + path) + KDUMP_PATH="$config_val" + ;; + core_collector) + [ -n "$config_val" ] && CORE_COLLECTOR="$config_val" + ;; + sshkey) + if [ -f "$config_val" ]; then + SSH_KEY_LOCATION=$config_val + fi + ;; + kdump_pre) + KDUMP_PRE="$config_val" + ;; + kdump_post) + KDUMP_POST="$config_val" + ;; + fence_kdump_args) + FENCE_KDUMP_ARGS="$config_val" + ;; + fence_kdump_nodes) + FENCE_KDUMP_NODES="$config_val" + ;; + default) + case $config_val in + shell) + DEFAULT_ACTION="kdump_emergency_shell" + ;; + reboot) + DEFAULT_ACTION="systemctl reboot -f" + ;; + halt) + DEFAULT_ACTION="halt" + ;; + poweroff) + DEFAULT_ACTION="poweroff" + ;; + dump_to_rootfs) + DEFAULT_ACTION="dump_to_rootfs" + ;; + esac + ;; + esac + done < $KDUMP_CONF + + if [ -z "$CORE_COLLECTOR" ]; then + CORE_COLLECTOR="$DEFAULT_CORE_COLLECTOR" + if is_ssh_dump_target || is_raw_dump_target; then + CORE_COLLECTOR="$CORE_COLLECTOR -F" + fi + fi +} + +# dump_fs +dump_fs() +{ + + local _dev=$(findmnt -k -f -n -r -o SOURCE $1) + local _mp=$(findmnt -k -f -n -r -o TARGET $1) + + echo "kdump: dump target is $_dev" + + if [ -z "$_mp" ]; then + echo "kdump: error: Dump target $_dev is not mounted." + return 1 + fi + + # Remove -F in makedumpfile case. We don't want a flat format dump here. + [[ $CORE_COLLECTOR = *makedumpfile* ]] && CORE_COLLECTOR=`echo $CORE_COLLECTOR | sed -e "s/-F//g"` + + echo "kdump: saving to $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" + + mount -o remount,rw $_mp || return 1 + mkdir -p $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR || return 1 + + save_vmcore_dmesg_fs ${DMESG_COLLECTOR} "$_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/" + + echo "kdump: saving vmcore" + $CORE_COLLECTOR /proc/vmcore $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete || return 1 + mv $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore-incomplete $_mp/$KDUMP_PATH/$HOST_IP-$DATEDIR/vmcore + sync + + echo "kdump: saving vmcore complete" + # improper kernel cmdline can cause the failure of echo, we can ignore this kind of failure + return 0 +} + +save_vmcore_dmesg_fs() { + local _dmesg_collector=$1 + local _path=$2 + + echo "kdump: saving vmcore-dmesg.txt" + $_dmesg_collector /proc/vmcore > ${_path}/vmcore-dmesg-incomplete.txt + _exitcode=$? + if [ $_exitcode -eq 0 ]; then + mv ${_path}/vmcore-dmesg-incomplete.txt ${_path}/vmcore-dmesg.txt + + # Make sure file is on disk. There have been instances where later + # saving vmcore failed and system rebooted without sync and there + # was no vmcore-dmesg.txt available. + sync + echo "kdump: saving vmcore-dmesg.txt complete" + else + echo "kdump: saving vmcore-dmesg.txt failed" + fi +} + +dump_to_rootfs() +{ + + echo "Kdump: trying to bring up rootfs device" + systemctl start dracut-initqueue + echo "Kdump: waiting for rootfs mount, will timeout after 90 seconds" + systemctl start sysroot.mount + + dump_fs $NEWROOT +} + +kdump_emergency_shell() +{ + echo "PS1=\"kdump:\\\${PWD}# \"" >/etc/profile + /bin/dracut-emergency + rm -f /etc/profile +} + +do_default_action() +{ + echo "Kdump: Executing default action $DEFAULT_ACTION" + eval $DEFAULT_ACTION +} + +do_final_action() +{ + eval $FINAL_ACTION +} diff --git a/SOURCES/kdump-lib.sh b/SOURCES/kdump-lib.sh new file mode 100755 index 0000000..2c5ced2 --- /dev/null +++ b/SOURCES/kdump-lib.sh @@ -0,0 +1,430 @@ +#!/bin/sh +# +# Kdump common variables and functions +# + +DEFAULT_PATH="/var/crash/" +FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump" +FENCE_KDUMP_SEND="/usr/libexec/fence_kdump_send" + +perror_exit() { + echo $@ >&2 + exit 1 +} + +perror() { + echo $@ >&2 +} + +is_ssh_dump_target() +{ + grep -q "^ssh[[:blank:]].*@" /etc/kdump.conf +} + +is_nfs_dump_target() +{ + grep -q "^nfs" /etc/kdump.conf || \ + [[ $(get_dracut_args_fstype "$(grep "^dracut_args .*\-\-mount" /etc/kdump.conf)") = nfs* ]] +} + +is_raw_dump_target() +{ + grep -q "^raw" /etc/kdump.conf +} + +is_fs_type_nfs() +{ + local _fstype=$1 + [ $_fstype = "nfs" ] || [ $_fstype = "nfs4" ] && return 0 + return 1 +} + +is_fs_dump_target() +{ + egrep -q "^ext[234]|^xfs|^btrfs|^minix" /etc/kdump.conf +} + +is_user_configured_dump_target() +{ + return $(is_mount_in_dracut_args || is_ssh_dump_target || is_nfs_dump_target || \ + is_raw_dump_target || is_fs_dump_target) +} + +strip_comments() +{ + echo $@ | sed -e 's/\(.*\)#.*/\1/' +} + +# Check if fence kdump is configured in Pacemaker cluster +is_pcs_fence_kdump() +{ + # no pcs or fence_kdump_send executables installed? + type -P pcs > /dev/null || return 1 + [ -x $FENCE_KDUMP_SEND ] || return 1 + + # fence kdump not configured? + (pcs cluster cib | grep 'type="fence_kdump"') &> /dev/null || return 1 +} + +# Check if fence_kdump is configured using kdump options +is_generic_fence_kdump() +{ + [ -x $FENCE_KDUMP_SEND ] || return 1 + + grep -q "^fence_kdump_nodes" /etc/kdump.conf +} + +to_dev_name() { + local dev="${1//\"/}" + + case "$dev" in + UUID=*) + dev=`blkid -U "${dev#UUID=}"` + ;; + LABEL=*) + dev=`blkid -L "${dev#LABEL=}"` + ;; + esac + echo $dev +} + +kdump_get_persistent_dev() { + local i _tmp _dev _lookup_dirs + + _dev=$(udevadm info --query=name --name="$1" 2>/dev/null) + [ -z "$_dev" ] && { + perror_exit "Kernel dev name of $1 is not found." + } + + if [[ $2 = "raw" ]];then + _lookup_dirs="/dev/mapper/* /dev/disk/by-id/*" + else + _lookup_dirs="/dev/mapper/* /dev/disk/by-uuid/* /dev/disk/by-id/*" + fi + + for i in $_lookup_dirs; do + _tmp=$(udevadm info --query=name --name="$i" 2>/dev/null) + if [ "$_tmp" = "$_dev" ]; then + echo $i + return + fi + done + + perror "WARNING: Persistent device name of $1 not found. Using $1 as dump target name" + echo $1 +} + +get_user_configured_dump_disk() +{ + local _target + + if is_ssh_dump_target || is_nfs_dump_target; then + return + fi + + _target=$(egrep "^ext[234]|^xfs|^btrfs|^minix|^raw" /etc/kdump.conf 2>/dev/null |awk '{print $2}') + [ -n "$_target" ] && echo $_target + + return +} + +get_root_fs_device() +{ + local _target + _target=$(findmnt -k -f -n -o SOURCE /) + [ -n "$_target" ] && echo $_target + + return +} + +# findmnt uses the option "-v, --nofsroot" to exclusive the [/dir] +# in the SOURCE column for bind-mounts, then if $_mntpoint equals to +# $_mntpoint_nofsroot, the mountpoint is not bind mounted directory. +is_bind_mount() +{ + local _mntpoint=$(findmnt $1 | tail -n 1 | awk '{print $2}') + local _mntpoint_nofsroot=$(findmnt -v $1 | tail -n 1 | awk '{print $2}') + + if [[ $_mntpoint = $_mntpoint_nofsroot ]]; then + return 1 + else + return 0 + fi +} + +# Below is just an example for mount info +# /dev/mapper/atomicos-root[/ostree/deploy/rhel-atomic-host/var], if the +# directory is bind mounted. The former part represents the device path, rest +# part is the bind mounted directory which quotes by bracket "[]". +get_bind_mount_directory() +{ + local _mntpoint=$(findmnt $1 | tail -n 1 | awk '{print $2}') + local _mntpoint_nofsroot=$(findmnt -v $1 | tail -n 1 | awk '{print $2}') + + _mntpoint=${_mntpoint#*$_mntpoint_nofsroot} + + _mntpoint=${_mntpoint#[} + _mntpoint=${_mntpoint%]} + + echo $_mntpoint +} + +get_mntpoint_from_path() +{ + echo $(df $1 | tail -1 | awk '{print $NF}') +} + +get_target_from_path() +{ + echo $(df $1 | tail -1 | awk '{print $1}') +} + +get_fs_type_from_target() +{ + echo $(findmnt -k -f -n -r -o FSTYPE $1) +} + +# input: device path +# output: the general mount point +# find the general mount point, not the bind mounted point in atomic +# As general system, Use the previous code +# +# ERROR and EXIT: +# the device can be umounted the general mount point, if one of the mount point is bind mounted +# For example: +# mount /dev/sda /mnt/ +# mount -o bind /mnt/var /var +# umount /mnt +get_mntpoint_from_target() +{ + if is_atomic; then + for _mnt in $(findmnt -k -n -r -o TARGET $1) + do + if ! is_bind_mount $_mnt; then + echo $_mnt + return + fi + done + + echo "Mount $1 firstly, without the bind mode" >&2 + exit 1 + else + echo $(findmnt -k -f -n -r -o TARGET $1) + fi +} + +# get_option_value +# retrieves value of option defined in kdump.conf +get_option_value() { + echo $(strip_comments `grep ^$1 /etc/kdump.conf | tail -1 | cut -d\ -f2-`) +} + +#This function compose a absolute path with the mount +#point and the relative $SAVE_PATH. +#target is passed in as argument, could be UUID, LABEL, +#block device or even nfs server export of the form of +#"my.server.com:/tmp/export"? +#And possibly this could be used for both default case +#as well as when dump taret is specified. When dump +#target is not specified, then $target would be null. +make_absolute_save_path() +{ + local _target=$1 + local _mnt + + [ -n $_target ] && _mnt=$(get_mntpoint_from_target $1) + _mnt="${_mnt}/$SAVE_PATH" + + # strip the duplicated "/" + echo "$_mnt" | tr -s / +} + +check_save_path_fs() +{ + local _path=$1 + + if [ ! -d $_path ]; then + perror_exit "Dump path $_path does not exist." + fi +} + +is_atomic() +{ + grep -q "ostree" /proc/cmdline +} + +is_ipv6_address() +{ + echo $1 | grep -q ":" +} + +# get ip address or hostname from nfs/ssh config value +get_remote_host() +{ + local _config_val=$1 + + # ipv6 address in kdump.conf is around with "[]", + # factor out the ipv6 address + _config_val=${_config_val#*@} + _config_val=${_config_val%:/*} + _config_val=${_config_val#[} + _config_val=${_config_val%]} + echo $_config_val +} + +is_hostname() +{ + local _hostname=`echo $1 | grep ":"` + + if [ -n "$_hostname" ]; then + return 1 + fi + echo $1 | grep -q "[a-zA-Z]" +} + +# Copied from "/etc/sysconfig/network-scripts/network-functions" +get_hwaddr() +{ + if [ -f "/sys/class/net/${1}/address" ]; then + awk '{ print toupper($0) }' < /sys/class/net/${1}/address + elif [ -d "/sys/class/net/${1}" ]; then + LC_ALL= LANG= ip -o link show ${1} 2>/dev/null | \ + awk '{ print toupper(gensub(/.*link\/[^ ]* ([[:alnum:]:]*).*/, + "\\1", 1)); }' + fi +} + +get_ifcfg_by_device() +{ + grep -E -i -l "^[[:space:]]*DEVICE=\"*${1}\"*[[:space:]]*$" \ + /etc/sysconfig/network-scripts/ifcfg-* 2>/dev/null | head -1 +} + +get_ifcfg_by_hwaddr() +{ + grep -E -i -l "^[[:space:]]*HWADDR=\"*${1}\"*[[:space:]]*$" \ + /etc/sysconfig/network-scripts/ifcfg-* 2>/dev/null | head -1 +} + +get_ifcfg_by_uuid() +{ + grep -E -i -l "^[[:space:]]*UUID=\"*${1}\"*[[:space:]]*$" \ + /etc/sysconfig/network-scripts/ifcfg-* 2>/dev/null | head -1 +} + +get_ifcfg_by_name() +{ + grep -E -i -l "^[[:space:]]*NAME=\"*${1}\"*[[:space:]]*$" \ + /etc/sysconfig/network-scripts/ifcfg-* 2>/dev/null | head -1 +} + +is_nm_running() +{ + [ "$(LANG=C nmcli -t --fields running general status 2>/dev/null)" = "running" ] +} + +is_nm_handling() +{ + LANG=C nmcli -t --fields device,state dev status 2>/dev/null \ + | grep -q "^\(${1}:connected\)\|\(${1}:connecting.*\)$" +} + +# $1: netdev name +get_ifcfg_nmcli() +{ + local nm_uuid nm_name + local ifcfg_file + + # Get the active nmcli config name of $1 + if is_nm_running && is_nm_handling "${1}" ; then + # The configuration "uuid" and "name" generated by nm is wrote to + # the ifcfg file as "UUID=" and "NAME=". + nm_uuid=$(LANG=C nmcli -t --fields uuid,device c show --active 2>/dev/null \ + | grep "${1}" | head -1 | cut -d':' -f1) + nm_name=$(LANG=C nmcli -t --fields name,device c show --active 2>/dev/null \ + | grep "${1}" | head -1 | cut -d':' -f1) + ifcfg_file=$(get_ifcfg_by_uuid "${nm_uuid}") + [ -z "${ifcfg_file}" ] && ifcfg_file=$(get_ifcfg_by_name "${nm_name}") + fi + + echo -n "${ifcfg_file}" +} + +# $1: netdev name +get_ifcfg_legacy() +{ + local ifcfg_file + + ifcfg_file="/etc/sysconfig/network-scripts/ifcfg-${1}" + [ -f "${ifcfg_file}" ] && echo -n "${ifcfg_file}" && return + + ifcfg_file=$(get_ifcfg_by_name "${1}") + [ -f "${ifcfg_file}" ] && echo -n "${ifcfg_file}" && return + + local hwaddr=$(get_hwaddr "${1}") + if [ -n "$hwaddr" ]; then + ifcfg_file=$(get_ifcfg_by_hwaddr "${hwaddr}") + [ -f "${ifcfg_file}" ] && echo -n "${ifcfg_file}" && return + fi + + ifcfg_file=$(get_ifcfg_by_device "${1}") + + echo -n "${ifcfg_file}" +} + +# $1: netdev name +# Return the ifcfg file whole name(including the path) of $1 if any. +get_ifcfg_filename() { + local ifcfg_file + + ifcfg_file=$(get_ifcfg_nmcli "${1}") + if [ -z "${ifcfg_file}" ]; then + ifcfg_file=$(get_ifcfg_legacy "${1}") + fi + + echo -n "${ifcfg_file}" +} + +# returns 0 when omission of watchdog module is desired in dracut_args +# returns 1 otherwise +is_wdt_mod_omitted() { + local dracut_args + local ret=1 + + dracut_args=$(grep "^dracut_args" /etc/kdump.conf) + [[ -z $dracut_args ]] && return $ret + + eval set -- $dracut_args + while :; do + [[ -z $1 ]] && break + case $1 in + -o|--omit) + echo $2 | grep -qw "watchdog" + [[ $? == 0 ]] && ret=0 + break + esac + shift + done + + return $ret +} + +# If "dracut_args" contains "--mount" information, use it +# directly without any check(users are expected to ensure +# its correctness). +is_mount_in_dracut_args() +{ + grep -q "^dracut_args .*\-\-mount" /etc/kdump.conf +} + +# If $1 contains dracut_args "--mount", return +get_dracut_args_fstype() +{ + echo $1 | grep "\-\-mount" | sed "s/.*--mount .\(.*\)/\1/" | cut -d' ' -f3 +} + +# If $1 contains dracut_args "--mount", return +get_dracut_args_target() +{ + echo $1 | grep "\-\-mount" | sed "s/.*--mount .\(.*\)/\1/" | cut -d' ' -f1 +} diff --git a/SOURCES/kdump.conf b/SOURCES/kdump.conf new file mode 100644 index 0000000..57af7b6 --- /dev/null +++ b/SOURCES/kdump.conf @@ -0,0 +1,163 @@ +# This file contains a series of commands to perform (in order) in the kdump +# kernel after a kernel crash in the crash kernel(1st kernel) has happened. +# +# Directives in this file are only applicable to the kdump initramfs, and have +# no effect once the root filesystem is mounted and the normal init scripts are +# processed. +# +# Currently, only one dump target and path can be specified. If the dumping to +# the configured target fails, the default action which can be configured via +# the "default" directive will be performed. +# +# Supported options: +# +# raw +# - Will dd /proc/vmcore into . +# Use persistent device names for partition devices, +# such as /dev/vg/. +# +# nfs +# - Will mount nfs to , and copy /proc/vmcore to +# //%HOST-%DATE/, supports DNS. +# +# ssh +# - Will scp /proc/vmcore to :/%HOST-%DATE/, +# supports DNS. +# NOTE: make sure the user has write permissions on the server. +# +# sshkey +# - Will use the sshkey to do ssh dump. +# Specify the path of the ssh key to use when dumping +# via ssh. The default value is /root/.ssh/kdump_id_rsa. +# +# +# - Will mount -t , and copy +# /proc/vmcore to //%DATE/. +# NOTE: can be a device node, label or uuid. +# It's recommended to use persistent device names +# such as /dev/vg/. +# Otherwise it's suggested to use label or uuid. +# +# path +# - "path" represents the file system path in which vmcore +# will be saved. If a dump target is specified in +# kdump.conf, then "path" is relative to the specified +# dump target. +# +# Interpretation of "path" changes a bit if the user didn't +# specify any dump target explicitly in kdump.conf. In this +# case, "path" represents the absolute path from root. The +# dump target and adjusted path are arrived at automatically +# depending on what's mounted in the current system. +# +# Ignored for raw device dumps. If unset, will use the default +# "/var/crash". +# +# core_collector +# - This allows you to specify the command to copy +# the vmcore. The default is makedumpfile, which on +# some architectures can drastically reduce vmcore size. +# See /sbin/makedumpfile --help for a list of options. +# Note that the -i and -g options are not needed here, +# as the initrd will automatically be populated with a +# config file appropriate for the running kernel. +# The default core_collector for raw/ssh dump is: +# "makedumpfile -F -l --message-level 1 -d 31". +# The default core_collector for other targets is: +# "makedumpfile -l --message-level 1 -d 31". +# +# "makedumpfile -F" will create a flattened vmcore. +# You need to use "makedumpfile -R" to rearrange the dump data to +# a normal dumpfile readable with analysis tools. For example: +# "makedumpfile -R vmcore < vmcore.flat". +# +# For core_collector format details, you can refer to +# kexec-kdump-howto.txt or kdump.conf manpage. +# +# kdump_post +# - This directive allows you to run a executable binary +# or script after the vmcore dump process terminates. +# The exit status of the current dump process is fed to +# the executable binary or script as its first argument. +# +# kdump_pre +# - Works like the "kdump_post" directive, but instead of running +# after the dump process, runs immediately before it. +# Exit status of this binary is interpreted as follows: +# 0 - continue with dump process as usual +# non 0 - reboot the system +# +# extra_bins +# - This directive allows you to specify additional binaries or +# shell scripts to be included in the kdump initrd. +# Generally they are useful in conjunction with a kdump_post +# or kdump_pre binary or script which depends on these extra_bins. +# +# extra_modules +# - This directive allows you to specify extra kernel modules +# that you want to be loaded in the kdump initrd. +# Multiple modules can be listed, separated by spaces, and any +# dependent modules will automatically be included. +# +# default +# - Action to perform in case dumping fails. +# reboot: Reboot the system. +# halt: Halt the system. +# poweroff: Power down the system. +# shell: Drop to a bash shell. +# Exiting the shell reboots the system. +# dump_to_rootfs: Dump vmcore to rootfs from initramfs context and +# reboot. Useful when non-root dump target is specified. +# The default option is "reboot". +# +# force_rebuild <0 | 1> +# - By default, kdump initrd will only be rebuilt when necessary. +# Specify 1 to force rebuilding kdump initrd every time when kdump +# service starts. +# +# force_no_rebuild <0 | 1> +# - By default, kdump initrd will be rebuilt when necessary. +# Specify 1 to bypass rebuilding of kdump initrd. +# +# force_no_rebuild and force_rebuild options are mutually +# exclusive and they should not be set to 1 simultaneously. +# +# override_resettable <0 | 1> +# - Usually an unresettable block device can't be a dump target. +# Specifying 1 when you want to dump even though the block +# target is unresettable +# By default, it is 0, which will not try dumping destined to fail. +# +# dracut_args +# - Pass extra dracut options when rebuilding kdump initrd. +# +# fence_kdump_args +# - Command line arguments for fence_kdump_send (it can contain +# all valid arguments except hosts to send notification to). +# +# fence_kdump_nodes +# - List of cluster node(s) except localhost, separated by spaces, +# to send fence_kdump notifications to. +# (this option is mandatory to enable fence_kdump). +# + +#raw /dev/vg/lv_kdump +#ext4 /dev/vg/lv_kdump +#ext4 LABEL=/boot +#ext4 UUID=03138356-5e61-4ab3-b58e-27507ac41937 +#nfs my.server.com:/export/tmp +#ssh user@my.server.com +#sshkey /root/.ssh/kdump_id_rsa +path /var/crash +core_collector makedumpfile -l --message-level 1 -d 31 +#core_collector scp +#kdump_post /var/crash/scripts/kdump-post.sh +#kdump_pre /var/crash/scripts/kdump-pre.sh +#extra_bins /usr/bin/lftp +#extra_modules gfs2 +#default shell +#force_rebuild 1 +#force_no_rebuild 1 +#dracut_args --omit-drivers "cfg80211 snd" --add-drivers "ext2 ext3" +#fence_kdump_args -p 7410 -f auto -c 0 -i 10 +#fence_kdump_nodes node1 node2 diff --git a/SOURCES/kdump.conf.5 b/SOURCES/kdump.conf.5 new file mode 100644 index 0000000..11b1fad --- /dev/null +++ b/SOURCES/kdump.conf.5 @@ -0,0 +1,344 @@ +.TH KDUMP.CONF 5 "07/23/2008" "kexec-tools" + +.SH NAME +kdump.conf \- configuration file for kdump kernel. + +.SH DESCRIPTION + +kdump.conf is a configuration file for the kdump kernel crash +collection service. + +kdump.conf provides post-kexec instructions to the kdump kernel. It is +stored in the initrd file managed by the kdump service. If you change +this file and do not want to reboot in order for the changes to take +effect, restart the kdump service to rebuild the initrd. + +For most configurations, you can simply review the examples provided +in the stock /etc/kdump.conf. + +.B NOTE: +For filesystem dumps the dump target must be mounted before building +kdump initramfs. + +kdump.conf only affects the behavior of the initramfs. Please read the +kdump operational flow section of kexec-kdump-howto.txt in the docs to better +understand how this configuration file affects the behavior of kdump. + +.SH OPTIONS + +.B raw +.RS +Will dd /proc/vmcore into . Use persistent device names for +partition devices, such as /dev/vg/. +.RE + +.B nfs +.RS +Will mount nfs to , and copy /proc/vmcore to //%HOST-%DATE/, +supports DNS. Note that a fqdn should be used as the server name in the +mount point. +.RE + +.B ssh +.RS +Will scp /proc/vmcore to :/%HOST-%DATE/, +supports DNS. NOTE: make sure user has necessary write permissions on +server and that a fqdn is used as the server name. +.RE + +.B sshkey +.RS +Specify the path of the ssh key to use when dumping via ssh. +The default value is /root/.ssh/kdump_id_rsa. +.RE + +.B +.RS +Will mount -t , and copy /proc/vmcore to +//%DATE/. NOTE: can be a device node, label +or uuid. It's recommended to use persistent device names such as +/dev/vg/. Otherwise it's suggested to use label or uuid. +.RE + +.B path +.RS +"path" represents the file system path in which vmcore will be saved. +If a dump target is specified in kdump.conf, then "path" is relative to the +specified dump target. +.PP +Interpretation of "path" changes a bit if the user didn't specify any dump +target explicitly in kdump.conf. In this case, "path" represents the +absolute path from root. The dump target and adjusted path are arrived +at automatically depending on what's mounted in the current system. +.PP +Ignored for raw device dumps. If unset, will use the default "/var/crash". +.RE + +.B core_collector +.RS +This allows you to specify the command to copy the vmcore. +The default is makedumpfile, which on some architectures can drastically reduce +core file size. See /sbin/makedumpfile --help for a list of options. +Note that the -i and -g options are not needed here, as the initrd +will automatically be populated with a config file appropriate +for the running kernel. +.PP +Note 1: About default core collector: +The default core_collector for raw/ssh dump is: +"makedumpfile -F -l --message-level 1 -d 31". +The default core_collector for other targets is: +"makedumpfile -l --message-level 1 -d 31". +Even if core_collector option is commented out in kdump.conf, makedumpfile +is the default core collector and kdump uses it internally. +If one does not want makedumpfile as default core_collector, then they +need to specify one using core_collector option to change the behavior. +.PP +Note 2: If "makedumpfile -F" is used then you will get a flattened format +vmcore.flat, you will need to use "makedumpfile -R" to rearrange the +dump data from standard input to a normal dumpfile (readable with analysis +tools). +ie. "makedumpfile -R vmcore < vmcore.flat" + +.RE + +.B kdump_post +.RS +This directive allows you to run a specified executable +just after the vmcore dump process terminates. The exit +status of the current dump process is fed to the kdump_post +executable as its first argument($1). Executable can modify +it to indicate the new exit status of succeeding dump process, +.PP +Note that scripts written for use with this directive must use +the /bin/bash interpreter. +.RE + +.B kdump_pre +.RS +Works just like the "kdump_post" directive, but instead +of running after the dump process, runs immediately +before. Exit status of this binary is interpreted +as follows: +.PP +0 - continue with dump process as usual +.PP +non 0 - reboot the system +.PP +Note that scripts written for this directive must use +the /bin/bash interpreter. +.RE + +.B extra_bins +.RS +This directive allows you to specify additional +binaries or shell scripts you'd like to include in +your kdump initrd. Generally only useful in +conjunction with a kdump_post binary or script that +relies on other binaries or scripts. +.RE + +.B extra_modules +.RS +This directive allows you to specify extra kernel +modules that you want to be loaded in the kdump +initrd, typically used to set up access to +non-boot-path dump targets that might otherwise +not be accessible in the kdump environment. Multiple +modules can be listed, separated by spaces, and any +dependent modules will automatically be included. +.RE + +.B default +.RS +Action to perform in case dumping to the intended target fails. The default is "reboot". +reboot: Reboot the system (this is what most people will want, as it returns the system +to a normal state). halt: Halt the system and lose the vmcore. poweroff: The system +will be powered down. shell: Drop to a shell session inside the initramfs, from which +you can manually perform additional recovery actions. Exiting this shell reboots the +system. Note: kdump uses bash as the default shell. dump_to_rootfs: If non-root dump +target is specified, the default action can be set as dump_to_rootfs. That means when +dumping to target fails, dump vmcore to rootfs from initramfs context and reboot. +.RE + +.B force_rebuild <0 | 1> +.RS +By default, kdump initrd will only be rebuilt when necessary. +Specify 1 to force rebuilding kdump initrd every time when kdump service starts. +.RE + +.B force_no_rebuild <0 | 1> +.RS +By default, kdump initrd will be rebuilt when necessary. +Specify 1 to bypass rebuilding of kdump initrd. + +.PP +force_no_rebuild and force_rebuild options are mutually exclusive and +they should not be set to 1 simultaneously. +.RE + +.B override_resettable <0 | 1> +.RS +Usually an unresettable block device can't be a dump target. Specifying 1 means +that even though the block target is unresettable, the user wants to try dumping anyway. +By default, it's set to 0, which will not try something destined to fail. +.RE + + +.B dracut_args +.RS +Kdump uses dracut to generate initramfs for second kernel. This option +allows a user to pass arguments to dracut directly. +.RE + + +.B fence_kdump_args +.RS +Command line arguments for fence_kdump_send (it can contain all valid +arguments except hosts to send notification to). +.RE + + +.B fence_kdump_nodes +.RS +List of cluster node(s) except localhost, separated by spaces, to send fence_kdump notification +to (this option is mandatory to enable fence_kdump). +.RE + + +.SH DEPRECATED OPTIONS + +.B net | +.RS +net option is replaced by nfs and ssh options. Use nfs or ssh options +directly. +.RE + +.B options