diff --git a/SOURCES/dracut-module-setup.sh b/SOURCES/dracut-module-setup.sh index f106259..3d63dab 100755 --- a/SOURCES/dracut-module-setup.sh +++ b/SOURCES/dracut-module-setup.sh @@ -24,7 +24,7 @@ depends() { _dep="$_dep drm" fi - if is_generic_fence_kdump -o is_pcs_fence_kdump; then + if is_generic_fence_kdump || is_pcs_fence_kdump; then _dep="$_dep network" fi @@ -209,7 +209,7 @@ kdump_setup_bond() { source_ifcfg_file $_netdev - bondoptions="$(echo :$BONDING_OPTS | sed 's/\s\+/,/')" + bondoptions=":$(echo $BONDING_OPTS | xargs echo | tr " " ",")" echo "$bondoptions" >> ${initdir}/etc/cmdline.d/42bond.conf } @@ -624,6 +624,40 @@ kdump_check_iscsi_targets () { } } +# hostname -a is deprecated, do it by ourself +get_alias() { + local ips + local entries + local alias_set + + ips=$(hostname -I) + for ip in $ips + do + entries=$(grep $ip /etc/hosts | awk '{ $1=$2=""; print $0 }') + if [ $? -eq 0 ]; then + alias_set="$alias_set $entries" + fi + done + + echo $alias_set +} + +is_localhost() { + local hostnames=$(hostname -A) + local shortnames=$(hostname -A -s) + local aliasname=$(get_alias) + local nodename=$1 + + hostnames="$hostnames $shortnames $aliasname" + + for name in ${hostnames}; do + if [ "$name" == "$nodename" ]; then + return 0 + fi + done + return 1 +} + # retrieves fence_kdump nodes from Pacemaker cluster configuration get_pcs_fence_kdump_nodes() { local nodes @@ -638,7 +672,7 @@ get_pcs_fence_kdump_nodes() { eval $node nodename=$uname # Skip its own node name - if [ "$nodename" = `hostname` -o "$nodename" = `hostname -s` ]; then + if is_localhost $nodename; then continue fi nodes="$nodes $nodename" @@ -655,6 +689,21 @@ get_pcs_fence_kdump_args() { fi } +get_generic_fence_kdump_nodes() { + local filtered + local nodes + + nodes=$(get_option_value "fence_kdump_nodes") + for node in ${nodes}; do + # Skip its own node name + if is_localhost $node; then + continue + fi + filtered="$filtered $node" + done + echo $filtered +} + # setup fence_kdump in cluster # setup proper network and install needed files kdump_configure_fence_kdump () { @@ -663,7 +712,7 @@ kdump_configure_fence_kdump () { local args if is_generic_fence_kdump; then - nodes=$(get_option_value "fence_kdump_nodes") + nodes=$(get_generic_fence_kdump_nodes) elif is_pcs_fence_kdump; then nodes=$(get_pcs_fence_kdump_nodes) diff --git a/SOURCES/kdump.conf.5 b/SOURCES/kdump.conf.5 index 990076e..61dd0c0 100644 --- a/SOURCES/kdump.conf.5 +++ b/SOURCES/kdump.conf.5 @@ -259,7 +259,7 @@ retaining blacklist option creates more confusing behavior. It has been deprecated. .PP Instead, use rd.driver.blacklist option on second kernel to blacklist -a certain module. One can edit /etc/sysconfig/kdump.conf and edit +a certain module. One can edit /etc/sysconfig/kdump and edit KDUMP_COMMANDLINE_APPEND to pass kernel command line options. Refer to dracut.cmdline man page for more details on module blacklist option. .RE diff --git a/SOURCES/kdump.sysconfig.x86_64 b/SOURCES/kdump.sysconfig.x86_64 index 61ad9c9..a9d6287 100644 --- a/SOURCES/kdump.sysconfig.x86_64 +++ b/SOURCES/kdump.sysconfig.x86_64 @@ -21,7 +21,7 @@ KDUMP_COMMANDLINE_REMOVE="hugepages hugepagesz slub_debug kaslr" # This variable lets us append arguments to the current kdump commandline # after processed by KDUMP_COMMANDLINE_REMOVE -KDUMP_COMMANDLINE_APPEND="irqpoll nr_cpus=1 reset_devices cgroup_disable=memory mce=off numa=off udev.children-max=2 panic=10 rootflags=nofail acpi_no_memhotplug transparent_hugepage=never nokaslr novmcoredd" +KDUMP_COMMANDLINE_APPEND="irqpoll nr_cpus=1 reset_devices cgroup_disable=memory mce=off numa=off udev.children-max=2 panic=10 rootflags=nofail acpi_no_memhotplug transparent_hugepage=never nokaslr novmcoredd hest_disable" # Any additional kexec arguments required. In most situations, this should # be left empty diff --git a/SOURCES/kdumpctl b/SOURCES/kdumpctl index 89fd73b..d4b05de 100755 --- a/SOURCES/kdumpctl +++ b/SOURCES/kdumpctl @@ -188,7 +188,7 @@ rebuild_fadump_initrd() # this file tells the initrd is fadump enabled touch /tmp/fadump.initramfs target_initrd_tmp="$TARGET_INITRD.tmp" - $MKDUMPRD $target_initrd_tmp --rebuild $TARGET_INITRD --kver $kdump_kver \ + $MKDUMPRD $target_initrd_tmp --rebuild $DEFAULT_INITRD_BAK --kver $kdump_kver \ -i /tmp/fadump.initramfs /etc/fadump.initramfs if [ $? != 0 ]; then echo "mkdumprd: failed to rebuild initrd with fadump support" >&2 @@ -217,6 +217,11 @@ rebuild_kdump_initrd() rebuild_initrd() { + if [[ ! -w "$KDUMP_BOOTDIR" ]];then + echo "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD" + return 1 + fi + if [ $DEFAULT_DUMP_MODE == "fadump" ]; then rebuild_fadump_initrd else @@ -398,12 +403,17 @@ setup_initrd() DEFAULT_INITRD_BAK="${KDUMP_BOOTDIR}/.initramfs-`uname -r`.img.default" if [ $DEFAULT_DUMP_MODE == "fadump" ]; then TARGET_INITRD="$DEFAULT_INITRD" - if [ ! -s "$TARGET_INITRD" ]; then - echo "Error: No initrd found to rebuild!" - return 1 - fi + + # backup initrd for reference before replacing it + # with fadump aware initrd + backup_default_initrd else TARGET_INITRD="${KDUMP_BOOTDIR}/initramfs-${kdump_kver}kdump.img" + + # check if a backup of default initrd exists. If yes, + # it signifies a switch from fadump mode. So, restore + # the backed up default initrd. + restore_default_initrd fi } @@ -702,8 +712,6 @@ check_rebuild() system_modified="1" fi - handle_mode_switch - if [ $image_time -eq 0 ]; then echo -n "No kdump initial ramdisk found."; echo elif [ "$capture_capable_initrd" == "0" ]; then @@ -716,11 +724,6 @@ check_rebuild() return 0 fi - if [[ ! -w "$KDUMP_BOOTDIR" ]];then - echo "$KDUMP_BOOTDIR does not have write permission. Can not rebuild $TARGET_INITRD" - return 1 - fi - echo "Rebuilding $TARGET_INITRD" rebuild_initrd return $? @@ -820,13 +823,60 @@ check_ssh_config() return 0 } +# ipv6 host address may takes a long time to be ready. +# Instead of checking against ipv6 address, we just check the network reachable +# by the return val of 'ssh' +check_and_wait_network_ready() +{ + local start_time=$(date +%s) + local warn_once=1 + local cur + local diff + local retval + local errmsg + + while true; do + errmsg=$(ssh -i $SSH_KEY_LOCATION -o BatchMode=yes $DUMP_TARGET mkdir -p $SAVE_PATH 2>&1) + retval=$? + + # ssh exits with the exit status of the remote command or with 255 if an error occurred + if [ $retval -eq 0 ]; then + return 0 + elif [ $retval -ne 255 ]; then + echo "Could not create $DUMP_TARGET:$SAVE_PATH, you should check the privilege on server side" >&2 + return 1 + fi + + # if server removes the authorized_keys or, no /root/.ssh/kdump_id_rsa + echo $errmsg | grep -q "Permission denied\|No such file or directory\|Host key verification failed" + if [ $? -eq 0 ]; then + echo "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" >&2 + return 1 + fi + + if [ $warn_once -eq 1 ]; then + echo "Network dump target is not usable, waiting for it to be ready" + warn_once=0 + fi + echo -n . + + cur=$(date +%s) + let "diff = $cur - $start_time" + # 60s time out + if [ $diff -gt 180 ]; then + break; + fi + sleep 1 + done + + echo "Could not create $DUMP_TARGET:$SAVE_PATH, ipaddr is not ready yet. You should check network connection" >&2 + return 1 +} + check_ssh_target() { - local _ret - ssh -q -i $SSH_KEY_LOCATION -o BatchMode=yes $DUMP_TARGET mkdir -p $SAVE_PATH - _ret=$? - if [ $_ret -ne 0 ]; then - echo "Could not create $DUMP_TARGET:$SAVE_PATH, you probably need to run \"kdumpctl propagate\"" >&2 + check_and_wait_network_ready + if [ $? -ne 0 ]; then return 1 fi return 0 @@ -876,20 +926,6 @@ show_reserved_mem() echo "Reserved "$mem_mb"MB memory for crash kernel" } -handle_mode_switch() -{ - if [ "$DEFAULT_DUMP_MODE" == "fadump" ]; then - # backup initrd for reference before replacing it - # with fadump aware initrd - backup_default_initrd - else - # check if a backup of default initrd exists. If yes, - # it signifies a switch from fadump mode. So, restore - # the backed up default initrd. - restore_default_initrd - fi -} - check_current_fadump_status() { # Check if firmware-assisted dump has been registered. @@ -1320,6 +1356,28 @@ stop() return 0 } +rebuild() { + check_config + if [ $? -ne 0 ]; then + return 1 + fi + + if check_ssh_config; then + if ! check_ssh_target; then + return 1 + fi + fi + + setup_initrd + if [ $? -ne 0 ]; then + return 1 + fi + + echo "Rebuilding $TARGET_INITRD" + rebuild_initrd + return $? +} + if [ ! -f "$KDUMP_CONFIG_FILE" ]; then echo "Error: No kdump config file found!" >&2 exit 1 @@ -1364,6 +1422,9 @@ main () stop start ;; + rebuild) + rebuild + ;; condrestart) ;; propagate) @@ -1373,7 +1434,7 @@ main () show_reserved_mem ;; *) - echo $"Usage: $0 {start|stop|status|restart|reload|propagate|showmem}" + echo $"Usage: $0 {start|stop|status|restart|reload|rebuild|propagate|showmem}" exit 1 esac } diff --git a/SOURCES/kdumpctl.8 b/SOURCES/kdumpctl.8 index 023562b..ae97af7 100644 --- a/SOURCES/kdumpctl.8 +++ b/SOURCES/kdumpctl.8 @@ -32,6 +32,12 @@ It returns non-zero value if kdump is not operational. Is equal to .I start; stop .TP +.I reload +reload crash kernel image and initramfs without triggering a rebuild. +.TP +.I rebuild +rebuild the crash kernel initramfs. +.TP .I propagate Helps to setup key authentication for ssh storage since it's impossible to use password authentication during kdump. diff --git a/SOURCES/kexec-kdump-howto.txt b/SOURCES/kexec-kdump-howto.txt index 2c51c6d..f4d943a 100644 --- a/SOURCES/kexec-kdump-howto.txt +++ b/SOURCES/kexec-kdump-howto.txt @@ -157,6 +157,22 @@ perform postmortem analysis: and so on... +Notes: + +When kdump starts, the kdump kernel is loaded together with the kdump +initramfs. To save memory usage and disk space, the kdump initramfs is +generated strictly against the system it will run on, and contains the +minimum set of kernel modules and utilities to boot the machine to a stage +where the dump target could be mounted. + +With kdump service enabled, kdumpctl will try to detect possible system +change and rebuild the kdump initramfs if needed. But it can not guarantee +to cover every possible case. So after a hardware change, disk migration, +storage setup update or any similar system level changes, it's highly +recommended to rebuild the initramfs manually with following command: + + # kdumpctl rebuild + Saving vmcore-dmesg.txt ---------------------- Kernel log bufferes are one of the most important information available @@ -675,6 +691,20 @@ not been written in watchdog-core framework then this option will not have any effect and module will not be added. Please note that only systemd watchdog daemon is supported as watchdog kick application. +Notes for disk images: + +Kdump initramfs is a critical component for capturing the crash dump. +But it's strictly generated for the machine it will run on, and have +no generality. If you install a new machine with a previous disk image +(eg. VMs created with disk image or snapshot), kdump could be broken +easily due to hardware changes or disk ID changes. So it's strongly +recommended to not include the kdump initramfs in the disk image in the +first place, this helps to save space, and kdumpctl will build the +initramfs automatically if it's missing. If you have already installed +a machine with a disk image which have kdump initramfs embedded, you +should rebuild the initramfs using "kdumpctl rebuild" command manually, +or else kdump may not work as expeceted. + Notes on device dump: Device dump allows drivers to append dump data to vmcore, so you can diff --git a/SOURCES/kexec-tools-2.0.15-Limit-the-size-of-vmcore-dmesg.txt-to-2G.patch b/SOURCES/kexec-tools-2.0.15-Limit-the-size-of-vmcore-dmesg.txt-to-2G.patch new file mode 100644 index 0000000..a67c927 --- /dev/null +++ b/SOURCES/kexec-tools-2.0.15-Limit-the-size-of-vmcore-dmesg.txt-to-2G.patch @@ -0,0 +1,59 @@ +From 4b23140193c5063438f6c5f1c0ba43f981609bb5 Mon Sep 17 00:00:00 2001 +From: Lianbo Jiang +Date: Mon, 9 Sep 2019 09:48:10 +0800 +Subject: [PATCH] Limit the size of vmcore-dmesg.txt to 2G + +With some corrupted vmcore files, the vmcore-dmesg.txt file may grow +forever till the kdump disk becomes full, and also probably causes +the disk error messages as follow: +... +sd 0:0:0:0: [sda] tag#6 FAILED Result: hostbyte=DID_BAD_TARGET driverbyte=DRIVER_OK +sd 0:0:0:0: [sda] tag#6 CDB: Read(10) 28 00 08 06 4c 98 00 00 08 00 +blk_update_request: I/O error, dev sda, sector 134630552 +sd 0:0:0:0: [sda] tag#7 FAILED Result: hostbyte=DID_BAD_TARGET driverbyte=DRIVER_OK +sd 0:0:0:0: [sda] tag#7 CDB: Read(10) 28 00 08 06 4c 98 00 00 08 00 +blk_update_request: I/O error, dev sda, sector 134630552 +... + +If vmcore-dmesg.txt occupies the whole disk, the vmcore can not be +saved, this is also a problem. + +Lets limit the size of vmcore-dmesg.txt to avoid such problems. + +Signed-off-by: Lianbo Jiang +Signed-off-by: Simon Horman +--- + vmcore-dmesg/vmcore-dmesg.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/vmcore-dmesg/vmcore-dmesg.c b/vmcore-dmesg/vmcore-dmesg.c +index a8f56dfbe51f..41ab73147155 100644 +--- a/vmcore-dmesg/vmcore-dmesg.c ++++ b/vmcore-dmesg/vmcore-dmesg.c +@@ -22,6 +22,9 @@ + /* The 32bit and 64bit note headers make it clear we don't care */ + typedef Elf32_Nhdr Elf_Nhdr; + ++/* stole this macro from kernel printk.c */ ++#define LOG_BUF_LEN_MAX (uint32_t)(1 << 31) ++ + static const char *fname; + static Elf64_Ehdr ehdr; + static Elf64_Phdr *phdr; +@@ -476,6 +479,13 @@ static int32_t read_file_s32(int fd, uint64_t addr) + static void write_to_stdout(char *buf, unsigned int nr) + { + ssize_t ret; ++ static uint32_t n_bytes = 0; ++ ++ n_bytes += nr; ++ if (n_bytes > LOG_BUF_LEN_MAX) { ++ fprintf(stderr, "The vmcore-dmesg.txt over 2G in size is not supported.\n"); ++ exit(53); ++ } + + ret = write(STDOUT_FILENO, buf, nr); + if (ret != nr) { +-- +2.17.1 + diff --git a/SOURCES/kexec-tools-2.0.15-makedumpfile-x86_64-Fix-incorrect-exclusion-by-e-option-wit.patch b/SOURCES/kexec-tools-2.0.15-makedumpfile-x86_64-Fix-incorrect-exclusion-by-e-option-wit.patch new file mode 100644 index 0000000..652dd14 --- /dev/null +++ b/SOURCES/kexec-tools-2.0.15-makedumpfile-x86_64-Fix-incorrect-exclusion-by-e-option-wit.patch @@ -0,0 +1,40 @@ +From aa5ab4cf6c7335392094577380d2eaee8a0a8d52 Mon Sep 17 00:00:00 2001 +From: Kazuhito Hagio +Date: Thu, 29 Aug 2019 12:26:34 -0400 +Subject: [PATCH] x86_64: Fix incorrect exclusion by -e option with KASLR + +The -e option uses info->vmemmap_start for creating a table to determine +the positions of page structures that should be excluded, but it is a +hardcoded value even with KASLR-enabled vmcore. As a result, the option +excludes incorrect pages from it. + +To fix this, get the vmemmap start address from info->mem_map_data. + +Signed-off-by: Kazuhito Hagio +--- + arch/x86_64.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/makedumpfile-1.6.2/arch/x86_64.c b/makedumpfile-1.6.2/arch/x86_64.c +index 3c0fdc5e72fb..4eeaf4925f43 100644 +--- a/makedumpfile-1.6.2/arch/x86_64.c ++++ b/makedumpfile-1.6.2/arch/x86_64.c +@@ -579,6 +579,16 @@ find_vmemmap_x86_64() + if (NUMBER(sme_mask) != NOT_FOUND_NUMBER) + pmask &= ~(NUMBER(sme_mask)); + ++ /* ++ * vmemmap region can be randomized by KASLR. ++ * (currently we don't utilize info->vmemmap_end on x86_64.) ++ */ ++ if (info->mem_map_data && ++ info->mem_map_data[0].mem_map != NOT_MEMMAP_ADDR) ++ info->vmemmap_start = info->mem_map_data[0].mem_map; ++ ++ DEBUG_MSG("vmemmap_start: %16lx\n", info->vmemmap_start); ++ + pagestructsize = size_table.page; + hugepagesize = PTRS_PER_PMD * info->page_size; + vaddr_base = info->vmemmap_start; +-- +2.18.1 diff --git a/SOURCES/kexec-tools-2.0.15-vmcore-dmesg-vmcore-dmesg.c-Fix-shifting-error-repor.patch b/SOURCES/kexec-tools-2.0.15-vmcore-dmesg-vmcore-dmesg.c-Fix-shifting-error-repor.patch new file mode 100644 index 0000000..4a43b81 --- /dev/null +++ b/SOURCES/kexec-tools-2.0.15-vmcore-dmesg-vmcore-dmesg.c-Fix-shifting-error-repor.patch @@ -0,0 +1,38 @@ +From 544ef98f873688d875b453c3f7afc60c4fd0329c Mon Sep 17 00:00:00 2001 +From: Bhupesh Sharma +Date: Thu, 12 Sep 2019 12:43:23 +0530 +Subject: [PATCH] [PATCH] vmcore-dmesg/vmcore-dmesg.c: Fix shifting error + reported by cppcheck + +Running 'cppcheck' static code analyzer (see cppcheck(1)) +on 'vmcore-dmesg/vmcore-dmesg.c' shows the following +shifting error: + + $ cppcheck --enable=all vmcore-dmesg/vmcore-dmesg.c + Checking vmcore-dmesg/vmcore-dmesg.c ... + [vmcore-dmesg/vmcore-dmesg.c:17]: (error) Shifting signed 32-bit value + by 31 bits is undefined behaviour + +Fix the same via this patch. + +Cc: Lianbo Jiang +Signed-off-by: Bhupesh Sharma +--- + vmcore-dmesg/vmcore-dmesg.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/vmcore-dmesg/vmcore-dmesg.c b/vmcore-dmesg/vmcore-dmesg.c +index 41ab73147155..8790738c0e23 100644 +--- a/vmcore-dmesg/vmcore-dmesg.c ++++ b/vmcore-dmesg/vmcore-dmesg.c +@@ -23,7 +23,7 @@ + typedef Elf32_Nhdr Elf_Nhdr; + + /* stole this macro from kernel printk.c */ +-#define LOG_BUF_LEN_MAX (uint32_t)(1 << 31) ++#define LOG_BUF_LEN_MAX (uint32_t)(1U << 31) + + static const char *fname; + static Elf64_Ehdr ehdr; +-- +2.7.4 diff --git a/SPECS/kexec-tools.spec b/SPECS/kexec-tools.spec index 87d8c46..c949e0d 100644 --- a/SPECS/kexec-tools.spec +++ b/SPECS/kexec-tools.spec @@ -1,6 +1,6 @@ Name: kexec-tools Version: 2.0.15 -Release: 33%{?dist} +Release: 43%{?dist} License: GPLv2 Group: Applications/System Summary: The kexec/kdump userspace component. @@ -121,6 +121,9 @@ Patch722: kexec-tools-2.0.15-makedumpfile-when-refiltering-initialize-refiltered Patch723: kexec-tools-2.0.15-makedumpfile-Update-help-text-to-indicate-mem-usage-is-supp.patch Patch724: kexec-tools-2.0.15-makedumpfile-exclude-pages-that-are-logically-offline.patch Patch725: kexec-tools-2.0.15-makedumpfile-x86_64-Add-support-for-AMD-Secure-Memory-Encry.patch +Patch726: kexec-tools-2.0.15-Limit-the-size-of-vmcore-dmesg.txt-to-2G.patch +Patch727: kexec-tools-2.0.15-vmcore-dmesg-vmcore-dmesg.c-Fix-shifting-error-repor.patch +Patch728: kexec-tools-2.0.15-makedumpfile-x86_64-Fix-incorrect-exclusion-by-e-option-wit.patch # @@ -193,6 +196,9 @@ tar -z -x -v -f %{SOURCE25} %patch723 -p1 %patch724 -p1 %patch725 -p1 +%patch726 -p1 +%patch727 -p1 +%patch728 -p1 %ifarch ppc @@ -437,6 +443,42 @@ done %doc %changelog + +* Wed Oct 23 2019 Bhupesh Sharma 2.0.15-43 +- module-setup: re-fix 99kdumpbase network dependency +- kdumpctl: bail out immediately if host key verification failed + +* Mon Oct 14 2019 Bhupesh Sharma 2.0.15-42 +- makedumpfile: x86_64: Fix incorrect exclusion by -e option with KASLR + +* Tue Sep 24 2019 Bhupesh Sharma 2.0.15-41 +- kdumpctl: echo msg when waiting for connection + +* Thu Sep 19 2019 Bhupesh Sharma 2.0.15-40 +- Doc: amend the man page of kdump.conf + +* Mon Sep 16 2019 Bhupesh Sharma 2.0.15-39 +- vmcore-dmesg/vmcore-dmesg.c: Fix shifting error reported by cppcheck + +* Tue Sep 10 2019 Bhupesh Sharma 2.0.15-38 +- Limit the size of vmcore-dmesg.txt to 2G +- dracut-module-setup: fix bond ifcfg processing + +* Tue Sep 3 2019 Bhupesh Sharma 2.0.15-37 +- dracut-module-setup: get localhost alias by manual +- dracut-module-setup: filter out localhost for generic_fence_kdump +- kdumpctl: distinguish the reason for ssh failure + +* Wed Aug 14 2019 Bhupesh Sharma 2.0.15-36 +- kdumpctl: wait a while for network ready if dump target is ssh + +* Tue Aug 13 2019 Bhupesh Sharma 2.0.15-35 +- dracut-module-setup.sh: skip alias of localhost in get_pcs_fence_kdump_nodes() + +* Tue Jul 30 2019 Bhupesh Sharma 2.0.15-34 +- kdump.sysconfig/x86_64: Disable HEST by default +- Introduce kdumpctl rebuild command and update document + * Sat Jun 22 2019 Bhupesh Sharma 2.0.15-33 - makedumpfile: x86_64: Add support for AMD Secure Memory Encryption [Fix the Fix the patch to make sure makedumpfile-1.6.2/ path is used]