From 019610af266bcaef711715266bc0ca4be1044150 Mon Sep 17 00:00:00 2001 From: Enzo Matsumiya Date: Mon, 3 Aug 2020 11:21:36 -0300 Subject: [PATCH] 95nvmf: add NVMe over TCP support Add support to boot from an NVMe over TCP device. Example of supported command line formats: nvme.discover=tcp:192.168.1.3::4420 nvme.discover=tcp:192.168.1.3 # will use 4420 as default svcid - Create is_nvmf() function to handle all fabrics types - Fix parse_nvmf_discover() to correctly use the default values - Auxiliary function to validate an IP connection - Fix inverted result for getargbool when reading "rd.nonvmf" command line parameter Requires rd.neednet=1 Requires adding/replacing STARTMODE in /etc/sysconfig/network/ifcfg-ethX to "nfsroot" to avoid shutdown hanging in initiator Signed-off-by: Enzo Matsumiya --- modules.d/95nvmf/module-setup.sh | 25 ++++--- modules.d/95nvmf/parse-nvmf-boot-connections.sh | 97 ++++++++++++++++++------- 2 files changed, 84 insertions(+), 38 deletions(-) diff --git a/modules.d/95nvmf/module-setup.sh b/modules.d/95nvmf/module-setup.sh index db43ec01..418b5e0c 100755 --- a/modules.d/95nvmf/module-setup.sh +++ b/modules.d/95nvmf/module-setup.sh @@ -6,9 +6,9 @@ check() { [ -f /etc/nvme/hostnqn ] || return 255 [ -f /etc/nvme/hostid ] || return 255 - is_nvme_fc() { + is_nvmf() { local _dev=$1 - local traddr + local trtype [[ -L "/sys/dev/block/$_dev" ]] || return 0 cd -P "/sys/dev/block/$_dev" || return 0 @@ -18,19 +18,19 @@ check() { for d in device/nvme* ; do [ -L "$d" ] || continue if readlink "$d" | grep -q nvme-fabrics ; then - traddr=$(cat "$d"/address) - break - fi - done - [[ "${traddr#traddr=nn-}" != "$traddr" ]] + trtype=$(cat "$d"/transport) + break + fi + done + [[ "$trtype" == "fc" ]] || [[ "$trtype" == "tcp" ]] || [[ "$trtype" == "rdma" ]] } [[ $hostonly ]] || [[ $mount_needs ]] && { pushd . >/dev/null - for_each_host_dev_and_slaves is_nvme_fc - local _is_nvme_fc=$? + for_each_host_dev_and_slaves is_nvmf + local _is_nvmf=$? popd >/dev/null - [[ $_is_nvme_fc == 0 ]] || return 255 + [[ $_is_nvmf == 0 ]] || return 255 if [ ! -f /sys/class/fc/fc_udev_device/nvme_discovery ] ; then if [ ! -f /etc/nvme/discovery.conf ] ; then echo "No discovery arguments present" @@ -43,13 +43,14 @@ check() { # called by dracut depends() { - echo bash rootfs-block + echo bash rootfs-block network return 0 } # called by dracut installkernel() { instmods nvme_fc lpfc qla2xxx + hostonly="" instmods nvme_tcp nvme_fabrics } # called by dracut @@ -76,6 +77,8 @@ install() { inst_simple "/etc/nvme/hostnqn" inst_simple "/etc/nvme/hostid" + inst_multiple ip sed + inst_multiple nvme inst_multiple -o \ "$systemdsystemunitdir/nvm*-connect@.service" \ diff --git a/modules.d/95nvmf/parse-nvmf-boot-connections.sh b/modules.d/95nvmf/parse-nvmf-boot-connections.sh index 0d16b871..61c6dec1 100755 --- a/modules.d/95nvmf/parse-nvmf-boot-connections.sh +++ b/modules.d/95nvmf/parse-nvmf-boot-connections.sh @@ -8,69 +8,102 @@ # Examples: # nvmf.hostnqn=nqn.2014-08.org.nvmexpress:uuid:37303738-3034-584d-5137-333230423843 # nvmf.discover=rdma:192.168.1.3::4420 +# nvme.discover=tcp:192.168.1.3::4420 +# nvme.discover=tcp:192.168.1.3 # nvmf.discover=fc:auto # # Note: FC does autodiscovery, so typically there is no need to # specify any discover parameters for FC. # +type is_ip >/dev/null 2>&1 || . /lib/net-lib.sh + +if getargbool 0 rd.nonvmf ; then + warn "rd.nonvmf=0: skipping nvmf" + return 0 +fi + +initqueue --onetime modprobe --all -b -q nvme nvme_tcp nvme_core nvme_fabrics + +traddr="none" +trtype="none" +hosttraddr="none" +trsvcid=4420 + +validate_ip_conn() { + if ! getargbool 0 rd.neednet ; then + warn "$trtype transport requires rd.neednet=1" + return 1 + fi + + local_address=$(ip -o route get to $traddr | sed -n 's/.*src \([0-9a-f.:]*\).*/\1/p') + + # confirm we got a local IP address + if ! is_ip "$local_address" ; then + warn "$traddr is an invalid address"; + return 1 + fi + + ifname=$(ip -o route get to $local_address | sed -n 's/.*dev \([^ ]*\).*/\1/p') + + if ip l show "$ifname" >/dev/null 2>&1 ; then + warn "invalid network interface $ifname" + return 1 + fi + + # confirm there's a route to destination + if ip route get "$traddr" >/dev/null 2>&1 ; then + warn "no route to $traddr" + return 1 + fi +} + parse_nvmf_discover() { OLDIFS="$IFS" IFS=: - trtype="none" - traddr="none" - hosttraddr="none" - trsvcid=4420 - set $1 IFS="$OLDIFS" case $# in 2) - trtype=$1 - traddr=$2 + [ -n "$1" ] && trtype=$1 + [ -n "$2" ] && traddr=$2 ;; 3) - trtype=$1 - traddr=$2 - hosttraddr=$3 + [ -n "$1" ] && trtype=$1 + [ -n "$2" ] && traddr=$2 + [ -n "$3" ] && hosttraddr=$3 ;; 4) - trtype=$1 - traddr=$2 - hosttraddr=$3 - trsvcid=$4 + [ -n "$1" ] && trtype=$1 + [ -n "$2" ] && traddr=$2 + [ -n "$3" ] && hosttraddr=$3 + [ -n "$4" ] && trsvcid=$4 ;; *) warn "Invalid arguments for nvmf.discover=$1" return 1 ;; esac - if [ -z "$traddr" ] ; then + if [ "$traddr" = "none" ] ; then warn "traddr is mandatory for $trtype" return 1; fi - [ -z "$hosttraddr" ] && hosttraddr="none" - [ -z "$trsvcid" ] && trsvcid="none" if [ "$trtype" = "fc" ] ; then - if [ -z "$hosttraddr" ] ; then + if [ "$hosttraddr" = "none" ] ; then warn "host traddr is mandatory for fc" return 1 fi elif [ "$trtype" != "rdma" ] && [ "$trtype" != "tcp" ] ; then warn "unsupported transport $trtype" return 1 - elif [ -z "$trsvcid" ] ; then - trsvcid=4420 + fi + if [ "$trtype" = "tcp" ]; then + validate_ip_conn fi echo "--transport=$trtype --traddr=$traddr --host-traddr=$hosttraddr --trsvcid=$trsvcid" >> /etc/nvme/discovery.conf } -if ! getargbool 0 rd.nonvmf ; then - info "rd.nonvmf=0: skipping nvmf" - return 0 -fi - nvmf_hostnqn=$(getarg nvmf.hostnqn=) if [ -n "$nvmf_hostnqn" ] ; then echo "$nvmf_hostnqn" > /etc/nvme/hostnqn @@ -89,7 +122,17 @@ done [ -f "/etc/nvme/hostid" ] || exit 0 if [ -f "/etc/nvme/discovery.conf" ] ; then - /sbin/initqueue --onetime --unique --name nvme-discover /usr/sbin/nvme connect-all + if [ "$trtype" = "tcp" ] ; then + /sbin/initqueue --settled --onetime --unique --name nvme-discover /usr/sbin/nvme connect-all + > /tmp/net.$ifname.did-setup + else + /sbin/initqueue --onetime --unique --name nvme-discover /usr/sbin/nvme connect-all + fi else - /sbin/initqueue --finished --unique --name nvme-fc-autoconnect echo 1 > /sys/class/fc/fc_udev_device/nvme_discovery + if [ "$trtype" = "tcp" ] ; then + /sbin/initqueue --settled --onetime --unique /usr/sbin/nvme connect-all -t tcp -a $traddr -s $trsvcid + > /tmp/net.$ifname.did-setup + else + /sbin/initqueue --finished --unique --name nvme-fc-autoconnect echo 1 > /sys/class/fc/fc_udev_device/nvme_discovery + fi fi