From 2a572d2d825735493a981b03b48877940ea2c17a Mon Sep 17 00:00:00 2001

From: Kir Kolyshkin <kolyshkin@gmail.com>

Date: Mon, 17 May 2021 14:11:35 -0700

Subject: [PATCH] rootfs: add mount destination validation

This is a manual backport of the CVE-2021-30465 fix. Original

description follows.

---

Because the target of a mount is inside a container (which may be a

volume that is shared with another container), there exists a race

condition where the target of the mount may change to a path containing

a symlink after we have sanitised the path -- resulting in us

inadvertently mounting the path outside of the container.

This is not immediately useful because we are in a mount namespace with

MS_SLAVE mount propagation applied to "/", so we cannot mount on top of

host paths in the host namespace. However, if any subsequent mountpoints

in the configuration use a subdirectory of that host path as a source,

those subsequent mounts will use an attacker-controlled source path

(resolved within the host rootfs) -- allowing the bind-mounting of "/"

into the container.

While arguably configuration issues like this are not entirely within

runc's threat model, within the context of Kubernetes (and possibly

other container managers that provide semi-arbitrary container creation

privileges to untrusted users) this is a legitimate issue. Since we

cannot block mounting from the host into the container, we need to block

the first stage of this attack (mounting onto a path outside the

container).

The long-term plan to solve this would be to migrate to libpathrs, but

as a stop-gap we implement libpathrs-like path verification through

readlink(/proc/self/fd/$n) and then do mount operations through the

procfd once it's been verified to be inside the container. The target

could move after we've checked it, but if it is inside the container

then we can assume that it is safe for the same reason that libpathrs

operations would be safe.

A slight wrinkle is the "copyup" functionality we provide for tmpfs,

which is the only case where we want to do a mount on the host

filesystem. To facilitate this, I split out the copy-up functionality

entirely so that the logic isn't interspersed with the regular tmpfs

logic. In addition, all dependencies on m.Destination being overwritten

have been removed since that pattern was just begging to be a source of

more mount-target bugs (we do still have to modify m.Destination for

tmpfs-copyup but we only do it temporarily).

Fixes: CVE-2021-30465

Reported-by: Etienne Champetier <champetier.etienne@gmail.com>

Co-authored-by: Noah Meyerhans <nmeyerha@amazon.com>

Reviewed-by: Samuel Karp <skarp@amazon.com>

Reviewed-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

Signed-off-by: Aleksa Sarai <cyphar@cyphar.com>

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>

---

 libcontainer/rootfs_linux.go | 154 +++++++++++++++++++----------------

 libcontainer/utils/utils.go  |  56 +++++++++++++

 2 files changed, 139 insertions(+), 71 deletions(-)

diff --git a/libcontainer/rootfs_linux.go b/libcontainer/rootfs_linux.go

index 42b66141..984fe289 100644

--- a/runc-66aedde759f33c190954815fb765eedc1d782dd9/libcontainer/rootfs_linux.go

+++ b/runc-66aedde759f33c190954815fb765eedc1d782dd9/libcontainer/rootfs_linux.go

@@ -14,6 +14,7 @@ import (

 	"syscall"

 	"time"

+	"github.com/Sirupsen/logrus"

 	securejoin "github.com/cyphar/filepath-securejoin"

 	"github.com/docker/docker/pkg/mount"

 	"github.com/mrunalp/fileutils"

@@ -145,8 +146,6 @@ func prepareBindMount(m *configs.Mount, rootfs string) error {

 	if err := checkProcMount(rootfs, dest, m.Source); err != nil {

 		return err

 	}

-	// update the mount with the correct dest after symlinks are resolved.

-	m.Destination = dest

 	if err := createIfNotExists(dest, stat.IsDir()); err != nil {

 		return err

 	}

@@ -154,12 +153,50 @@ func prepareBindMount(m *configs.Mount, rootfs string) error {

 	return nil

 }

+func doTmpfsCopyUp(m *configs.Mount, rootfs, mountLabel string) (Err error) {

+	// Set up a scratch dir for the tmpfs on the host.

+	tmpDir, err := ioutil.TempDir("/tmp", "runctmpdir")

+	if err != nil {

+		return newSystemErrorWithCause(err, "tmpcopyup: failed to create tmpdir")

+	}

+	defer os.RemoveAll(tmpDir)

+

+	// Configure the *host* tmpdir as if it's the container mount. We change

+	// m.Destination since we are going to mount *on the host*.

+	oldDest := m.Destination

+	m.Destination = tmpDir

+	err = mountPropagate(m, "/", mountLabel)

+	m.Destination = oldDest

+	if err != nil {

+		return err

+	}

+	defer func() {

+		if Err != nil {

+			if err := syscall.Unmount(tmpDir, syscall.MNT_DETACH); err != nil {

+				logrus.Warnf("tmpcopyup: failed to unmount tmpdir on error: %v", err)

+			}

+		}

+	}()

+

+	return libcontainerUtils.WithProcfd(rootfs, m.Destination, func(procfd string) (Err error) {

+		// Copy the container data to the host tmpdir. We append "/" to force

+		// CopyDirectory to resolve the symlink rather than trying to copy the

+		// symlink itself.

+		if err := fileutils.CopyDirectory(procfd+"/", tmpDir); err != nil {

+			return fmt.Errorf("tmpcopyup: failed to copy %s to %s (%s): %v", m.Destination, procfd, tmpDir, err)

+		}

+		// Now move the mount into the container.

+		if err := syscall.Mount(tmpDir, procfd, "", syscall.MS_MOVE, ""); err != nil {

+			return fmt.Errorf("tmpcopyup: failed to move mount %s to %s (%s): %v", tmpDir, procfd, m.Destination, err)

+		}

+		return nil

+	})

+}

+

 func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {

-	var (

-		dest = m.Destination

-	)

-	if !strings.HasPrefix(dest, rootfs) {

-		dest = filepath.Join(rootfs, dest)

+	dest, err := securejoin.SecureJoin(rootfs, m.Destination)

+	if err != nil {

+		return err

 	}

 	switch m.Device {

@@ -182,41 +219,20 @@ func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {

 		}

 		return nil

 	case "tmpfs":

-		copyUp := m.Extensions&configs.EXT_COPYUP == configs.EXT_COPYUP

-		tmpDir := ""

 		stat, err := os.Stat(dest)

 		if err != nil {

 			if err := os.MkdirAll(dest, 0755); err != nil {

 				return err

 			}

 		}

-		if copyUp {

-			tmpDir, err = ioutil.TempDir("/tmp", "runctmpdir")

-			if err != nil {

-				return newSystemErrorWithCause(err, "tmpcopyup: failed to create tmpdir")

-			}

-			defer os.RemoveAll(tmpDir)

-			m.Destination = tmpDir

+		if m.Extensions&configs.EXT_COPYUP == configs.EXT_COPYUP {

+			err = doTmpfsCopyUp(m, rootfs, mountLabel)

+		} else {

+			err = mountPropagate(m, rootfs, mountLabel)

 		}

-		if err := mountPropagate(m, rootfs, mountLabel); err != nil {

+		if err != nil {

 			return err

 		}

-		if copyUp {

-			if err := fileutils.CopyDirectory(dest, tmpDir); err != nil {

-				errMsg := fmt.Errorf("tmpcopyup: failed to copy %s to %s: %v", dest, tmpDir, err)

-				if err1 := syscall.Unmount(tmpDir, syscall.MNT_DETACH); err1 != nil {

-					return newSystemErrorWithCausef(err1, "tmpcopyup: %v: failed to unmount", errMsg)

-				}

-				return errMsg

-			}

-			if err := syscall.Mount(tmpDir, dest, "", syscall.MS_MOVE, ""); err != nil {

-				errMsg := fmt.Errorf("tmpcopyup: failed to move mount %s to %s: %v", tmpDir, dest, err)

-				if err1 := syscall.Unmount(tmpDir, syscall.MNT_DETACH); err1 != nil {

-					return newSystemErrorWithCausef(err1, "tmpcopyup: %v: failed to unmount", errMsg)

-				}

-				return errMsg

-			}

-		}

 		if stat != nil {

 			if err = os.Chmod(dest, stat.Mode()); err != nil {

 				return err

@@ -299,19 +315,9 @@ func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {

 			}

 		}

 	default:

-		// ensure that the destination of the mount is resolved of symlinks at mount time because

-		// any previous mounts can invalidate the next mount's destination.

-		// this can happen when a user specifies mounts within other mounts to cause breakouts or other

-		// evil stuff to try to escape the container's rootfs.

-		var err error

-		if dest, err = securejoin.SecureJoin(rootfs, m.Destination); err != nil {

-			return err

-		}

 		if err := checkProcMount(rootfs, dest, m.Source); err != nil {

 			return err

 		}

-		// update the mount with the correct dest after symlinks are resolved.

-		m.Destination = dest

 		if err := os.MkdirAll(dest, 0755); err != nil {

 			return err

 		}

@@ -485,7 +491,7 @@ func createDevices(config *configs.Config) error {

 	return nil

 }

-func bindMountDeviceNode(dest string, node *configs.Device) error {

+func bindMountDeviceNode(rootfs, dest string, node *configs.Device) error {

 	f, err := os.Create(dest)

 	if err != nil && !os.IsExist(err) {

 		return err

@@ -493,24 +499,29 @@ func bindMountDeviceNode(dest string, node *configs.Device) error {

 	if f != nil {

 		f.Close()

 	}

-	return syscall.Mount(node.Path, dest, "bind", syscall.MS_BIND, "")

+	return libcontainerUtils.WithProcfd(rootfs, dest, func(procfd string) error {

+		return syscall.Mount(node.Path, procfd, "bind", syscall.MS_BIND, "")

+	})

 }

 // Creates the device node in the rootfs of the container.

 func createDeviceNode(rootfs string, node *configs.Device, bind bool) error {

-	dest := filepath.Join(rootfs, node.Path)

+	dest, err := securejoin.SecureJoin(rootfs, node.Path)

+	if err != nil {

+		return err

+	}

 	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {

 		return err

 	}

 	if bind {

-		return bindMountDeviceNode(dest, node)

+		return bindMountDeviceNode(rootfs, dest, node)

 	}

 	if err := mknodDevice(dest, node); err != nil {

 		if os.IsExist(err) {

 			return nil

 		} else if os.IsPermission(err) {

-			return bindMountDeviceNode(dest, node)

+			return bindMountDeviceNode(rootfs, dest, node)

 		}

 		return err

 	}

@@ -807,43 +818,44 @@ func writeSystemProperty(key, value string) error {

 }

 func remount(m *configs.Mount, rootfs string) error {

-	var (

-		dest = m.Destination

-	)

-	if !strings.HasPrefix(dest, rootfs) {

-		dest = filepath.Join(rootfs, dest)

-	}

-	if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags|syscall.MS_REMOUNT), ""); err != nil {

-		return err

-	}

-	return nil

+	return libcontainerUtils.WithProcfd(rootfs, m.Destination, func(procfd string) error {

+		return syscall.Mount(m.Source, procfd, m.Device, uintptr(m.Flags|syscall.MS_REMOUNT), "")

+	})

 }

 // Do the mount operation followed by additional mounts required to take care

-// of propagation flags.

+// of propagation flags.  This will always be scoped inside the container rootfs.

 func mountPropagate(m *configs.Mount, rootfs string, mountLabel string) error {

 	var (

-		dest  = m.Destination

 		data  = label.FormatMountLabel(m.Data, mountLabel)

 		flags = m.Flags

 	)

-	if libcontainerUtils.CleanPath(dest) == "/dev" {

+	if libcontainerUtils.CleanPath(m.Destination) == "/dev" {

 		flags &= ^syscall.MS_RDONLY

 	}

-	copyUp := m.Extensions&configs.EXT_COPYUP == configs.EXT_COPYUP

-	if !(copyUp || strings.HasPrefix(dest, rootfs)) {

-		dest = filepath.Join(rootfs, dest)

+	// Because the destination is inside a container path which might be

+	// mutating underneath us, we verify that we are actually going to mount

+	// inside the container with WithProcfd() -- mounting through a procfd

+	// mounts on the target.

+	if err := libcontainerUtils.WithProcfd(rootfs, m.Destination, func(procfd string) error {

+		return syscall.Mount(m.Source, procfd, m.Device, uintptr(flags), data)

+	}); err != nil {

+		return fmt.Errorf("mount through procfd: %v", err)

 	}

-	if err := syscall.Mount(m.Source, dest, m.Device, uintptr(flags), data); err != nil {

-		return err

-	}

-

-	for _, pflag := range m.PropagationFlags {

-		if err := syscall.Mount("", dest, "", uintptr(pflag), ""); err != nil {

-			return err

+	// We have to apply mount propagation flags in a separate WithProcfd() call

+	// because the previous call invalidates the passed procfd -- the mount

+	// target needs to be re-opened.

+	if err := libcontainerUtils.WithProcfd(rootfs, m.Destination, func(procfd string) error {

+		for _, pflag := range m.PropagationFlags {

+			if err := syscall.Mount("", procfd, "", uintptr(pflag), ""); err != nil {

+				return err

+			}

 		}

+		return nil

+	}); err != nil {

+		return fmt.Errorf("change mount propagation through procfd: %v", err)

 	}

 	return nil

 }

diff --git a/libcontainer/utils/utils.go b/libcontainer/utils/utils.go

index 2b35b9a7..a89fc6a2 100644

--- a/runc-66aedde759f33c190954815fb765eedc1d782dd9/libcontainer/utils/utils.go

+++ b/runc-66aedde759f33c190954815fb765eedc1d782dd9/libcontainer/utils/utils.go

@@ -4,12 +4,17 @@ import (

 	"crypto/rand"

 	"encoding/hex"

 	"encoding/json"

+	"fmt"

 	"io"

 	"os"

 	"path/filepath"

+	"strconv"

 	"strings"

 	"syscall"

 	"unsafe"

+

+	securejoin "github.com/cyphar/filepath-securejoin"

+	"golang.org/x/sys/unix"

 )

 const (

@@ -87,6 +92,57 @@ func CleanPath(path string) string {

 	return filepath.Clean(path)

 }

+// stripRoot returns the passed path, stripping the root path if it was

+// (lexicially) inside it. Note that both passed paths will always be treated

+// as absolute, and the returned path will also always be absolute. In

+// addition, the paths are cleaned before stripping the root.

+func stripRoot(root, path string) string {

+	// Make the paths clean and absolute.

+	root, path = CleanPath("/"+root), CleanPath("/"+path)

+	switch {

+	case path == root:

+		path = "/"

+	case root == "/":

+		// do nothing

+	case strings.HasPrefix(path, root+"/"):

+		path = strings.TrimPrefix(path, root+"/")

+	}

+	return CleanPath("/" + path)

+}

+

+// WithProcfd runs the passed closure with a procfd path (/proc/self/fd/...)

+// corresponding to the unsafePath resolved within the root. Before passing the

+// fd, this path is verified to have been inside the root -- so operating on it

+// through the passed fdpath should be safe. Do not access this path through

+// the original path strings, and do not attempt to use the pathname outside of

+// the passed closure (the file handle will be freed once the closure returns).

+func WithProcfd(root, unsafePath string, fn func(procfd string) error) error {

+	// Remove the root then forcefully resolve inside the root.

+	unsafePath = stripRoot(root, unsafePath)

+	path, err := securejoin.SecureJoin(root, unsafePath)

+	if err != nil {

+		return fmt.Errorf("resolving path inside rootfs failed: %v", err)

+	}

+

+	// Open the target path.

+	fh, err := os.OpenFile(path, unix.O_PATH|unix.O_CLOEXEC, 0)

+	if err != nil {

+		return fmt.Errorf("open o_path procfd: %v", err)

+	}

+	defer fh.Close()

+

+	// Double-check the path is the one we expected.

+	procfd := "/proc/self/fd/" + strconv.Itoa(int(fh.Fd()))

+	if realpath, err := os.Readlink(procfd); err != nil {

+		return fmt.Errorf("procfd verification failed: %v", err)

+	} else if realpath != path {

+		return fmt.Errorf("possibly malicious path detected -- refusing to operate on %s", realpath)

+	}

+

+	// Run the closure.

+	return fn(procfd)

+}

+

 // SearchLabels searches a list of key-value pairs for the provided key and

 // returns the corresponding value. The pairs must be separated with '='.

 func SearchLabels(labels []string, query string) string {