Blame SOURCES/coreutils-8.22-cp-sparsecorrupt.patch

f180de
From eea6b49210edf69682b2d0606bee17bbccb3765b Mon Sep 17 00:00:00 2001
f180de
From: Dmitry Monakhov <dmonakhov@openvz.org>
f180de
Date: Fri, 30 Oct 2015 22:04:46 +0000
f180de
Subject: [PATCH] copy: fix copying of extents beyond the apparent file size
f180de
f180de
fallocate can allocate extents beyond EOF via FALLOC_FL_KEEP_SIZE.
f180de
Where there is a gap (hole) between the extents, and EOF is within
f180de
that gap, the final hole wasn't reproduced, resulting in silent
f180de
data corruption in the copied file (size too small).
f180de
f180de
* src/copy.c (extent_copy): Ensure we don't process extents
f180de
beyond the apparent file size, since processing and allocating
f180de
those is not currently supported.
f180de
* tests/cp/fiemap-extents.sh: Renamed from tests/cp/fiemap-empty.sh
f180de
and re-enable parts checking the extents at and beyond EOF.
f180de
* tests/local.mk: Reference the renamed test.
f180de
Fixes http://bugs.gnu.org/21790
f180de
---
f180de
 src/copy.c                 |   19 ++++++++-
f180de
 tests/cp/fiemap-empty.sh   |  102 --------------------------------------------
f180de
 tests/cp/fiemap-extents.sh |   81 +++++++++++++++++++++++++++++++++++
f180de
 tests/local.mk             |    2 +-
f180de
 5 files changed, 100 insertions(+), 104 deletions(-)
f180de
 delete mode 100755 tests/cp/fiemap-empty.sh
f180de
 create mode 100755 tests/cp/fiemap-extents.sh
f180de
f180de
diff --git a/src/copy.c b/src/copy.c
f180de
index dc1cd29..6771bb5 100644
f180de
--- a/src/copy.c
f180de
+++ b/src/copy.c
f180de
@@ -432,6 +432,20 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
f180de
               ext_len = 0;
f180de
             }
f180de
 
f180de
+          /* Truncate extent to EOF.  Extents starting after EOF are
f180de
+             treated as zero length extents starting right after EOF.
f180de
+             Generally this will trigger with an extent starting after
f180de
+             src_total_size, and result in creating a hole or zeros until EOF.
f180de
+             Though in a file in which extents have changed since src_total_size
f180de
+             was determined, we might have an extent spanning that size,
f180de
+             in which case we'll only copy data up to that size.  */
f180de
+          if (src_total_size < ext_start + ext_len)
f180de
+            {
f180de
+              if (src_total_size < ext_start)
f180de
+                ext_start = src_total_size;
f180de
+              ext_len = src_total_size - ext_start;
f180de
+            }
f180de
+
f180de
           hole_size = ext_start - last_ext_start - last_ext_len;
f180de
 
f180de
           wrote_hole_at_eof = false;
f180de
@@ -495,14 +509,17 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
f180de
               off_t n_read;
f180de
               empty_extent = false;
f180de
               last_ext_len = ext_len;
f180de
+              bool read_hole;
f180de
 
f180de
               if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size,
f180de
                                   sparse_mode == SPARSE_ALWAYS,
f180de
                                   src_name, dst_name, ext_len, &n_read,
f180de
-                                  &wrote_hole_at_eof))
f180de
+                                  &read_hole))
f180de
                 goto fail;
f180de
 
f180de
               dest_pos = ext_start + n_read;
f180de
+              if (n_read)
f180de
+                wrote_hole_at_eof = read_hole;
f180de
             }
f180de
 
f180de
           /* If the file ends with unwritten extents not accounted for in the
f180de
deleted file mode 100755
f180de
index b3b2cd7..0000000
f180de
--- a/tests/cp/fiemap-empty.sh
f180de
+++ /dev/null
f180de
@@ -1,101 +0,0 @@
f180de
-#!/bin/sh
f180de
-# Test cp reads unwritten extents efficiently
f180de
-
f180de
-# Copyright (C) 2011-2013 Free Software Foundation, Inc.
f180de
-
f180de
-# This program is free software: you can redistribute it and/or modify
f180de
-# it under the terms of the GNU General Public License as published by
f180de
-# the Free Software Foundation, either version 3 of the License, or
f180de
-# (at your option) any later version.
f180de
-
f180de
-# This program is distributed in the hope that it will be useful,
f180de
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
f180de
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
f180de
-# GNU General Public License for more details.
f180de
-
f180de
-# You should have received a copy of the GNU General Public License
f180de
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
f180de
-
f180de
-. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
f180de
-print_ver_ cp
f180de
-
f180de
-# FIXME: enable any part of this test that is still relevant,
f180de
-# or, if none are relevant (now that cp does not handle unwritten
f180de
-# extents), just remove the test altogether.
f180de
-skip_ 'disabled for now'
f180de
-
f180de
-touch fiemap_chk
f180de
-fiemap_capable_ fiemap_chk ||
f180de
-  skip_ 'this file system lacks FIEMAP support'
f180de
-rm fiemap_chk
f180de
-
f180de
-# TODO: rather than requiring $(fallocate), possible add
f180de
-# this functionality to truncate --alloc
f180de
-fallocate --help >/dev/null || skip_ 'The fallocate utility is required'
f180de
-fallocate -l 1 -n falloc.test ||
f180de
-  skip_ 'this file system lacks FALLOCATE support'
f180de
-rm falloc.test
f180de
-
f180de
-# Require more space than we'll actually use, so that
f180de
-# tests run in parallel do not run out of space.
f180de
-# Otherwise, with inadequate space, simply running the following
f180de
-# fallocate command would induce a temporary disk-full condition,
f180de
-# which would cause failure of unrelated tests run in parallel.
f180de
-require_file_system_bytes_free_ 800000000
f180de
-
f180de
-fallocate -l 600MiB space.test ||
f180de
-  skip_ 'this test needs at least 600MiB free space'
f180de
-
f180de
-# Disable this test on old BTRFS (e.g. Fedora 14)
f180de
-# which reports ordinary extents for unwritten ones.
f180de
-filefrag space.test || skip_ 'the 'filefrag' utility is missing'
f180de
-filefrag -v space.test | grep -F 'unwritten' > /dev/null ||
f180de
-  skip_ 'this file system does not report empty extents as "unwritten"'
f180de
-
f180de
-rm space.test
f180de
-
f180de
-# Ensure we read a large empty file quickly
f180de
-fallocate -l 300MiB empty.big || framework_failure_
f180de
-timeout 3 cp --sparse=always empty.big cp.test || fail=1
f180de
-test $(stat -c %s empty.big) = $(stat -c %s cp.test) || fail=1
f180de
-rm empty.big cp.test
f180de
-
f180de
-# Ensure we handle extents beyond file size correctly.
f180de
-# Note until we support fallocate, we will not maintain
f180de
-# the file allocation.  FIXME: amend this test when fallocate is supported.
f180de
-fallocate -l 10MiB -n unwritten.withdata || framework_failure_
f180de
-dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unwritten.withdata
f180de
-cp unwritten.withdata cp.test || fail=1
f180de
-test $(stat -c %s unwritten.withdata) = $(stat -c %s cp.test) || fail=1
f180de
-cmp unwritten.withdata cp.test || fail=1
f180de
-rm unwritten.withdata cp.test
f180de
-
f180de
-# The following to generate unaccounted extents followed by a hole, is not
f180de
-# supported by ext4 at least. The ftruncate discards all extents not
f180de
-# accounted for in the size.
f180de
-#  fallocate -l 10MiB -n unacc.withholes
f180de
-#  dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unacc.withholes
f180de
-#  truncate -s20M unacc.withholes
f180de
-
f180de
-# Ensure we handle a hole after empty extents correctly.
f180de
-# Since all extents are accounted for in the size,
f180de
-# we can maintain the allocation independently from
f180de
-# fallocate() support.
f180de
-fallocate -l 10MiB empty.withholes
f180de
-truncate -s 20M empty.withholes
f180de
-sectors_per_block=$(expr $(stat -c %o .) / 512)
f180de
-cp empty.withholes cp.test || fail=1
f180de
-test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1
f180de
-# These are usually equal but can vary by an IO block due to alignment
f180de
-alloc_diff=$(expr $(stat -c %b empty.withholes) - $(stat -c %b cp.test))
f180de
-alloc_diff=$(echo $alloc_diff | tr -d -- -) # abs()
f180de
-test $alloc_diff -le $sectors_per_block || fail=1
f180de
-# Again with SPARSE_ALWAYS
f180de
-cp --sparse=always empty.withholes cp.test || fail=1
f180de
-test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1
f180de
-# cp.test should take 0 space, but allowing for some systems
f180de
-# that store default extended attributes in data blocks
f180de
-test $(stat -c %b cp.test) -le $sectors_per_block || fail=1
f180de
-rm empty.withholes cp.test
f180de
-
f180de
-Exit $fail
f180de
diff --git a/tests/cp/fiemap-extents.sh b/tests/cp/fiemap-extents.sh
f180de
new file mode 100755
f180de
index 0000000..55ec5df
f180de
--- /dev/null
f180de
+++ b/tests/cp/fiemap-extents.sh
f180de
@@ -0,0 +1,81 @@
f180de
+#!/bin/sh
f180de
+# Test cp handles extents correctly
f180de
+
f180de
+# Copyright (C) 2011-2015 Free Software Foundation, Inc.
f180de
+
f180de
+# This program is free software: you can redistribute it and/or modify
f180de
+# it under the terms of the GNU General Public License as published by
f180de
+# the Free Software Foundation, either version 3 of the License, or
f180de
+# (at your option) any later version.
f180de
+
f180de
+# This program is distributed in the hope that it will be useful,
f180de
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
f180de
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
f180de
+# GNU General Public License for more details.
f180de
+
f180de
+# You should have received a copy of the GNU General Public License
f180de
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
f180de
+
f180de
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
f180de
+print_ver_ cp
f180de
+
f180de
+require_sparse_support_
f180de
+
f180de
+touch fiemap_chk
f180de
+fiemap_capable_ fiemap_chk ||
f180de
+  skip_ 'this file system lacks FIEMAP support'
f180de
+rm fiemap_chk
f180de
+
f180de
+fallocate --help >/dev/null || skip_ 'The fallocate utility is required'
f180de
+touch falloc.test || framework_failure_
f180de
+fallocate -l 1 -o 0 -n falloc.test ||
f180de
+  skip_ 'this file system lacks FALLOCATE support'
f180de
+rm falloc.test
f180de
+
f180de
+# We don't currently handle unwritten extents specially
f180de
+if false; then
f180de
+# Require more space than we'll actually use, so that
f180de
+# tests run in parallel do not run out of space.
f180de
+# Otherwise, with inadequate space, simply running the following
f180de
+# fallocate command would induce a temporary disk-full condition,
f180de
+# which would cause failure of unrelated tests run in parallel.
f180de
+require_file_system_bytes_free_ 800000000
f180de
+
f180de
+fallocate -l 600MiB space.test ||
f180de
+  skip_ 'this test needs at least 600MiB free space'
f180de
+
f180de
+# Disable this test on old BTRFS (e.g. Fedora 14)
f180de
+# which reports ordinary extents for unwritten ones.
f180de
+filefrag space.test || skip_ 'the 'filefrag' utility is missing'
f180de
+filefrag -v space.test | grep -F 'unwritten' > /dev/null ||
f180de
+  skip_ 'this file system does not report empty extents as "unwritten"'
f180de
+
f180de
+rm space.test
f180de
+
f180de
+# Ensure we read a large empty file quickly
f180de
+fallocate -l 300MiB empty.big || framework_failure_
f180de
+timeout 3 cp --sparse=always empty.big cp.test || fail=1
f180de
+test $(stat -c %s empty.big) = $(stat -c %s cp.test) || fail=1
f180de
+rm empty.big cp.test
f180de
+fi
f180de
+
f180de
+# Ensure we handle extents beyond file size correctly.
f180de
+# Note until we support fallocate, we will not maintain
f180de
+# the file allocation.  FIXME: amend this test if fallocate is supported.
f180de
+# Note currently this only uses fiemap logic when the allocation (-l)
f180de
+# is smaller than the size, thus identifying the file as sparse.
f180de
+# Note the '-l 1' case is an effective noop, and just checks
f180de
+# a file with a trailing hole is copied correctly.
f180de
+for sparse_mode in always auto never; do
f180de
+  for alloc in '-l 4MiB ' '-l 1MiB -o 4MiB' '-l 1'; do
f180de
+    dd count=10 if=/dev/urandom iflag=fullblock of=unwritten.withdata
f180de
+    truncate -s 2MiB unwritten.withdata || framework_failure_
f180de
+    fallocate $alloc -n unwritten.withdata || framework_failure_
f180de
+    cp --sparse=$sparse_mode unwritten.withdata cp.test || fail=1
f180de
+    test $(stat -c %s unwritten.withdata) = $(stat -c %s cp.test) || fail=1
f180de
+    cmp unwritten.withdata cp.test || fail=1
f180de
+    rm unwritten.withdata cp.test
f180de
+  done
f180de
+done
f180de
+
f180de
+Exit $fail
f180de
diff --git a/tests/local.mk b/tests/local.mk
f180de
index adf96f0..89fdbb0 100644
f180de
--- a/tests/local.mk
f180de
+++ b/tests/local.mk
f180de
@@ -446,7 +446,7 @@ all_tests =					\
f180de
   tests/cp/existing-perm-dir.sh			\
f180de
   tests/cp/existing-perm-race.sh		\
f180de
   tests/cp/fail-perm.sh				\
f180de
-  tests/cp/fiemap-empty.sh			\
f180de
+  tests/cp/fiemap-extents.sh			\
f180de
   tests/cp/fiemap-FMR.sh			\
f180de
   tests/cp/fiemap-perf.sh			\
f180de
   tests/cp/fiemap-2.sh				\
f180de
-- 
f180de
1.7.2.5
f180de