958e1b
From 5f9e1e7be5043bce75d940db098b497d3dd78813 Mon Sep 17 00:00:00 2001
958e1b
Message-Id: <5f9e1e7be5043bce75d940db098b497d3dd78813.1418766606.git.jen@redhat.com>
958e1b
In-Reply-To: <6f81b4847eb68ebdf54a8f1a771e19d112d74152.1418766606.git.jen@redhat.com>
958e1b
References: <6f81b4847eb68ebdf54a8f1a771e19d112d74152.1418766606.git.jen@redhat.com>
958e1b
From: Fam Zheng <famz@redhat.com>
958e1b
Date: Thu, 4 Dec 2014 00:05:19 -0600
958e1b
Subject: [CHANGE 25/31] vmdk: Optimize cluster allocation
958e1b
To: rhvirt-patches@redhat.com,
958e1b
    jen@redhat.com
958e1b
958e1b
RH-Author: Fam Zheng <famz@redhat.com>
958e1b
Message-id: <1417651524-18041-26-git-send-email-famz@redhat.com>
958e1b
Patchwork-id: 62698
958e1b
O-Subject: [RHEL-7.1 qemu-kvm PATCH v5 25/30] vmdk: Optimize cluster allocation
958e1b
Bugzilla: 1002493
958e1b
RH-Acked-by: Jeffrey Cody <jcody@redhat.com>
958e1b
RH-Acked-by: Markus Armbruster <armbru@redhat.com>
958e1b
RH-Acked-by: Max Reitz <mreitz@redhat.com>
958e1b
958e1b
This drops the unnecessary bdrv_truncate() from, and also improves,
958e1b
cluster allocation code path.
958e1b
958e1b
Before, when we need a new cluster, get_cluster_offset truncates the
958e1b
image to bdrv_getlength() + cluster_size, and returns the offset of
958e1b
added area, i.e. the image length before truncating.
958e1b
958e1b
This is not efficient, so it's now rewritten as:
958e1b
958e1b
  - Save the extent file length when opening.
958e1b
958e1b
  - When allocating cluster, use the saved length as cluster offset.
958e1b
958e1b
  - Don't truncate image, because we'll anyway write data there: just
958e1b
    write any data at the EOF position, in descending priority:
958e1b
958e1b
    * New user data (cluster allocation happens in a write request).
958e1b
958e1b
    * Filling data in the beginning and/or ending of the new cluster, if
958e1b
      not covered by user data: either backing file content (COW), or
958e1b
      zero for standalone images.
958e1b
958e1b
One major benifit of this change is, on host mounted NFS images, even
958e1b
over a fast network, ftruncate is slow (see the example below). This
958e1b
change significantly speeds up cluster allocation. Comparing by
958e1b
converting a cirros image (296M) to VMDK on an NFS mount point, over
958e1b
1Gbe LAN:
958e1b
958e1b
    $ time qemu-img convert cirros-0.3.1.img /mnt/a.raw -O vmdk
958e1b
958e1b
    Before:
958e1b
        real    0m21.796s
958e1b
        user    0m0.130s
958e1b
        sys     0m0.483s
958e1b
958e1b
    After:
958e1b
        real    0m2.017s
958e1b
        user    0m0.047s
958e1b
        sys     0m0.190s
958e1b
958e1b
We also get rid of unchecked bdrv_getlength() and bdrv_truncate(), and
958e1b
get a little more documentation in function comments.
958e1b
958e1b
Tested that this passes qemu-iotests for all VMDK subformats.
958e1b
958e1b
Signed-off-by: Fam Zheng <famz@redhat.com>
958e1b
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
958e1b
(cherry picked from commit c6ac36e14569794b3f3e66f796dea19bc0f0b8d3)
958e1b
Signed-off-by: Fam Zheng <famz@redhat.com>
958e1b
Signed-off-by: Jeff E. Nelson <jen@redhat.com>
958e1b
---
958e1b
 block/vmdk.c | 222 +++++++++++++++++++++++++++++++++++++----------------------
958e1b
 1 file changed, 140 insertions(+), 82 deletions(-)
958e1b
958e1b
diff --git a/block/vmdk.c b/block/vmdk.c
958e1b
index a625e7a..70b616a 100644
958e1b
--- a/block/vmdk.c
958e1b
+++ b/block/vmdk.c
958e1b
@@ -106,6 +106,7 @@ typedef struct VmdkExtent {
958e1b
     uint32_t l2_cache_counts[L2_CACHE_SIZE];
958e1b
 
958e1b
     int64_t cluster_sectors;
958e1b
+    int64_t next_cluster_sector;
958e1b
     char *type;
958e1b
 } VmdkExtent;
958e1b
 
958e1b
@@ -124,7 +125,6 @@ typedef struct BDRVVmdkState {
958e1b
 } BDRVVmdkState;
958e1b
 
958e1b
 typedef struct VmdkMetaData {
958e1b
-    uint32_t offset;
958e1b
     unsigned int l1_index;
958e1b
     unsigned int l2_index;
958e1b
     unsigned int l2_offset;
958e1b
@@ -397,6 +397,7 @@ static int vmdk_add_extent(BlockDriverState *bs,
958e1b
 {
958e1b
     VmdkExtent *extent;
958e1b
     BDRVVmdkState *s = bs->opaque;
958e1b
+    int64_t length;
958e1b
 
958e1b
     if (cluster_sectors > 0x200000) {
958e1b
         /* 0x200000 * 512Bytes = 1GB for one cluster is unrealistic */
958e1b
@@ -412,6 +413,11 @@ static int vmdk_add_extent(BlockDriverState *bs,
958e1b
         return -EFBIG;
958e1b
     }
958e1b
 
958e1b
+    length = bdrv_getlength(file);
958e1b
+    if (length < 0) {
958e1b
+        return length;
958e1b
+    }
958e1b
+
958e1b
     s->extents = g_realloc(s->extents,
958e1b
                               (s->num_extents + 1) * sizeof(VmdkExtent));
958e1b
     extent = &s->extents[s->num_extents];
958e1b
@@ -427,6 +433,8 @@ static int vmdk_add_extent(BlockDriverState *bs,
958e1b
     extent->l1_entry_sectors = l2_size * cluster_sectors;
958e1b
     extent->l2_size = l2_size;
958e1b
     extent->cluster_sectors = flat ? sectors : cluster_sectors;
958e1b
+    extent->next_cluster_sector =
958e1b
+        ROUND_UP(DIV_ROUND_UP(length, BDRV_SECTOR_SIZE), cluster_sectors);
958e1b
 
958e1b
     if (s->num_extents > 1) {
958e1b
         extent->end_sector = (*(extent - 1)).end_sector + extent->sectors;
958e1b
@@ -953,57 +961,97 @@ static int vmdk_refresh_limits(BlockDriverState *bs)
958e1b
     return 0;
958e1b
 }
958e1b
 
958e1b
+/**
958e1b
+ * get_whole_cluster
958e1b
+ *
958e1b
+ * Copy backing file's cluster that covers @sector_num, otherwise write zero,
958e1b
+ * to the cluster at @cluster_sector_num.
958e1b
+ *
958e1b
+ * If @skip_start_sector < @skip_end_sector, the relative range
958e1b
+ * [@skip_start_sector, @skip_end_sector) is not copied or written, and leave
958e1b
+ * it for call to write user data in the request.
958e1b
+ */
958e1b
 static int get_whole_cluster(BlockDriverState *bs,
958e1b
-                VmdkExtent *extent,
958e1b
-                uint64_t cluster_offset,
958e1b
-                uint64_t offset,
958e1b
-                bool allocate)
958e1b
+                             VmdkExtent *extent,
958e1b
+                             uint64_t cluster_sector_num,
958e1b
+                             uint64_t sector_num,
958e1b
+                             uint64_t skip_start_sector,
958e1b
+                             uint64_t skip_end_sector)
958e1b
 {
958e1b
     int ret = VMDK_OK;
958e1b
-    uint8_t *whole_grain = NULL;
958e1b
+    int64_t cluster_bytes;
958e1b
+    uint8_t *whole_grain;
958e1b
 
958e1b
+    /* For COW, align request sector_num to cluster start */
958e1b
+    sector_num = QEMU_ALIGN_DOWN(sector_num, extent->cluster_sectors);
958e1b
+    cluster_bytes = extent->cluster_sectors << BDRV_SECTOR_BITS;
958e1b
+    whole_grain = qemu_blockalign(bs, cluster_bytes);
958e1b
+
958e1b
+    if (!bs->backing_hd) {
958e1b
+        memset(whole_grain, 0,  skip_start_sector << BDRV_SECTOR_BITS);
958e1b
+        memset(whole_grain + (skip_end_sector << BDRV_SECTOR_BITS), 0,
958e1b
+               cluster_bytes - (skip_end_sector << BDRV_SECTOR_BITS));
958e1b
+    }
958e1b
+
958e1b
+    assert(skip_end_sector <= extent->cluster_sectors);
958e1b
     /* we will be here if it's first write on non-exist grain(cluster).
958e1b
      * try to read from parent image, if exist */
958e1b
-    if (bs->backing_hd) {
958e1b
-        whole_grain =
958e1b
-            qemu_blockalign(bs, extent->cluster_sectors << BDRV_SECTOR_BITS);
958e1b
-        if (!vmdk_is_cid_valid(bs)) {
958e1b
-            ret = VMDK_ERROR;
958e1b
-            goto exit;
958e1b
-        }
958e1b
+    if (bs->backing_hd && !vmdk_is_cid_valid(bs)) {
958e1b
+        ret = VMDK_ERROR;
958e1b
+        goto exit;
958e1b
+    }
958e1b
 
958e1b
-        /* floor offset to cluster */
958e1b
-        offset -= offset % (extent->cluster_sectors * 512);
958e1b
-        ret = bdrv_read(bs->backing_hd, offset >> 9, whole_grain,
958e1b
-                extent->cluster_sectors);
958e1b
+    /* Read backing data before skip range */
958e1b
+    if (skip_start_sector > 0) {
958e1b
+        if (bs->backing_hd) {
958e1b
+            ret = bdrv_read(bs->backing_hd, sector_num,
958e1b
+                            whole_grain, skip_start_sector);
958e1b
+            if (ret < 0) {
958e1b
+                ret = VMDK_ERROR;
958e1b
+                goto exit;
958e1b
+            }
958e1b
+        }
958e1b
+        ret = bdrv_write(extent->file, cluster_sector_num, whole_grain,
958e1b
+                         skip_start_sector);
958e1b
         if (ret < 0) {
958e1b
             ret = VMDK_ERROR;
958e1b
             goto exit;
958e1b
         }
958e1b
-
958e1b
-        /* Write grain only into the active image */
958e1b
-        ret = bdrv_write(extent->file, cluster_offset, whole_grain,
958e1b
-                extent->cluster_sectors);
958e1b
+    }
958e1b
+    /* Read backing data after skip range */
958e1b
+    if (skip_end_sector < extent->cluster_sectors) {
958e1b
+        if (bs->backing_hd) {
958e1b
+            ret = bdrv_read(bs->backing_hd, sector_num + skip_end_sector,
958e1b
+                            whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
958e1b
+                            extent->cluster_sectors - skip_end_sector);
958e1b
+            if (ret < 0) {
958e1b
+                ret = VMDK_ERROR;
958e1b
+                goto exit;
958e1b
+            }
958e1b
+        }
958e1b
+        ret = bdrv_write(extent->file, cluster_sector_num + skip_end_sector,
958e1b
+                         whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
958e1b
+                         extent->cluster_sectors - skip_end_sector);
958e1b
         if (ret < 0) {
958e1b
             ret = VMDK_ERROR;
958e1b
             goto exit;
958e1b
         }
958e1b
     }
958e1b
+
958e1b
 exit:
958e1b
     qemu_vfree(whole_grain);
958e1b
     return ret;
958e1b
 }
958e1b
 
958e1b
-static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data)
958e1b
+static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data,
958e1b
+                         uint32_t offset)
958e1b
 {
958e1b
-    uint32_t offset;
958e1b
-    QEMU_BUILD_BUG_ON(sizeof(offset) != sizeof(m_data->offset));
958e1b
-    offset = cpu_to_le32(m_data->offset);
958e1b
+    offset = cpu_to_le32(offset);
958e1b
     /* update L2 table */
958e1b
     if (bdrv_pwrite_sync(
958e1b
                 extent->file,
958e1b
                 ((int64_t)m_data->l2_offset * 512)
958e1b
-                    + (m_data->l2_index * sizeof(m_data->offset)),
958e1b
+                    + (m_data->l2_index * sizeof(offset)),
958e1b
                 &offset, sizeof(offset)) < 0) {
958e1b
         return VMDK_ERROR;
958e1b
     }
958e1b
@@ -1013,7 +1061,7 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data)
958e1b
         if (bdrv_pwrite_sync(
958e1b
                     extent->file,
958e1b
                     ((int64_t)m_data->l2_offset * 512)
958e1b
-                        + (m_data->l2_index * sizeof(m_data->offset)),
958e1b
+                        + (m_data->l2_index * sizeof(offset)),
958e1b
                     &offset, sizeof(offset)) < 0) {
958e1b
             return VMDK_ERROR;
958e1b
         }
958e1b
@@ -1025,17 +1073,41 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data)
958e1b
     return VMDK_OK;
958e1b
 }
958e1b
 
958e1b
+/**
958e1b
+ * get_cluster_offset
958e1b
+ *
958e1b
+ * Look up cluster offset in extent file by sector number, and store in
958e1b
+ * @cluster_offset.
958e1b
+ *
958e1b
+ * For flat extents, the start offset as parsed from the description file is
958e1b
+ * returned.
958e1b
+ *
958e1b
+ * For sparse extents, look up in L1, L2 table. If allocate is true, return an
958e1b
+ * offset for a new cluster and update L2 cache. If there is a backing file,
958e1b
+ * COW is done before returning; otherwise, zeroes are written to the allocated
958e1b
+ * cluster. Both COW and zero writing skips the sector range
958e1b
+ * [@skip_start_sector, @skip_end_sector) passed in by caller, because caller
958e1b
+ * has new data to write there.
958e1b
+ *
958e1b
+ * Returns: VMDK_OK if cluster exists and mapped in the image.
958e1b
+ *          VMDK_UNALLOC if cluster is not mapped and @allocate is false.
958e1b
+ *          VMDK_ERROR if failed.
958e1b
+ */
958e1b
 static int get_cluster_offset(BlockDriverState *bs,
958e1b
-                                    VmdkExtent *extent,
958e1b
-                                    VmdkMetaData *m_data,
958e1b
-                                    uint64_t offset,
958e1b
-                                    int allocate,
958e1b
-                                    uint64_t *cluster_offset)
958e1b
+                              VmdkExtent *extent,
958e1b
+                              VmdkMetaData *m_data,
958e1b
+                              uint64_t offset,
958e1b
+                              bool allocate,
958e1b
+                              uint64_t *cluster_offset,
958e1b
+                              uint64_t skip_start_sector,
958e1b
+                              uint64_t skip_end_sector)
958e1b
 {
958e1b
     unsigned int l1_index, l2_offset, l2_index;
958e1b
     int min_index, i, j;
958e1b
     uint32_t min_count, *l2_table;
958e1b
     bool zeroed = false;
958e1b
+    int64_t ret;
958e1b
+    int32_t cluster_sector;
958e1b
 
958e1b
     if (m_data) {
958e1b
         m_data->valid = 0;
958e1b
@@ -1089,52 +1161,41 @@ static int get_cluster_offset(BlockDriverState *bs,
958e1b
     extent->l2_cache_counts[min_index] = 1;
958e1b
  found:
958e1b
     l2_index = ((offset >> 9) / extent->cluster_sectors) % extent->l2_size;
958e1b
-    *cluster_offset = le32_to_cpu(l2_table[l2_index]);
958e1b
+    cluster_sector = le32_to_cpu(l2_table[l2_index]);
958e1b
 
958e1b
     if (m_data) {
958e1b
         m_data->valid = 1;
958e1b
         m_data->l1_index = l1_index;
958e1b
         m_data->l2_index = l2_index;
958e1b
-        m_data->offset = *cluster_offset;
958e1b
         m_data->l2_offset = l2_offset;
958e1b
         m_data->l2_cache_entry = &l2_table[l2_index];
958e1b
     }
958e1b
-    if (extent->has_zero_grain && *cluster_offset == VMDK_GTE_ZEROED) {
958e1b
+    if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) {
958e1b
         zeroed = true;
958e1b
     }
958e1b
 
958e1b
-    if (!*cluster_offset || zeroed) {
958e1b
+    if (!cluster_sector || zeroed) {
958e1b
         if (!allocate) {
958e1b
             return zeroed ? VMDK_ZEROED : VMDK_UNALLOC;
958e1b
         }
958e1b
 
958e1b
-        /* Avoid the L2 tables update for the images that have snapshots. */
958e1b
-        *cluster_offset = bdrv_getlength(extent->file);
958e1b
-        if (!extent->compressed) {
958e1b
-            bdrv_truncate(
958e1b
-                extent->file,
958e1b
-                *cluster_offset + (extent->cluster_sectors << 9)
958e1b
-            );
958e1b
-        }
958e1b
-
958e1b
-        *cluster_offset >>= 9;
958e1b
-        l2_table[l2_index] = cpu_to_le32(*cluster_offset);
958e1b
+        cluster_sector = extent->next_cluster_sector;
958e1b
+        extent->next_cluster_sector += extent->cluster_sectors;
958e1b
 
958e1b
         /* First of all we write grain itself, to avoid race condition
958e1b
          * that may to corrupt the image.
958e1b
          * This problem may occur because of insufficient space on host disk
958e1b
          * or inappropriate VM shutdown.
958e1b
          */
958e1b
-        if (get_whole_cluster(
958e1b
-                bs, extent, *cluster_offset, offset, allocate) == -1) {
958e1b
-            return VMDK_ERROR;
958e1b
-        }
958e1b
-
958e1b
-        if (m_data) {
958e1b
-            m_data->offset = *cluster_offset;
958e1b
+        ret = get_whole_cluster(bs, extent,
958e1b
+                                cluster_sector,
958e1b
+                                offset >> BDRV_SECTOR_BITS,
958e1b
+                                skip_start_sector, skip_end_sector);
958e1b
+        if (ret) {
958e1b
+            return ret;
958e1b
         }
958e1b
     }
958e1b
-    *cluster_offset <<= 9;
958e1b
+    *cluster_offset = cluster_sector << BDRV_SECTOR_BITS;
958e1b
     return VMDK_OK;
958e1b
 }
958e1b
 
958e1b
@@ -1169,7 +1230,8 @@ static int64_t coroutine_fn vmdk_co_get_block_status(BlockDriverState *bs,
958e1b
     }
958e1b
     qemu_co_mutex_lock(&s->lock);
958e1b
     ret = get_cluster_offset(bs, extent, NULL,
958e1b
-                            sector_num * 512, 0, &offset);
958e1b
+                             sector_num * 512, false, &offset,
958e1b
+                             0, 0);
958e1b
     qemu_co_mutex_unlock(&s->lock);
958e1b
 
958e1b
     switch (ret) {
958e1b
@@ -1322,9 +1384,9 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
958e1b
         if (!extent) {
958e1b
             return -EIO;
958e1b
         }
958e1b
-        ret = get_cluster_offset(
958e1b
-                            bs, extent, NULL,
958e1b
-                            sector_num << 9, 0, &cluster_offset);
958e1b
+        ret = get_cluster_offset(bs, extent, NULL,
958e1b
+                                 sector_num << 9, false, &cluster_offset,
958e1b
+                                 0, 0);
958e1b
         extent_begin_sector = extent->end_sector - extent->sectors;
958e1b
         extent_relative_sector_num = sector_num - extent_begin_sector;
958e1b
         index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
958e1b
@@ -1405,12 +1467,17 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
958e1b
         if (!extent) {
958e1b
             return -EIO;
958e1b
         }
958e1b
-        ret = get_cluster_offset(
958e1b
-                                bs,
958e1b
-                                extent,
958e1b
-                                &m_data,
958e1b
-                                sector_num << 9, !extent->compressed,
958e1b
-                                &cluster_offset);
958e1b
+        extent_begin_sector = extent->end_sector - extent->sectors;
958e1b
+        extent_relative_sector_num = sector_num - extent_begin_sector;
958e1b
+        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
958e1b
+        n = extent->cluster_sectors - index_in_cluster;
958e1b
+        if (n > nb_sectors) {
958e1b
+            n = nb_sectors;
958e1b
+        }
958e1b
+        ret = get_cluster_offset(bs, extent, &m_data, sector_num << 9,
958e1b
+                                 !(extent->compressed || zeroed),
958e1b
+                                 &cluster_offset,
958e1b
+                                 index_in_cluster, index_in_cluster + n);
958e1b
         if (extent->compressed) {
958e1b
             if (ret == VMDK_OK) {
958e1b
                 /* Refuse write to allocated cluster for streamOptimized */
958e1b
@@ -1419,24 +1486,13 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
958e1b
                 return -EIO;
958e1b
             } else {
958e1b
                 /* allocate */
958e1b
-                ret = get_cluster_offset(
958e1b
-                                        bs,
958e1b
-                                        extent,
958e1b
-                                        &m_data,
958e1b
-                                        sector_num << 9, 1,
958e1b
-                                        &cluster_offset);
958e1b
+                ret = get_cluster_offset(bs, extent, &m_data, sector_num << 9,
958e1b
+                                         true, &cluster_offset, 0, 0);
958e1b
             }
958e1b
         }
958e1b
         if (ret == VMDK_ERROR) {
958e1b
             return -EINVAL;
958e1b
         }
958e1b
-        extent_begin_sector = extent->end_sector - extent->sectors;
958e1b
-        extent_relative_sector_num = sector_num - extent_begin_sector;
958e1b
-        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
958e1b
-        n = extent->cluster_sectors - index_in_cluster;
958e1b
-        if (n > nb_sectors) {
958e1b
-            n = nb_sectors;
958e1b
-        }
958e1b
         if (zeroed) {
958e1b
             /* Do zeroed write, buf is ignored */
958e1b
             if (extent->has_zero_grain &&
958e1b
@@ -1444,9 +1500,9 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
958e1b
                     n >= extent->cluster_sectors) {
958e1b
                 n = extent->cluster_sectors;
958e1b
                 if (!zero_dry_run) {
958e1b
-                    m_data.offset = VMDK_GTE_ZEROED;
958e1b
                     /* update L2 tables */
958e1b
-                    if (vmdk_L2update(extent, &m_data) != VMDK_OK) {
958e1b
+                    if (vmdk_L2update(extent, &m_data, VMDK_GTE_ZEROED)
958e1b
+                            != VMDK_OK) {
958e1b
                         return -EIO;
958e1b
                     }
958e1b
                 }
958e1b
@@ -1462,7 +1518,9 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
958e1b
             }
958e1b
             if (m_data.valid) {
958e1b
                 /* update L2 tables */
958e1b
-                if (vmdk_L2update(extent, &m_data) != VMDK_OK) {
958e1b
+                if (vmdk_L2update(extent, &m_data,
958e1b
+                                  cluster_offset >> BDRV_SECTOR_BITS)
958e1b
+                        != VMDK_OK) {
958e1b
                     return -EIO;
958e1b
                 }
958e1b
             }
958e1b
@@ -2020,7 +2078,7 @@ static int vmdk_check(BlockDriverState *bs, BdrvCheckResult *result,
958e1b
         }
958e1b
         ret = get_cluster_offset(bs, extent, NULL,
958e1b
                                  sector_num << BDRV_SECTOR_BITS,
958e1b
-                                 0, &cluster_offset);
958e1b
+                                 false, &cluster_offset, 0, 0);
958e1b
         if (ret == VMDK_ERROR) {
958e1b
             fprintf(stderr,
958e1b
                     "ERROR: could not get cluster_offset for sector %"
958e1b
-- 
958e1b
2.1.0
958e1b