From 91cfe700849856245f58a8c0ee98c0fd1e9d47f6 Mon Sep 17 00:00:00 2001

From: Kotresh HR <khiremat@redhat.com>

Date: Mon, 28 May 2018 03:05:26 -0400

Subject: [PATCH 304/305] cluster/dht: Fix rename journal in changelog

With patch [1], renames are journalled only

on cached subvolume. The dht sends the special

key on the cached subvolume so that the changelog

journals the rename. With single distribute

sub-volume, the key is not being set. This patch

fixes the same.

[1] https://review.gluster.org/10410

Backport of:

 > Patch: https://review.gluster.org/20093/

 > fixes: bz#1583018

 > Change-Id: Ic2e35b40535916fa506a714f257ba325e22d0961

 > Signed-off-by: Kotresh HR <khiremat@redhat.com>

BUG: 1583047

Change-Id: Ic2e35b40535916fa506a714f257ba325e22d0961

Signed-off-by: Kotresh HR <khiremat@redhat.com>

Reviewed-on: https://code.engineering.redhat.com/gerrit/142601

Tested-by: RHGS Build Bot <nigelb@redhat.com>

Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

---

 tests/basic/changelog/changelog-rename.t |  44 ++++++

 tests/utils/changelogparser.py           | 234 +++++++++++++++++++++++++++++++

 tests/volume.rc                          |   7 +

 xlators/cluster/dht/src/dht-rename.c     |  11 ++

 4 files changed, 296 insertions(+)

 create mode 100644 tests/basic/changelog/changelog-rename.t

 create mode 100644 tests/utils/changelogparser.py

diff --git a/tests/basic/changelog/changelog-rename.t b/tests/basic/changelog/changelog-rename.t

new file mode 100644

index 0000000..9a0ef52

--- /dev/null

+++ b/tests/basic/changelog/changelog-rename.t

@@ -0,0 +1,44 @@

+#!/bin/bash

+. $(dirname $0)/../../include.rc

+. $(dirname $0)/../../volume.rc

+cleanup;

+

+CHANGELOG_PATH_0="$B0/${V0}0/.glusterfs/changelogs"

+ROLLOVER_TIME=30

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}0

+TEST $CLI volume set $V0 changelog.changelog on

+TEST $CLI volume set $V0 changelog.rollover-time $ROLLOVER_TIME

+TEST $CLI volume start $V0

+

+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0;

+touch $M0/file1

+mv $M0/file1 $M0/rn_file1

+mkdir $M0/dir1

+mv $M0/dir1 $M0/rn_dir1

+

+EXPECT "2" check_changelog_op ${CHANGELOG_PATH_0} "RENAME"

+

+cleanup;

+

+#####Test on multiple subvolume#####

+#==========================================#

+

+TEST glusterd

+TEST pidof glusterd

+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1}

+TEST $CLI volume set $V0 changelog.changelog on

+TEST $CLI volume set $V0 changelog.rollover-time $ROLLOVER_TIME

+TEST $CLI volume start $V0

+

+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0;

+touch $M0/gluster_file

+mv $M0/gluster_file $M0/rn_gluster_file

+mkdir $M0/dir1

+mv $M0/dir1 $M0/rn_dir1

+

+EXPECT "2" check_changelog_op ${CHANGELOG_PATH_0} "RENAME"

+

+cleanup;

diff --git a/tests/utils/changelogparser.py b/tests/utils/changelogparser.py

new file mode 100644

index 0000000..e173e52

--- /dev/null

+++ b/tests/utils/changelogparser.py

@@ -0,0 +1,234 @@

+#!/usr/bin/env python

+# -*- coding: utf-8 -*-

+"""

+Why?

+

+Converts this

+

+GlusterFS Changelog | version: v1.1 | encoding : 2

+E0b99ef11-4b79-4cd0-9730-b5a0e8c4a8c0^@4^@16877^@0^@0^@00000000-0000-0000-0000-

+000000000001/dir1^@Ec5250af6-720e-4bfe-b938-827614304f39^@23^@33188^@0^@0^@0b99

+ef11-4b79-4cd0-9730-b5a0e8c4a8c0/hello.txt^@Dc5250af6-720e-4bfe-b938-827614304f

+39^@Dc5250af6-720e-4bfe-b938-827614304f39^@

+

+

+to human readable :)

+

+E 0b99ef11-4b79-4cd0-9730-b5a0e8c4a8c0 MKDIR 16877 0 000000000-0000-0000-0000

+  -000000000001/dir1

+E c5250af6-720e-4bfe-b938-827614304f39 CREATE 33188 0 0 0b99ef11-4b79-4cd0-9730

+  -b5a0e8c4a8c0/hello.txt

+D c5250af6-720e-4bfe-b938-827614304f39

+D c5250af6-720e-4bfe-b938-827614304f39

+

+

+"""

+import sys

+import codecs

+

+ENTRY = 'E'

+META = 'M'

+DATA = 'D'

+SEP = "\x00"

+

+GF_FOP = [

+    "NULL", "STAT", "READLINK", "MKNOD", "MKDIR", "UNLINK",

+    "RMDIR", "SYMLINK", "RENAME", "LINK", "TRUNCATE", "OPEN",

+    "READ", "WRITE", "STATFS", "FLUSH", "FSYNC", "SETXATTR",

+    "GETXATTR", "REMOVEXATTR", "OPENDIR", "FSYNCDIR", "ACCESS",

+    "CREATE", "FTRUNCATE", "FSTAT", "LK", "LOOKUP", "READDIR",

+    "INODELK", "FINODELK", "ENTRYLK", "FENTRYLK", "XATTROP",

+    "FXATTROP", "FSETXATTR", "FGETXATTR", "RCHECKSUM", "SETATTR",

+    "FSETATTR", "READDIRP", "GETSPEC", "FORGET", "RELEASE",

+    "RELEASEDIR", "FREMOVEXATTR", "FALLOCATE", "DISCARD", "ZEROFILL"]

+

+

+class NumTokens_V11(object):

+    E = 7

+    M = 3

+    D = 2

+    NULL = 3

+    MKNOD = 7

+    MKDIR = 7

+    UNLINK = 4

+    RMDIR = 4

+    SYMLINK = 4

+    RENAME = 5

+    LINK = 4

+    SETXATTR = 3

+    REMOVEXATTR = 3

+    CREATE = 7

+    SETATTR = 3

+    FTRUNCATE = 3

+    FXATTROP = 3

+

+

+class NumTokens_V12(NumTokens_V11):

+    UNLINK = 5

+    RMDIR = 5

+

+

+class Version:

+    V11 = "v1.1"

+    V12 = "v1.2"

+

+

+class Record(object):

+    def __init__(self, **kwargs):

+        self.ts = kwargs.get("ts", None)

+        self.fop_type = kwargs.get("fop_type", None)

+        self.gfid = kwargs.get("gfid", None)

+        self.path = kwargs.get("path", None)

+        self.fop = kwargs.get("fop", None)

+        self.path1 = kwargs.get("path1", None)

+        self.path2 = kwargs.get("path2", None)

+        self.mode = kwargs.get("mode", None)

+        self.uid = kwargs.get("uid", None)

+        self.gid = kwargs.get("gid", None)

+

+    def create_mknod_mkdir(self, **kwargs):

+        self.path = kwargs.get("path", None)

+        self.fop = kwargs.get("fop", None)

+        self.mode = kwargs.get("mode", None)

+        self.uid = kwargs.get("uid", None)

+        self.gid = kwargs.get("gid", None)

+

+    def metadata(self, **kwargs):

+        self.fop = kwargs.get("fop", None)

+

+    def rename(self, **kwargs):

+        self.fop = kwargs.get("fop", None)

+        self.path1 = kwargs.get("path1", None)

+        self.path2 = kwargs.get("path2", None)

+

+    def link_symlink_unlink_rmdir(self, **kwargs):

+        self.path = kwargs.get("path", None)

+        self.fop = kwargs.get("fop", None)

+

+    def __unicode__(self):

+        if self.fop_type == "D":

+            return u"{ts} {fop_type} {gfid}".format(**self.__dict__)

+        elif self.fop_type == "M":

+            return u"{ts} {fop_type} {gfid} {fop}".format(**self.__dict__)

+        elif self.fop_type == "E":

+            if self.fop in ["CREATE", "MKNOD", "MKDIR"]:

+                return (u"{ts} {fop_type} {gfid} {fop} "

+                        u"{path} {mode} {uid} {gid}".format(**self.__dict__))

+            elif self.fop == "RENAME":

+                return (u"{ts} {fop_type} {gfid} {fop} "

+                        u"{path1} {path2}".format(**self.__dict__))

+            elif self.fop in ["LINK", "SYMLINK", "UNLINK", "RMDIR"]:

+                return (u"{ts} {fop_type} {gfid} {fop} "

+                        u"{path}".format(**self.__dict__))

+            else:

+                return repr(self.__dict__)

+        else:

+            return repr(self.__dict__)

+

+    def __str__(self):

+        return unicode(self).encode('utf-8')

+

+

+def get_num_tokens(data, tokens, version=Version.V11):

+    if version == Version.V11:

+        cls_numtokens = NumTokens_V11

+    elif version == Version.V12:

+        cls_numtokens = NumTokens_V12

+    else:

+        sys.stderr.write("Unknown Changelog Version\n")

+        sys.exit(1)

+

+    if data[tokens[0]] in [ENTRY, META]:

+        if len(tokens) >= 3:

+            return getattr(cls_numtokens, GF_FOP[int(data[tokens[2]])])

+        else:

+            return None

+    else:

+        return getattr(cls_numtokens, data[tokens[0]])

+

+

+def process_record(data, tokens, changelog_ts, callback):

+    if data[tokens[0]] in [ENTRY, META]:

+        try:

+            tokens[2] = GF_FOP[int(data[tokens[2]])]

+        except ValueError:

+            tokens[2] = "NULL"

+

+    if not changelog_ts:

+        ts1 = int(changelog_ts)

+    else:

+        ts1=""

+    record = Record(ts=ts1, fop_type=data[tokens[0]],

+                    gfid=data[tokens[1]])

+    if data[tokens[0]] == META:

+        record.metadata(fop=tokens[2])

+    elif data[tokens[0]] == ENTRY:

+        if tokens[2] in ["CREATE", "MKNOD", "MKDIR"]:

+            record.create_mknod_mkdir(fop=tokens[2],

+                                      path=data[tokens[6]],

+                                      mode=int(data[tokens[3]]),

+                                      uid=int(data[tokens[4]]),

+                                      gid=int(data[tokens[5]]))

+        elif tokens[2] == "RENAME":

+            record.rename(fop=tokens[2],

+                          path1=data[tokens[3]],

+                          path2=data[tokens[4]])

+        if tokens[2] in ["LINK", "SYMLINK", "UNLINK", "RMDIR"]:

+            record.link_symlink_unlink_rmdir(fop=tokens[2],

+                                             path=data[tokens[3]])

+    callback(record)

+

+

+def default_callback(record):

+    sys.stdout.write(u"{0}\n".format(record))

+

+

+def parse(filename, callback=default_callback):

+    data = None

+    tokens = []

+    changelog_ts = filename.rsplit(".")[-1]

+    with codecs.open(filename, mode="rb", encoding="utf-8") as f:

+        # GlusterFS Changelog | version: v1.1 | encoding : 2

+        header = f.readline()

+        version = header.split()[4]

+

+        data = f.readline()

+

+        slice_start = 0

+        in_record = False

+

+        prev_char = ""

+        next_char = ""

+        for i, c in enumerate(data):

+            next_char = ""

+            if len(data) >= (i + 2):

+                next_char = data[i+1]

+

+            if not in_record and c in [ENTRY, META, DATA]:

+                tokens.append(slice(slice_start, i+1))

+                slice_start = i+1

+                in_record = True

+                continue

+

+            if c == SEP and ((prev_char != SEP and next_char == SEP) or

+                             (prev_char == SEP and next_char != SEP) or

+                             (prev_char != SEP and next_char != SEP)):

+                tokens.append(slice(slice_start, i))

+                slice_start = i+1

+

+                num_tokens = get_num_tokens(data, tokens, version)

+

+                if num_tokens == len(tokens):

+                    process_record(data, tokens, changelog_ts, callback)

+                    in_record = False

+                    tokens = []

+

+            prev_char = c

+

+        # process last record

+        if slice_start < (len(data) - 1):

+            tokens.append(slice(slice_start, len(data)))

+            process_record(data, tokens, changelog_ts, callback)

+            tokens = []

+

+parse(sys.argv[1])

diff --git a/tests/volume.rc b/tests/volume.rc

index f9e16c5..bba7e4e 100644

--- a/tests/volume.rc

+++ b/tests/volume.rc

@@ -865,3 +865,10 @@ function get_mount_lru_size_value {

         rm -f $statedump

         echo $val

 }

+

+function check_changelog_op {

+        local clog_path=$1

+        local op=$2

+

+        $PYTHON $(dirname $0)/../../utils/changelogparser.py ${clog_path}/CHANGELOG | grep $op | wc -l

+}

diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c

index d311ac6..1d0c2bb 100644

--- a/xlators/cluster/dht/src/dht-rename.c

+++ b/xlators/cluster/dht/src/dht-rename.c

@@ -1948,6 +1948,7 @@ dht_rename (call_frame_t *frame, xlator_t *this,

         dht_conf_t  *conf                   = NULL;

         char         gfid[GF_UUID_BUF_SIZE] = {0};

         char         newgfid[GF_UUID_BUF_SIZE] = {0};

+        gf_boolean_t free_xdata             = _gf_false;

         VALIDATE_OR_GOTO (frame, err);

         VALIDATE_OR_GOTO (this, err);

@@ -1957,7 +1958,17 @@ dht_rename (call_frame_t *frame, xlator_t *this,

         conf = this->private;

         if (conf->subvolume_cnt == 1) {

+                if (!IA_ISDIR (oldloc->inode->ia_type)) {

+                        if (!xdata) {

+                                free_xdata = _gf_true;

+                        }

+                        DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);

+                }

                 default_rename (frame, this, oldloc, newloc, xdata);

+                if (free_xdata && xdata) {

+                        dict_unref(xdata);

+                        xdata = NULL;

+                }

                 return 0;

         }

-- 

1.8.3.1