From 91cfe700849856245f58a8c0ee98c0fd1e9d47f6 Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Mon, 28 May 2018 03:05:26 -0400
Subject: [PATCH 304/305] cluster/dht: Fix rename journal in changelog
With patch [1], renames are journalled only
on cached subvolume. The dht sends the special
key on the cached subvolume so that the changelog
journals the rename. With single distribute
sub-volume, the key is not being set. This patch
fixes the same.
[1] https://review.gluster.org/10410
Backport of:
> Patch: https://review.gluster.org/20093/
> fixes: bz#1583018
> Change-Id: Ic2e35b40535916fa506a714f257ba325e22d0961
> Signed-off-by: Kotresh HR <khiremat@redhat.com>
BUG: 1583047
Change-Id: Ic2e35b40535916fa506a714f257ba325e22d0961
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/142601
Tested-by: RHGS Build Bot <nigelb@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
tests/basic/changelog/changelog-rename.t | 44 ++++++
tests/utils/changelogparser.py | 234 +++++++++++++++++++++++++++++++
tests/volume.rc | 7 +
xlators/cluster/dht/src/dht-rename.c | 11 ++
4 files changed, 296 insertions(+)
create mode 100644 tests/basic/changelog/changelog-rename.t
create mode 100644 tests/utils/changelogparser.py
diff --git a/tests/basic/changelog/changelog-rename.t b/tests/basic/changelog/changelog-rename.t
new file mode 100644
index 0000000..9a0ef52
--- /dev/null
+++ b/tests/basic/changelog/changelog-rename.t
@@ -0,0 +1,44 @@
+#!/bin/bash
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+cleanup;
+
+CHANGELOG_PATH_0="$B0/${V0}0/.glusterfs/changelogs"
+ROLLOVER_TIME=30
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}0
+TEST $CLI volume set $V0 changelog.changelog on
+TEST $CLI volume set $V0 changelog.rollover-time $ROLLOVER_TIME
+TEST $CLI volume start $V0
+
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0;
+touch $M0/file1
+mv $M0/file1 $M0/rn_file1
+mkdir $M0/dir1
+mv $M0/dir1 $M0/rn_dir1
+
+EXPECT "2" check_changelog_op ${CHANGELOG_PATH_0} "RENAME"
+
+cleanup;
+
+#####Test on multiple subvolume#####
+#==========================================#
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1}
+TEST $CLI volume set $V0 changelog.changelog on
+TEST $CLI volume set $V0 changelog.rollover-time $ROLLOVER_TIME
+TEST $CLI volume start $V0
+
+TEST $GFS --volfile-id=$V0 --volfile-server=$H0 $M0;
+touch $M0/gluster_file
+mv $M0/gluster_file $M0/rn_gluster_file
+mkdir $M0/dir1
+mv $M0/dir1 $M0/rn_dir1
+
+EXPECT "2" check_changelog_op ${CHANGELOG_PATH_0} "RENAME"
+
+cleanup;
diff --git a/tests/utils/changelogparser.py b/tests/utils/changelogparser.py
new file mode 100644
index 0000000..e173e52
--- /dev/null
+++ b/tests/utils/changelogparser.py
@@ -0,0 +1,234 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Why?
+
+Converts this
+
+GlusterFS Changelog | version: v1.1 | encoding : 2
+E0b99ef11-4b79-4cd0-9730-b5a0e8c4a8c0^@4^@16877^@0^@0^@00000000-0000-0000-0000-
+000000000001/dir1^@Ec5250af6-720e-4bfe-b938-827614304f39^@23^@33188^@0^@0^@0b99
+ef11-4b79-4cd0-9730-b5a0e8c4a8c0/hello.txt^@Dc5250af6-720e-4bfe-b938-827614304f
+39^@Dc5250af6-720e-4bfe-b938-827614304f39^@
+
+
+to human readable :)
+
+E 0b99ef11-4b79-4cd0-9730-b5a0e8c4a8c0 MKDIR 16877 0 000000000-0000-0000-0000
+ -000000000001/dir1
+E c5250af6-720e-4bfe-b938-827614304f39 CREATE 33188 0 0 0b99ef11-4b79-4cd0-9730
+ -b5a0e8c4a8c0/hello.txt
+D c5250af6-720e-4bfe-b938-827614304f39
+D c5250af6-720e-4bfe-b938-827614304f39
+
+
+"""
+import sys
+import codecs
+
+ENTRY = 'E'
+META = 'M'
+DATA = 'D'
+SEP = "\x00"
+
+GF_FOP = [
+ "NULL", "STAT", "READLINK", "MKNOD", "MKDIR", "UNLINK",
+ "RMDIR", "SYMLINK", "RENAME", "LINK", "TRUNCATE", "OPEN",
+ "READ", "WRITE", "STATFS", "FLUSH", "FSYNC", "SETXATTR",
+ "GETXATTR", "REMOVEXATTR", "OPENDIR", "FSYNCDIR", "ACCESS",
+ "CREATE", "FTRUNCATE", "FSTAT", "LK", "LOOKUP", "READDIR",
+ "INODELK", "FINODELK", "ENTRYLK", "FENTRYLK", "XATTROP",
+ "FXATTROP", "FSETXATTR", "FGETXATTR", "RCHECKSUM", "SETATTR",
+ "FSETATTR", "READDIRP", "GETSPEC", "FORGET", "RELEASE",
+ "RELEASEDIR", "FREMOVEXATTR", "FALLOCATE", "DISCARD", "ZEROFILL"]
+
+
+class NumTokens_V11(object):
+ E = 7
+ M = 3
+ D = 2
+ NULL = 3
+ MKNOD = 7
+ MKDIR = 7
+ UNLINK = 4
+ RMDIR = 4
+ SYMLINK = 4
+ RENAME = 5
+ LINK = 4
+ SETXATTR = 3
+ REMOVEXATTR = 3
+ CREATE = 7
+ SETATTR = 3
+ FTRUNCATE = 3
+ FXATTROP = 3
+
+
+class NumTokens_V12(NumTokens_V11):
+ UNLINK = 5
+ RMDIR = 5
+
+
+class Version:
+ V11 = "v1.1"
+ V12 = "v1.2"
+
+
+class Record(object):
+ def __init__(self, **kwargs):
+ self.ts = kwargs.get("ts", None)
+ self.fop_type = kwargs.get("fop_type", None)
+ self.gfid = kwargs.get("gfid", None)
+ self.path = kwargs.get("path", None)
+ self.fop = kwargs.get("fop", None)
+ self.path1 = kwargs.get("path1", None)
+ self.path2 = kwargs.get("path2", None)
+ self.mode = kwargs.get("mode", None)
+ self.uid = kwargs.get("uid", None)
+ self.gid = kwargs.get("gid", None)
+
+ def create_mknod_mkdir(self, **kwargs):
+ self.path = kwargs.get("path", None)
+ self.fop = kwargs.get("fop", None)
+ self.mode = kwargs.get("mode", None)
+ self.uid = kwargs.get("uid", None)
+ self.gid = kwargs.get("gid", None)
+
+ def metadata(self, **kwargs):
+ self.fop = kwargs.get("fop", None)
+
+ def rename(self, **kwargs):
+ self.fop = kwargs.get("fop", None)
+ self.path1 = kwargs.get("path1", None)
+ self.path2 = kwargs.get("path2", None)
+
+ def link_symlink_unlink_rmdir(self, **kwargs):
+ self.path = kwargs.get("path", None)
+ self.fop = kwargs.get("fop", None)
+
+ def __unicode__(self):
+ if self.fop_type == "D":
+ return u"{ts} {fop_type} {gfid}".format(**self.__dict__)
+ elif self.fop_type == "M":
+ return u"{ts} {fop_type} {gfid} {fop}".format(**self.__dict__)
+ elif self.fop_type == "E":
+ if self.fop in ["CREATE", "MKNOD", "MKDIR"]:
+ return (u"{ts} {fop_type} {gfid} {fop} "
+ u"{path} {mode} {uid} {gid}".format(**self.__dict__))
+ elif self.fop == "RENAME":
+ return (u"{ts} {fop_type} {gfid} {fop} "
+ u"{path1} {path2}".format(**self.__dict__))
+ elif self.fop in ["LINK", "SYMLINK", "UNLINK", "RMDIR"]:
+ return (u"{ts} {fop_type} {gfid} {fop} "
+ u"{path}".format(**self.__dict__))
+ else:
+ return repr(self.__dict__)
+ else:
+ return repr(self.__dict__)
+
+ def __str__(self):
+ return unicode(self).encode('utf-8')
+
+
+def get_num_tokens(data, tokens, version=Version.V11):
+ if version == Version.V11:
+ cls_numtokens = NumTokens_V11
+ elif version == Version.V12:
+ cls_numtokens = NumTokens_V12
+ else:
+ sys.stderr.write("Unknown Changelog Version\n")
+ sys.exit(1)
+
+ if data[tokens[0]] in [ENTRY, META]:
+ if len(tokens) >= 3:
+ return getattr(cls_numtokens, GF_FOP[int(data[tokens[2]])])
+ else:
+ return None
+ else:
+ return getattr(cls_numtokens, data[tokens[0]])
+
+
+def process_record(data, tokens, changelog_ts, callback):
+ if data[tokens[0]] in [ENTRY, META]:
+ try:
+ tokens[2] = GF_FOP[int(data[tokens[2]])]
+ except ValueError:
+ tokens[2] = "NULL"
+
+ if not changelog_ts:
+ ts1 = int(changelog_ts)
+ else:
+ ts1=""
+ record = Record(ts=ts1, fop_type=data[tokens[0]],
+ gfid=data[tokens[1]])
+ if data[tokens[0]] == META:
+ record.metadata(fop=tokens[2])
+ elif data[tokens[0]] == ENTRY:
+ if tokens[2] in ["CREATE", "MKNOD", "MKDIR"]:
+ record.create_mknod_mkdir(fop=tokens[2],
+ path=data[tokens[6]],
+ mode=int(data[tokens[3]]),
+ uid=int(data[tokens[4]]),
+ gid=int(data[tokens[5]]))
+ elif tokens[2] == "RENAME":
+ record.rename(fop=tokens[2],
+ path1=data[tokens[3]],
+ path2=data[tokens[4]])
+ if tokens[2] in ["LINK", "SYMLINK", "UNLINK", "RMDIR"]:
+ record.link_symlink_unlink_rmdir(fop=tokens[2],
+ path=data[tokens[3]])
+ callback(record)
+
+
+def default_callback(record):
+ sys.stdout.write(u"{0}\n".format(record))
+
+
+def parse(filename, callback=default_callback):
+ data = None
+ tokens = []
+ changelog_ts = filename.rsplit(".")[-1]
+ with codecs.open(filename, mode="rb", encoding="utf-8") as f:
+ # GlusterFS Changelog | version: v1.1 | encoding : 2
+ header = f.readline()
+ version = header.split()[4]
+
+ data = f.readline()
+
+ slice_start = 0
+ in_record = False
+
+ prev_char = ""
+ next_char = ""
+ for i, c in enumerate(data):
+ next_char = ""
+ if len(data) >= (i + 2):
+ next_char = data[i+1]
+
+ if not in_record and c in [ENTRY, META, DATA]:
+ tokens.append(slice(slice_start, i+1))
+ slice_start = i+1
+ in_record = True
+ continue
+
+ if c == SEP and ((prev_char != SEP and next_char == SEP) or
+ (prev_char == SEP and next_char != SEP) or
+ (prev_char != SEP and next_char != SEP)):
+ tokens.append(slice(slice_start, i))
+ slice_start = i+1
+
+ num_tokens = get_num_tokens(data, tokens, version)
+
+ if num_tokens == len(tokens):
+ process_record(data, tokens, changelog_ts, callback)
+ in_record = False
+ tokens = []
+
+ prev_char = c
+
+ # process last record
+ if slice_start < (len(data) - 1):
+ tokens.append(slice(slice_start, len(data)))
+ process_record(data, tokens, changelog_ts, callback)
+ tokens = []
+
+parse(sys.argv[1])
diff --git a/tests/volume.rc b/tests/volume.rc
index f9e16c5..bba7e4e 100644
--- a/tests/volume.rc
+++ b/tests/volume.rc
@@ -865,3 +865,10 @@ function get_mount_lru_size_value {
rm -f $statedump
echo $val
}
+
+function check_changelog_op {
+ local clog_path=$1
+ local op=$2
+
+ $PYTHON $(dirname $0)/../../utils/changelogparser.py ${clog_path}/CHANGELOG | grep $op | wc -l
+}
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index d311ac6..1d0c2bb 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -1948,6 +1948,7 @@ dht_rename (call_frame_t *frame, xlator_t *this,
dht_conf_t *conf = NULL;
char gfid[GF_UUID_BUF_SIZE] = {0};
char newgfid[GF_UUID_BUF_SIZE] = {0};
+ gf_boolean_t free_xdata = _gf_false;
VALIDATE_OR_GOTO (frame, err);
VALIDATE_OR_GOTO (this, err);
@@ -1957,7 +1958,17 @@ dht_rename (call_frame_t *frame, xlator_t *this,
conf = this->private;
if (conf->subvolume_cnt == 1) {
+ if (!IA_ISDIR (oldloc->inode->ia_type)) {
+ if (!xdata) {
+ free_xdata = _gf_true;
+ }
+ DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
+ }
default_rename (frame, this, oldloc, newloc, xdata);
+ if (free_xdata && xdata) {
+ dict_unref(xdata);
+ xdata = NULL;
+ }
return 0;
}
--
1.8.3.1