From 70a866b408cff38f60293b729e715aa56789326f Mon Sep 17 00:00:00 2001

From: Aravinda VK <avishwan@redhat.com>

Date: Mon, 21 Mar 2016 16:57:48 +0530

Subject: [PATCH 64/80] tools/glusterfind: Handling Unicode file names

Unicode filenames handled cleanly with this patch. Changelog

files and output files are opened with utf-8 encoding using codecs.open.

urllib.quote_plus and unquote_plus will not handle Unicode so, encode

Unicode to 8-bit string version before calling unquote. urllib.quote_plus

requires 8-bit string itself so do not decode to Unicode if we need to use

quote_plus(when --no-encode=false). Decode to unicode in --no-encode is set.

BUG: 1309437

Change-Id: If5561c749ab5529445650d322c831eb4da22b65a

Signed-off-by: Aravinda VK <avishwan@redhat.com>

Reviewed-on: http://review.gluster.org/13798

Reviewed-by: Milind Changire <mchangir@redhat.com>

Reviewed-by: Kotresh HR <khiremat@redhat.com>

Reviewed-on: http://review.gluster.org/13856

Reviewed-on: https://code.engineering.redhat.com/gerrit/71831

Reviewed-by: Kotresh Hiremath Ravishankar <khiremat@redhat.com>

---

 tools/glusterfind/src/changelog.py     |    3 +-

 tools/glusterfind/src/changelogdata.py |   65 +++++++++++++-------------------

 tools/glusterfind/src/main.py          |   12 ++---

 tools/glusterfind/src/utils.py         |    2 +-

 4 files changed, 34 insertions(+), 48 deletions(-)

diff --git a/tools/glusterfind/src/changelog.py b/tools/glusterfind/src/changelog.py

index a58a7eb..283a035 100644

--- a/tools/glusterfind/src/changelog.py

+++ b/tools/glusterfind/src/changelog.py

@@ -17,6 +17,7 @@ import logging

 from argparse import ArgumentParser, RawDescriptionHelpFormatter

 import hashlib

 import urllib

+import codecs

 import libgfchangelog

 from utils import mkdirp, symlink_gfid_to_path

@@ -212,7 +213,7 @@ def parse_changelog_to_db(changelog_data, filename, args):

     """

     Parses a Changelog file and populates data in gfidpath table

     """

-    with open(filename) as f:

+    with codecs.open(filename, encoding="utf-8") as f:

         changelogfile = os.path.basename(filename)

         for line in f:

             data = line.strip().split(" ")

diff --git a/tools/glusterfind/src/changelogdata.py b/tools/glusterfind/src/changelogdata.py

index abb8b01..0e32d7b 100644

--- a/tools/glusterfind/src/changelogdata.py

+++ b/tools/glusterfind/src/changelogdata.py

@@ -146,10 +146,7 @@ class ChangelogData(object):

         for key, value in filters.items():

             query += " AND %s = ?" % key

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         return self.cursor_reader.execute(query, params)

@@ -161,10 +158,7 @@ class ChangelogData(object):

         for key, value in filters.items():

             query += " AND %s = ?" % key

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         return self.cursor_reader.execute(query, params)

@@ -175,10 +169,7 @@ class ChangelogData(object):

         for key, value in filters.items():

             query += " AND %s = ?" % key

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         self.cursor.execute(query, params)

@@ -189,10 +180,7 @@ class ChangelogData(object):

         params = []

         for key, value in data.items():

             fields.append(key)

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         values_substitute = len(fields)*["?"]

         query += "%s) VALUES(%s)" % (",".join(fields),

@@ -205,20 +193,14 @@ class ChangelogData(object):

         update_fields = []

         for key, value in data.items():

             update_fields.append("%s = ?" % key)

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         query = "UPDATE %s SET %s WHERE 1 = 1" % (tablename,

                                                   ", ".join(update_fields))

         for key, value in filters.items():

             query += " AND %s = ?" % key

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         self.cursor.execute(query, params)

@@ -230,12 +212,8 @@ class ChangelogData(object):

         params = []

         for key, value in filters.items():

-            print value

             query += " AND %s = ?" % key

-            if isinstance(value, int):

-                params.append(value)

-            else:

-                params.append(unicode(value, "utf8"))

+            params.append(value)

         self.cursor.execute(query, params)

         row = self.cursor.fetchone()

@@ -344,10 +322,15 @@ class ChangelogData(object):

     def when_create_mknod_mkdir(self, changelogfile, data):

         # E <GFID> <MKNOD|CREATE|MKDIR> <MODE> <USER> <GRP> <PGFID>/<BNAME>

         # Add the Entry to DB

-        pgfid1, bn1 = urllib.unquote_plus(data[6]).split("/", 1)

+        # urllib.unquote_plus will not handle unicode so, encode Unicode to

+        # represent in 8 bit format and then unquote

+        pgfid1, bn1 = urllib.unquote_plus(

+            data[6].encode("utf-8")).split("/", 1)

         if self.args.no_encode:

-            bn1 = bn1.strip()

+            # No urlencode since no_encode is set, so convert again to Unicode

+            # format from previously encoded.

+            bn1 = bn1.decode("utf-8").strip()

         else:

             # Quote again the basename

             bn1 = urllib.quote_plus(bn1.strip())

@@ -356,13 +339,15 @@ class ChangelogData(object):

     def when_rename(self, changelogfile, data):

         # E <GFID> RENAME <OLD_PGFID>/<BNAME> <PGFID>/<BNAME>

-        pgfid1, bn1 = urllib.unquote_plus(data[3]).split("/", 1)

-        pgfid2, bn2 = urllib.unquote_plus(data[4]).split("/", 1)

+        pgfid1, bn1 = urllib.unquote_plus(

+            data[3].encode("utf-8")).split("/", 1)

+        pgfid2, bn2 = urllib.unquote_plus(

+            data[4].encode("utf-8")).split("/", 1)

         if self.args.no_encode:

             # Quote again the basename

-            bn1 = bn1.strip()

-            bn2 = bn2.strip()

+            bn1 = bn1.decode("utf-8").strip()

+            bn2 = bn2.decode("utf-8").strip()

         else:

             # Quote again the basename

             bn1 = urllib.quote_plus(bn1.strip())

@@ -406,10 +391,11 @@ class ChangelogData(object):

     def when_link_symlink(self, changelogfile, data):

         # E <GFID> <LINK|SYMLINK> <PGFID>/<BASENAME>

         # Add as New record in Db as Type NEW

-        pgfid1, bn1 = urllib.unquote_plus(data[3]).split("/", 1)

+        pgfid1, bn1 = urllib.unquote_plus(

+            data[3].encode("utf-8")).split("/", 1)

         if self.args.no_encode:

             # Quote again the basename

-            bn1 = bn1.strip()

+            bn1 = bn1.decode("utf-8").strip()

         else:

             # Quote again the basename

             bn1 = urllib.quote_plus(bn1.strip())

@@ -424,10 +410,11 @@ class ChangelogData(object):

     def when_unlink_rmdir(self, changelogfile, data):

         # E <GFID> <UNLINK|RMDIR> <PGFID>/<BASENAME>

-        pgfid1, bn1 = urllib.unquote_plus(data[3]).split("/", 1)

+        pgfid1, bn1 = urllib.unquote_plus(

+            data[3].encode("utf-8")).split("/", 1)

         if self.args.no_encode:

-            bn1 = bn1.strip()

+            bn1 = bn1.decode("utf-8").strip()

         else:

             # Quote again the basename

             bn1 = urllib.quote_plus(bn1.strip())

diff --git a/tools/glusterfind/src/main.py b/tools/glusterfind/src/main.py

index 72f11a7..a87fac4 100644

--- a/tools/glusterfind/src/main.py

+++ b/tools/glusterfind/src/main.py

@@ -24,6 +24,7 @@ from utils import setup_logger, human_time, handle_rm_error

 from utils import get_changelog_rollover_time, cache_output, create_file

 import conf

 from changelogdata import OutputMerger

+import codecs

 PROG_DESCRIPTION = """

 GlusterFS Incremental API

@@ -394,7 +395,7 @@ def enable_volume_options(args):

 def write_output(args, outfilemerger):

-    with open(args.outfile, "a") as f:

+    with codecs.open(args.outfile, "a", encoding="utf-8") as f:

         for row in outfilemerger.get():

             # Multiple paths in case of Hardlinks

             paths = row[1].split(",")

@@ -409,12 +410,9 @@ def write_output(args, outfilemerger):

                 if p_rep == row_2_rep:

                     continue

-                p_rep = p_rep.encode('utf8', 'replace')

-                row_2_rep = row_2_rep.encode('utf8', 'replace')

-

-                f.write("{0} {1} {2}\n".format(row[0],

-                                               p_rep,

-                                               row_2_rep))

+                f.write(u"{0} {1} {2}\n".format(row[0],

+                                                p_rep,

+                                                row_2_rep))

 def mode_create(session_dir, args):

diff --git a/tools/glusterfind/src/utils.py b/tools/glusterfind/src/utils.py

index b3b0bdf..b05f08e 100644

--- a/tools/glusterfind/src/utils.py

+++ b/tools/glusterfind/src/utils.py

@@ -242,4 +242,4 @@ def output_path_prepare(path, args):

     if args.no_encode:

         return path

     else:

-        return urllib.quote_plus(path)

+        return urllib.quote_plus(path.encode("utf-8"))

-- 

1.7.1