|
 |
a3470f |
From d41cb3f53614dcf514d96717b5bde67b8d4c1335 Mon Sep 17 00:00:00 2001
|
|
|
a3470f |
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
|
|
|
a3470f |
Date: Mon, 12 Feb 2018 17:58:48 +0530
|
|
|
a3470f |
Subject: [PATCH 152/180] glusterfind: Speed up gfid lookup 100x by using an
|
|
|
a3470f |
SQL index
|
|
|
a3470f |
MIME-Version: 1.0
|
|
|
a3470f |
Content-Type: text/plain; charset=UTF-8
|
|
|
a3470f |
Content-Transfer-Encoding: 8bit
|
|
|
a3470f |
|
|
|
a3470f |
Fixes #1529883.
|
|
|
a3470f |
|
|
|
a3470f |
This fixes some bits of `glusterfind`'s horrible performance,
|
|
|
a3470f |
making it 100x faster.
|
|
|
a3470f |
|
|
|
a3470f |
Until now, glusterfind was, for each line in each CHANGELOG.* file,
|
|
|
a3470f |
linearly reading the entire contents of the sqlite database in
|
|
|
a3470f |
4096-bytes-sized pread64() syscalls when executing the
|
|
|
a3470f |
|
|
|
a3470f |
SELECT COUNT(1) FROM %s WHERE 1=1 AND gfid = ?
|
|
|
a3470f |
|
|
|
a3470f |
query through the code path:
|
|
|
a3470f |
|
|
|
a3470f |
get_changes()
|
|
|
a3470f |
parse_changelog_to_db()
|
|
|
a3470f |
when_data_meta()
|
|
|
a3470f |
gfidpath_exists()
|
|
|
a3470f |
_exists()
|
|
|
a3470f |
|
|
|
a3470f |
In a quick benchmark on my laptop, doing one such `SELECT` query
|
|
|
a3470f |
took ~75ms on a 10MB-sized sqlite DB, while doing the same query
|
|
|
a3470f |
with an index took < 1ms.
|
|
|
a3470f |
|
|
|
a3470f |
mainline:
|
|
|
a3470f |
> BUG: 1529883
|
|
|
a3470f |
> Reviewed-on: https://review.gluster.org/19114
|
|
|
a3470f |
> Reviewed-by: Aravinda VK <avishwan@redhat.com>
|
|
|
a3470f |
> Signed-off-by: Niklas Hambüchen <mail@nh2.me>
|
|
|
a3470f |
(cherry picked from commit 14dbd5da1cae64e6d4d2c69966e19844d090ce98)
|
|
|
a3470f |
|
|
|
a3470f |
Change-Id: I8e7fe60f1f45a06c102f56b54d2ead9e0377794e
|
|
|
a3470f |
Signed-off-by: Niklas Hambüchen <mail@nh2.me>
|
|
|
a3470f |
Reviewed-on: https://code.engineering.redhat.com/gerrit/130064
|
|
|
a3470f |
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
|
|
|
a3470f |
Tested-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
|
|
|
a3470f |
---
|
|
|
a3470f |
tools/glusterfind/src/changelogdata.py | 5 +++++
|
|
|
a3470f |
1 file changed, 5 insertions(+)
|
|
|
a3470f |
|
|
|
a3470f |
diff --git a/tools/glusterfind/src/changelogdata.py b/tools/glusterfind/src/changelogdata.py
|
|
|
a3470f |
index 3140d94..641593c 100644
|
|
|
a3470f |
--- a/tools/glusterfind/src/changelogdata.py
|
|
|
a3470f |
+++ b/tools/glusterfind/src/changelogdata.py
|
|
|
a3470f |
@@ -112,6 +112,11 @@ class ChangelogData(object):
|
|
|
a3470f |
"""
|
|
|
a3470f |
self.cursor.execute(create_table)
|
|
|
a3470f |
|
|
|
a3470f |
+ create_index = """
|
|
|
a3470f |
+ CREATE INDEX gfid_index ON gfidpath(gfid);
|
|
|
a3470f |
+ """
|
|
|
a3470f |
+ self.cursor.execute(create_index)
|
|
|
a3470f |
+
|
|
|
a3470f |
def _create_table_inodegfid(self):
|
|
|
a3470f |
drop_table = "DROP TABLE IF EXISTS inodegfid"
|
|
|
a3470f |
self.cursor.execute(drop_table)
|
|
|
a3470f |
--
|
|
|
a3470f |
1.8.3.1
|
|
|
a3470f |
|