|
 |
e7a346 |
From d41cb3f53614dcf514d96717b5bde67b8d4c1335 Mon Sep 17 00:00:00 2001
|
|
|
e7a346 |
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
|
|
|
e7a346 |
Date: Mon, 12 Feb 2018 17:58:48 +0530
|
|
|
e7a346 |
Subject: [PATCH 152/180] glusterfind: Speed up gfid lookup 100x by using an
|
|
|
e7a346 |
SQL index
|
|
|
e7a346 |
MIME-Version: 1.0
|
|
|
e7a346 |
Content-Type: text/plain; charset=UTF-8
|
|
|
e7a346 |
Content-Transfer-Encoding: 8bit
|
|
|
e7a346 |
|
|
|
e7a346 |
Fixes #1529883.
|
|
|
e7a346 |
|
|
|
e7a346 |
This fixes some bits of `glusterfind`'s horrible performance,
|
|
|
e7a346 |
making it 100x faster.
|
|
|
e7a346 |
|
|
|
e7a346 |
Until now, glusterfind was, for each line in each CHANGELOG.* file,
|
|
|
e7a346 |
linearly reading the entire contents of the sqlite database in
|
|
|
e7a346 |
4096-bytes-sized pread64() syscalls when executing the
|
|
|
e7a346 |
|
|
|
e7a346 |
SELECT COUNT(1) FROM %s WHERE 1=1 AND gfid = ?
|
|
|
e7a346 |
|
|
|
e7a346 |
query through the code path:
|
|
|
e7a346 |
|
|
|
e7a346 |
get_changes()
|
|
|
e7a346 |
parse_changelog_to_db()
|
|
|
e7a346 |
when_data_meta()
|
|
|
e7a346 |
gfidpath_exists()
|
|
|
e7a346 |
_exists()
|
|
|
e7a346 |
|
|
|
e7a346 |
In a quick benchmark on my laptop, doing one such `SELECT` query
|
|
|
e7a346 |
took ~75ms on a 10MB-sized sqlite DB, while doing the same query
|
|
|
e7a346 |
with an index took < 1ms.
|
|
|
e7a346 |
|
|
|
e7a346 |
mainline:
|
|
|
e7a346 |
> BUG: 1529883
|
|
|
e7a346 |
> Reviewed-on: https://review.gluster.org/19114
|
|
|
e7a346 |
> Reviewed-by: Aravinda VK <avishwan@redhat.com>
|
|
|
e7a346 |
> Signed-off-by: Niklas Hambüchen <mail@nh2.me>
|
|
|
e7a346 |
(cherry picked from commit 14dbd5da1cae64e6d4d2c69966e19844d090ce98)
|
|
|
e7a346 |
|
|
|
e7a346 |
Change-Id: I8e7fe60f1f45a06c102f56b54d2ead9e0377794e
|
|
|
e7a346 |
Signed-off-by: Niklas Hambüchen <mail@nh2.me>
|
|
|
e7a346 |
Reviewed-on: https://code.engineering.redhat.com/gerrit/130064
|
|
|
e7a346 |
Reviewed-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
|
|
|
e7a346 |
Tested-by: Sunil Kumar Heggodu Gopala Acharya <sheggodu@redhat.com>
|
|
|
e7a346 |
---
|
|
|
e7a346 |
tools/glusterfind/src/changelogdata.py | 5 +++++
|
|
|
e7a346 |
1 file changed, 5 insertions(+)
|
|
|
e7a346 |
|
|
|
e7a346 |
diff --git a/tools/glusterfind/src/changelogdata.py b/tools/glusterfind/src/changelogdata.py
|
|
|
e7a346 |
index 3140d94..641593c 100644
|
|
|
e7a346 |
--- a/tools/glusterfind/src/changelogdata.py
|
|
|
e7a346 |
+++ b/tools/glusterfind/src/changelogdata.py
|
|
|
e7a346 |
@@ -112,6 +112,11 @@ class ChangelogData(object):
|
|
|
e7a346 |
"""
|
|
|
e7a346 |
self.cursor.execute(create_table)
|
|
|
e7a346 |
|
|
|
e7a346 |
+ create_index = """
|
|
|
e7a346 |
+ CREATE INDEX gfid_index ON gfidpath(gfid);
|
|
|
e7a346 |
+ """
|
|
|
e7a346 |
+ self.cursor.execute(create_index)
|
|
|
e7a346 |
+
|
|
|
e7a346 |
def _create_table_inodegfid(self):
|
|
|
e7a346 |
drop_table = "DROP TABLE IF EXISTS inodegfid"
|
|
|
e7a346 |
self.cursor.execute(drop_table)
|
|
|
e7a346 |
--
|
|
|
e7a346 |
1.8.3.1
|
|
|
e7a346 |
|