pgreco / rpms / ipa

Forked from forks/areguera/rpms/ipa 4 years ago
Clone

Blame SOURCES/0060-schema-Speed-up-schema-cache.patch

403b09
From beff42632d1db674802c817afd49a3ac8bcd8fb6 Mon Sep 17 00:00:00 2001
403b09
From: David Kupka <dkupka@redhat.com>
403b09
Date: Wed, 27 Jul 2016 10:46:40 +0200
403b09
Subject: [PATCH] schema: Speed up schema cache
403b09
403b09
Check presence of schema in cache (and download it if necessary) on
403b09
__init__ instead of with each __getitem__ call. Prefill internal
403b09
dictionary with empty record for each command to be able to quickly
403b09
determine if requested command exist in schema or not. Rest of schema
403b09
data are read from cache on first attempt to retrive them.
403b09
403b09
https://fedorahosted.org/freeipa/ticket/6048
403b09
https://fedorahosted.org/freeipa/ticket/6069
403b09
403b09
Reviewed-By: Jan Cholasta <jcholast@redhat.com>
403b09
---
403b09
 ipaclient/remote_plugins/schema.py | 301 ++++++++++++++++++++++---------------
403b09
 1 file changed, 177 insertions(+), 124 deletions(-)
403b09
403b09
diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py
403b09
index 0301e54127dc236ebc14e1409484626f1427800d..d039fb41991c26a9c7b7f76f6959668efb677586 100644
403b09
--- a/ipaclient/remote_plugins/schema.py
403b09
+++ b/ipaclient/remote_plugins/schema.py
403b09
@@ -5,10 +5,8 @@
403b09
 import collections
403b09
 import errno
403b09
 import fcntl
403b09
-import glob
403b09
 import json
403b09
 import os
403b09
-import re
403b09
 import sys
403b09
 import time
403b09
 import types
403b09
@@ -65,8 +63,6 @@ USER_CACHE_PATH = (
403b09
         '.cache'
403b09
     )
403b09
 )
403b09
-SCHEMA_DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'schema')
403b09
-SERVERS_DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'servers')
403b09
 
403b09
 logger = log_mgr.get_logger(__name__)
403b09
 
403b09
@@ -274,15 +270,6 @@ class _SchemaObjectPlugin(_SchemaPlugin):
403b09
     schema_key = 'classes'
403b09
 
403b09
 
403b09
-def _ensure_dir_created(d):
403b09
-    try:
403b09
-        os.makedirs(d)
403b09
-    except OSError as e:
403b09
-        if e.errno != errno.EEXIST:
403b09
-            raise RuntimeError("Unable to create cache directory: {}"
403b09
-                               "".format(e))
403b09
-
403b09
-
403b09
 class _LockedZipFile(zipfile.ZipFile):
403b09
     """ Add locking to zipfile.ZipFile
403b09
     Shared lock is used with read mode, exclusive with write mode.
403b09
@@ -308,7 +295,10 @@ class _SchemaNameSpace(collections.Mapping):
403b09
         self._schema = schema
403b09
 
403b09
     def __getitem__(self, key):
403b09
-        return self._schema.read_namespace_member(self.name, key)
403b09
+        try:
403b09
+            return self._schema.read_namespace_member(self.name, key)
403b09
+        except KeyError:
403b09
+            raise KeyError(key)
403b09
 
403b09
     def __iter__(self):
403b09
         for key in self._schema.iter_namespace(self.name):
403b09
@@ -322,6 +312,62 @@ class NotAvailable(Exception):
403b09
     pass
403b09
 
403b09
 
403b09
+class ServerInfo(collections.MutableMapping):
403b09
+    _DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'servers')
403b09
+
403b09
+    def __init__(self, api):
403b09
+        hostname = DNSName(api.env.server).ToASCII()
403b09
+        self._path = os.path.join(self._DIR, hostname)
403b09
+        self._dict = {}
403b09
+        self._dirty = False
403b09
+
403b09
+        self._read()
403b09
+
403b09
+    def __enter__(self):
403b09
+        return self
403b09
+
403b09
+    def __exit__(self, *_exc_info):
403b09
+        if self._dirty:
403b09
+            self._write()
403b09
+
403b09
+    def _read(self):
403b09
+        try:
403b09
+            with open(self._path, 'r') as sc:
403b09
+                self._dict = json.load(sc)
403b09
+        except EnvironmentError as e:
403b09
+            if e.errno != errno.ENOENT:
403b09
+                logger.warning('Failed to read server info: {}'.format(e))
403b09
+
403b09
+    def _write(self):
403b09
+        try:
403b09
+            try:
403b09
+                os.makedirs(self._DIR)
403b09
+            except EnvironmentError as e:
403b09
+                if e.errno != errno.EEXIST:
403b09
+                    raise
403b09
+            with open(self._path, 'w') as sc:
403b09
+                json.dump(self._dict, sc)
403b09
+        except EnvironmentError as e:
403b09
+            logger.warning('Failed to write server info: {}'.format(e))
403b09
+
403b09
+    def __getitem__(self, key):
403b09
+        return self._dict[key]
403b09
+
403b09
+    def __setitem__(self, key, value):
403b09
+        self._dirty = key not in self._dict or self._dict[key] != value
403b09
+        self._dict[key] = value
403b09
+
403b09
+    def __delitem__(self, key):
403b09
+        del self._dict[key]
403b09
+        self._dirty = True
403b09
+
403b09
+    def __iter__(self):
403b09
+        return iter(self._dict)
403b09
+
403b09
+    def __len__(self):
403b09
+        return len(self._dict)
403b09
+
403b09
+
403b09
 class Schema(object):
403b09
     """
403b09
     Store and provide schema for commands and topics
403b09
@@ -342,38 +388,76 @@ class Schema(object):
403b09
     u'Ping the remote IPA server to ...'
403b09
 
403b09
     """
403b09
-    schema_path_template = os.path.join(SCHEMA_DIR, '{}')
403b09
-    servers_path_template = os.path.join(SERVERS_DIR, '{}')
403b09
-    ns_member_pattern_template = '^{}/(?P<name>.+)$'
403b09
-    ns_member_path_template = '{}/{}'
403b09
     namespaces = {'classes', 'commands', 'topics'}
403b09
     schema_info_path = 'schema'
403b09
+    _DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'schema')
403b09
 
403b09
-    @classmethod
403b09
-    def _list(cls):
403b09
-        for f in glob.glob(cls.schema_path_template.format('*')):
403b09
-            yield os.path.splitext(os.path.basename(f))[0]
403b09
+    def __init__(self, api, server_info, client):
403b09
+        self._dict = {}
403b09
+        self._namespaces = {}
403b09
+        self._help = None
403b09
 
403b09
-    @classmethod
403b09
-    def _in_cache(cls, fingeprint):
403b09
-        return os.path.exists(cls.schema_path_template.format(fingeprint))
403b09
+        for ns in self.namespaces:
403b09
+            self._dict[ns] = {}
403b09
+            self._namespaces[ns] = _SchemaNameSpace(self, ns)
403b09
 
403b09
-    def __init__(self, api, client):
403b09
-        self._api = api
403b09
-        self._client = client
403b09
-        self._dict = {}
403b09
+        is_known = False
403b09
+        if not api.env.force_schema_check:
403b09
+            try:
403b09
+                self._fingerprint = server_info['fingerprint']
403b09
+                self._expiration = server_info['expiration']
403b09
+            except KeyError:
403b09
+                pass
403b09
+            else:
403b09
+                is_known = True
403b09
+
403b09
+        if is_known:
403b09
+            try:
403b09
+                self._read_schema()
403b09
+            except Exception:
403b09
+                pass
403b09
+            else:
403b09
+                return
403b09
 
403b09
-    def _open_server_info(self, hostname, mode):
403b09
-        encoded_hostname = DNSName(hostname).ToASCII()
403b09
-        path = self.servers_path_template.format(encoded_hostname)
403b09
-        return open(path, mode)
403b09
+        try:
403b09
+            self._fetch(client)
403b09
+        except NotAvailable:
403b09
+            raise
403b09
+        else:
403b09
+            self._write_schema()
403b09
+        finally:
403b09
+            try:
403b09
+                server_info['fingerprint'] = self._fingerprint
403b09
+                server_info['expiration'] = self._expiration
403b09
+            except AttributeError:
403b09
+                pass
403b09
 
403b09
-    def _get_schema(self):
403b09
-        client = self._client
403b09
+    def _open_schema(self, filename, mode):
403b09
+        path = os.path.join(self._DIR, filename)
403b09
+        return _LockedZipFile(path, mode)
403b09
+
403b09
+    def _get_schema_fingerprint(self, schema):
403b09
+        schema_info = json.loads(schema.read(self.schema_info_path))
403b09
+        return schema_info['fingerprint']
403b09
+
403b09
+    def _fetch(self, client):
403b09
         if not client.isconnected():
403b09
             client.connect(verbose=False)
403b09
 
403b09
-        fps = [unicode(f) for f in Schema._list()]
403b09
+        fps = []
403b09
+        try:
403b09
+            files = os.listdir(self._DIR)
403b09
+        except EnvironmentError:
403b09
+            pass
403b09
+        else:
403b09
+            for filename in files:
403b09
+                try:
403b09
+                    with self._open_schema(filename, 'r') as schema:
403b09
+                        fps.append(
403b09
+                            unicode(self._get_schema_fingerprint(schema)))
403b09
+                except Exception:
403b09
+                    continue
403b09
+
403b09
         kwargs = {u'version': u'2.170'}
403b09
         if fps:
403b09
             kwargs[u'known_fingerprints'] = fps
403b09
@@ -386,110 +470,80 @@ class Schema(object):
403b09
             ttl = e.ttl
403b09
         else:
403b09
             fp = schema['fingerprint']
403b09
-            ttl = schema['ttl']
403b09
-            self._store(fp, schema)
403b09
-        finally:
403b09
-            client.disconnect()
403b09
+            ttl = schema.pop('ttl', 0)
403b09
 
403b09
-        exp = ttl + time.time()
403b09
-        return (fp, exp)
403b09
+            for key, value in schema.items():
403b09
+                if key in self.namespaces:
403b09
+                    value = {m['full_name']: m for m in value}
403b09
+                self._dict[key] = value
403b09
 
403b09
-    def _ensure_cached(self):
403b09
-        no_info = False
403b09
-        try:
403b09
-            # pylint: disable=access-member-before-definition
403b09
-            fp = self._server_schema_fingerprint
403b09
-            exp = self._server_schema_expiration
403b09
-        except AttributeError:
403b09
-            try:
403b09
-                with self._open_server_info(self._api.env.server, 'r') as sc:
403b09
-                    si = json.load(sc)
403b09
-
403b09
-                fp = si['fingerprint']
403b09
-                exp = si['expiration']
403b09
-            except Exception as e:
403b09
-                no_info = True
403b09
-                if not (isinstance(e, EnvironmentError) and
403b09
-                        e.errno == errno.ENOENT):  # pylint: disable=no-member
403b09
-                    logger.warning('Failed to load server properties: {}'
403b09
-                                   ''.format(e))
403b09
-
403b09
-        force_check = ((not getattr(self, '_schema_checked', False)) and
403b09
-                       self._api.env.force_schema_check)
403b09
-
403b09
-        if (force_check or
403b09
-                no_info or exp < time.time() or not Schema._in_cache(fp)):
403b09
-            (fp, exp) = self._get_schema()
403b09
-            self._schema_checked = True
403b09
-            _ensure_dir_created(SERVERS_DIR)
403b09
-            try:
403b09
-                with self._open_server_info(self._api.env.server, 'w') as sc:
403b09
-                    json.dump(dict(fingerprint=fp, expiration=exp), sc)
403b09
-            except Exception as e:
403b09
-                logger.warning('Failed to store server properties: {}'
403b09
-                               ''.format(e))
403b09
-
403b09
-        if not self._dict:
403b09
-            self._dict['fingerprint'] = fp
403b09
-            schema_info = self._read(self.schema_info_path)
403b09
+        self._fingerprint = fp
403b09
+        self._expiration = ttl + time.time()
403b09
+
403b09
+    def _read_schema(self):
403b09
+        with self._open_schema(self._fingerprint, 'r') as schema:
403b09
+            self._dict['fingerprint'] = self._get_schema_fingerprint(schema)
403b09
+            schema_info = json.loads(schema.read(self.schema_info_path))
403b09
             self._dict['version'] = schema_info['version']
403b09
-            for ns in self.namespaces:
403b09
-                self._dict[ns] = _SchemaNameSpace(self, ns)
403b09
 
403b09
-        self._server_schema_fingerprintr = fp
403b09
-        self._server_schema_expiration = exp
403b09
+            for name in schema.namelist():
403b09
+                ns, _slash, key = name.partition('/')
403b09
+                if ns in self.namespaces:
403b09
+                    self._dict[ns][key] = {}
403b09
 
403b09
     def __getitem__(self, key):
403b09
-        self._ensure_cached()
403b09
-        return self._dict[key]
403b09
+        try:
403b09
+            return self._namespaces[key]
403b09
+        except KeyError:
403b09
+            return self._dict[key]
403b09
 
403b09
-    def _open_archive(self, mode, fp=None):
403b09
-        if not fp:
403b09
-            fp = self['fingerprint']
403b09
-        arch_path = self.schema_path_template.format(fp)
403b09
-        return _LockedZipFile(arch_path, mode)
403b09
-
403b09
-    def _store(self, fingerprint, schema={}):
403b09
-        _ensure_dir_created(SCHEMA_DIR)
403b09
-
403b09
-        schema_info = dict(version=schema['version'],
403b09
-                           fingerprint=schema['fingerprint'])
403b09
-
403b09
-        with self._open_archive('w', fingerprint) as zf:
403b09
-            # store schema information
403b09
-            zf.writestr(self.schema_info_path, json.dumps(schema_info))
403b09
-            # store namespaces
403b09
-            for namespace in self.namespaces:
403b09
-                for member in schema[namespace]:
403b09
-                    path = self.ns_member_path_template.format(
403b09
-                        namespace,
403b09
-                        member['full_name']
403b09
-                    )
403b09
-                    zf.writestr(path, json.dumps(member))
403b09
+    def _write_schema(self):
403b09
+        try:
403b09
+            os.makedirs(self._DIR)
403b09
+        except EnvironmentError as e:
403b09
+            if e.errno != errno.EEXIST:
403b09
+                logger.warning("Failed ti write schema: {}".format(e))
403b09
+                return
403b09
+
403b09
+        with self._open_schema(self._fingerprint, 'w') as schema:
403b09
+            schema_info = {}
403b09
+            for key, value in self._dict.items():
403b09
+                if key in self.namespaces:
403b09
+                    ns = value
403b09
+                    for member in ns:
403b09
+                        path = '{}/{}'.format(key, member)
403b09
+                        schema.writestr(path, json.dumps(ns[member]))
403b09
+                else:
403b09
+                    schema_info[key] = value
403b09
+
403b09
+            schema.writestr(self.schema_info_path, json.dumps(schema_info))
403b09
 
403b09
     def _read(self, path):
403b09
-        with self._open_archive('r') as zf:
403b09
+        with self._open_schema(self._fingerprint, 'r') as zf:
403b09
             return json.loads(zf.read(path))
403b09
 
403b09
     def read_namespace_member(self, namespace, member):
403b09
-        path = self.ns_member_path_template.format(namespace, member)
403b09
-        return self._read(path)
403b09
+        value = self._dict[namespace][member]
403b09
+
403b09
+        if (not value) or ('full_name' not in value):
403b09
+            path = '{}/{}'.format(namespace, member)
403b09
+            value = self._dict[namespace].setdefault(
403b09
+                member, {}
403b09
+            ).update(self._read(path))
403b09
+
403b09
+        return value
403b09
 
403b09
     def iter_namespace(self, namespace):
403b09
-        pattern = self.ns_member_pattern_template.format(namespace)
403b09
-        with self._open_archive('r') as zf:
403b09
-            for name in zf.namelist():
403b09
-                r = re.match(pattern, name)
403b09
-                if r:
403b09
-                    yield r.groups('name')[0]
403b09
+        return iter(self._dict[namespace])
403b09
 
403b09
 
403b09
 def get_package(api, client):
403b09
     try:
403b09
         schema = api._schema
403b09
     except AttributeError:
403b09
-        schema = Schema(api, client)
403b09
-        object.__setattr__(api, '_schema', schema)
403b09
+        with ServerInfo(api.env.hostname) as server_info:
403b09
+            schema = Schema(api, server_info, client)
403b09
+            object.__setattr__(api, '_schema', schema)
403b09
 
403b09
     fingerprint = str(schema['fingerprint'])
403b09
     package_name = '{}${}'.format(__name__, fingerprint)
403b09
@@ -509,10 +563,9 @@ def get_package(api, client):
403b09
     module = types.ModuleType(module_name)
403b09
     module.__file__ = os.path.join(package_dir, 'plugins.py')
403b09
     module.register = plugable.Registry()
403b09
-    for key, plugin_cls in (('commands', _SchemaCommandPlugin),
403b09
-                            ('classes', _SchemaObjectPlugin)):
403b09
-        for full_name in schema[key]:
403b09
-            plugin = plugin_cls(full_name)
403b09
+    for plugin_cls in (_SchemaCommandPlugin, _SchemaObjectPlugin):
403b09
+        for full_name in schema[plugin_cls.schema_key]:
403b09
+            plugin = plugin_cls(str(full_name))
403b09
             plugin = module.register()(plugin)
403b09
     sys.modules[module_name] = module
403b09
 
403b09
-- 
403b09
2.7.4
403b09