From beff42632d1db674802c817afd49a3ac8bcd8fb6 Mon Sep 17 00:00:00 2001

From: David Kupka <dkupka@redhat.com>

Date: Wed, 27 Jul 2016 10:46:40 +0200

Subject: [PATCH] schema: Speed up schema cache

Check presence of schema in cache (and download it if necessary) on

__init__ instead of with each __getitem__ call. Prefill internal

dictionary with empty record for each command to be able to quickly

determine if requested command exist in schema or not. Rest of schema

data are read from cache on first attempt to retrive them.

https://fedorahosted.org/freeipa/ticket/6048

https://fedorahosted.org/freeipa/ticket/6069

Reviewed-By: Jan Cholasta <jcholast@redhat.com>

---

 ipaclient/remote_plugins/schema.py | 301 ++++++++++++++++++++++---------------

 1 file changed, 177 insertions(+), 124 deletions(-)

diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py

index 0301e54127dc236ebc14e1409484626f1427800d..d039fb41991c26a9c7b7f76f6959668efb677586 100644

--- a/ipaclient/remote_plugins/schema.py

+++ b/ipaclient/remote_plugins/schema.py

@@ -5,10 +5,8 @@

 import collections

 import errno

 import fcntl

-import glob

 import json

 import os

-import re

 import sys

 import time

 import types

@@ -65,8 +63,6 @@ USER_CACHE_PATH = (

         '.cache'

     )

 )

-SCHEMA_DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'schema')

-SERVERS_DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'servers')

 logger = log_mgr.get_logger(__name__)

@@ -274,15 +270,6 @@ class _SchemaObjectPlugin(_SchemaPlugin):

     schema_key = 'classes'

-def _ensure_dir_created(d):

-    try:

-        os.makedirs(d)

-    except OSError as e:

-        if e.errno != errno.EEXIST:

-            raise RuntimeError("Unable to create cache directory: {}"

-                               "".format(e))

-

-

 class _LockedZipFile(zipfile.ZipFile):

     """ Add locking to zipfile.ZipFile

     Shared lock is used with read mode, exclusive with write mode.

@@ -308,7 +295,10 @@ class _SchemaNameSpace(collections.Mapping):

         self._schema = schema

     def __getitem__(self, key):

-        return self._schema.read_namespace_member(self.name, key)

+        try:

+            return self._schema.read_namespace_member(self.name, key)

+        except KeyError:

+            raise KeyError(key)

     def __iter__(self):

         for key in self._schema.iter_namespace(self.name):

@@ -322,6 +312,62 @@ class NotAvailable(Exception):

     pass

+class ServerInfo(collections.MutableMapping):

+    _DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'servers')

+

+    def __init__(self, api):

+        hostname = DNSName(api.env.server).ToASCII()

+        self._path = os.path.join(self._DIR, hostname)

+        self._dict = {}

+        self._dirty = False

+

+        self._read()

+

+    def __enter__(self):

+        return self

+

+    def __exit__(self, *_exc_info):

+        if self._dirty:

+            self._write()

+

+    def _read(self):

+        try:

+            with open(self._path, 'r') as sc:

+                self._dict = json.load(sc)

+        except EnvironmentError as e:

+            if e.errno != errno.ENOENT:

+                logger.warning('Failed to read server info: {}'.format(e))

+

+    def _write(self):

+        try:

+            try:

+                os.makedirs(self._DIR)

+            except EnvironmentError as e:

+                if e.errno != errno.EEXIST:

+                    raise

+            with open(self._path, 'w') as sc:

+                json.dump(self._dict, sc)

+        except EnvironmentError as e:

+            logger.warning('Failed to write server info: {}'.format(e))

+

+    def __getitem__(self, key):

+        return self._dict[key]

+

+    def __setitem__(self, key, value):

+        self._dirty = key not in self._dict or self._dict[key] != value

+        self._dict[key] = value

+

+    def __delitem__(self, key):

+        del self._dict[key]

+        self._dirty = True

+

+    def __iter__(self):

+        return iter(self._dict)

+

+    def __len__(self):

+        return len(self._dict)

+

+

 class Schema(object):

     """

     Store and provide schema for commands and topics

@@ -342,38 +388,76 @@ class Schema(object):

     u'Ping the remote IPA server to ...'

     """

-    schema_path_template = os.path.join(SCHEMA_DIR, '{}')

-    servers_path_template = os.path.join(SERVERS_DIR, '{}')

-    ns_member_pattern_template = '^{}/(?P<name>.+)$'

-    ns_member_path_template = '{}/{}'

     namespaces = {'classes', 'commands', 'topics'}

     schema_info_path = 'schema'

+    _DIR = os.path.join(USER_CACHE_PATH, 'ipa', 'schema')

-    @classmethod

-    def _list(cls):

-        for f in glob.glob(cls.schema_path_template.format('*')):

-            yield os.path.splitext(os.path.basename(f))[0]

+    def __init__(self, api, server_info, client):

+        self._dict = {}

+        self._namespaces = {}

+        self._help = None

-    @classmethod

-    def _in_cache(cls, fingeprint):

-        return os.path.exists(cls.schema_path_template.format(fingeprint))

+        for ns in self.namespaces:

+            self._dict[ns] = {}

+            self._namespaces[ns] = _SchemaNameSpace(self, ns)

-    def __init__(self, api, client):

-        self._api = api

-        self._client = client

-        self._dict = {}

+        is_known = False

+        if not api.env.force_schema_check:

+            try:

+                self._fingerprint = server_info['fingerprint']

+                self._expiration = server_info['expiration']

+            except KeyError:

+                pass

+            else:

+                is_known = True

+

+        if is_known:

+            try:

+                self._read_schema()

+            except Exception:

+                pass

+            else:

+                return

-    def _open_server_info(self, hostname, mode):

-        encoded_hostname = DNSName(hostname).ToASCII()

-        path = self.servers_path_template.format(encoded_hostname)

-        return open(path, mode)

+        try:

+            self._fetch(client)

+        except NotAvailable:

+            raise

+        else:

+            self._write_schema()

+        finally:

+            try:

+                server_info['fingerprint'] = self._fingerprint

+                server_info['expiration'] = self._expiration

+            except AttributeError:

+                pass

-    def _get_schema(self):

-        client = self._client

+    def _open_schema(self, filename, mode):

+        path = os.path.join(self._DIR, filename)

+        return _LockedZipFile(path, mode)

+

+    def _get_schema_fingerprint(self, schema):

+        schema_info = json.loads(schema.read(self.schema_info_path))

+        return schema_info['fingerprint']

+

+    def _fetch(self, client):

         if not client.isconnected():

             client.connect(verbose=False)

-        fps = [unicode(f) for f in Schema._list()]

+        fps = []

+        try:

+            files = os.listdir(self._DIR)

+        except EnvironmentError:

+            pass

+        else:

+            for filename in files:

+                try:

+                    with self._open_schema(filename, 'r') as schema:

+                        fps.append(

+                            unicode(self._get_schema_fingerprint(schema)))

+                except Exception:

+                    continue

+

         kwargs = {u'version': u'2.170'}

         if fps:

             kwargs[u'known_fingerprints'] = fps

@@ -386,110 +470,80 @@ class Schema(object):

             ttl = e.ttl

         else:

             fp = schema['fingerprint']

-            ttl = schema['ttl']

-            self._store(fp, schema)

-        finally:

-            client.disconnect()

+            ttl = schema.pop('ttl', 0)

-        exp = ttl + time.time()

-        return (fp, exp)

+            for key, value in schema.items():

+                if key in self.namespaces:

+                    value = {m['full_name']: m for m in value}

+                self._dict[key] = value

-    def _ensure_cached(self):

-        no_info = False

-        try:

-            # pylint: disable=access-member-before-definition

-            fp = self._server_schema_fingerprint

-            exp = self._server_schema_expiration

-        except AttributeError:

-            try:

-                with self._open_server_info(self._api.env.server, 'r') as sc:

-                    si = json.load(sc)

-

-                fp = si['fingerprint']

-                exp = si['expiration']

-            except Exception as e:

-                no_info = True

-                if not (isinstance(e, EnvironmentError) and

-                        e.errno == errno.ENOENT):  # pylint: disable=no-member

-                    logger.warning('Failed to load server properties: {}'

-                                   ''.format(e))

-

-        force_check = ((not getattr(self, '_schema_checked', False)) and

-                       self._api.env.force_schema_check)

-

-        if (force_check or

-                no_info or exp < time.time() or not Schema._in_cache(fp)):

-            (fp, exp) = self._get_schema()

-            self._schema_checked = True

-            _ensure_dir_created(SERVERS_DIR)

-            try:

-                with self._open_server_info(self._api.env.server, 'w') as sc:

-                    json.dump(dict(fingerprint=fp, expiration=exp), sc)

-            except Exception as e:

-                logger.warning('Failed to store server properties: {}'

-                               ''.format(e))

-

-        if not self._dict:

-            self._dict['fingerprint'] = fp

-            schema_info = self._read(self.schema_info_path)

+        self._fingerprint = fp

+        self._expiration = ttl + time.time()

+

+    def _read_schema(self):

+        with self._open_schema(self._fingerprint, 'r') as schema:

+            self._dict['fingerprint'] = self._get_schema_fingerprint(schema)

+            schema_info = json.loads(schema.read(self.schema_info_path))

             self._dict['version'] = schema_info['version']

-            for ns in self.namespaces:

-                self._dict[ns] = _SchemaNameSpace(self, ns)

-        self._server_schema_fingerprintr = fp

-        self._server_schema_expiration = exp

+            for name in schema.namelist():

+                ns, _slash, key = name.partition('/')

+                if ns in self.namespaces:

+                    self._dict[ns][key] = {}

     def __getitem__(self, key):

-        self._ensure_cached()

-        return self._dict[key]

+        try:

+            return self._namespaces[key]

+        except KeyError:

+            return self._dict[key]

-    def _open_archive(self, mode, fp=None):

-        if not fp:

-            fp = self['fingerprint']

-        arch_path = self.schema_path_template.format(fp)

-        return _LockedZipFile(arch_path, mode)

-

-    def _store(self, fingerprint, schema={}):

-        _ensure_dir_created(SCHEMA_DIR)

-

-        schema_info = dict(version=schema['version'],

-                           fingerprint=schema['fingerprint'])

-

-        with self._open_archive('w', fingerprint) as zf:

-            # store schema information

-            zf.writestr(self.schema_info_path, json.dumps(schema_info))

-            # store namespaces

-            for namespace in self.namespaces:

-                for member in schema[namespace]:

-                    path = self.ns_member_path_template.format(

-                        namespace,

-                        member['full_name']

-                    )

-                    zf.writestr(path, json.dumps(member))

+    def _write_schema(self):

+        try:

+            os.makedirs(self._DIR)

+        except EnvironmentError as e:

+            if e.errno != errno.EEXIST:

+                logger.warning("Failed ti write schema: {}".format(e))

+                return

+

+        with self._open_schema(self._fingerprint, 'w') as schema:

+            schema_info = {}

+            for key, value in self._dict.items():

+                if key in self.namespaces:

+                    ns = value

+                    for member in ns:

+                        path = '{}/{}'.format(key, member)

+                        schema.writestr(path, json.dumps(ns[member]))

+                else:

+                    schema_info[key] = value

+

+            schema.writestr(self.schema_info_path, json.dumps(schema_info))

     def _read(self, path):

-        with self._open_archive('r') as zf:

+        with self._open_schema(self._fingerprint, 'r') as zf:

             return json.loads(zf.read(path))

     def read_namespace_member(self, namespace, member):

-        path = self.ns_member_path_template.format(namespace, member)

-        return self._read(path)

+        value = self._dict[namespace][member]

+

+        if (not value) or ('full_name' not in value):

+            path = '{}/{}'.format(namespace, member)

+            value = self._dict[namespace].setdefault(

+                member, {}

+            ).update(self._read(path))

+

+        return value

     def iter_namespace(self, namespace):

-        pattern = self.ns_member_pattern_template.format(namespace)

-        with self._open_archive('r') as zf:

-            for name in zf.namelist():

-                r = re.match(pattern, name)

-                if r:

-                    yield r.groups('name')[0]

+        return iter(self._dict[namespace])

 def get_package(api, client):

     try:

         schema = api._schema

     except AttributeError:

-        schema = Schema(api, client)

-        object.__setattr__(api, '_schema', schema)

+        with ServerInfo(api.env.hostname) as server_info:

+            schema = Schema(api, server_info, client)

+            object.__setattr__(api, '_schema', schema)

     fingerprint = str(schema['fingerprint'])

     package_name = '{}${}'.format(__name__, fingerprint)

@@ -509,10 +563,9 @@ def get_package(api, client):

     module = types.ModuleType(module_name)

     module.__file__ = os.path.join(package_dir, 'plugins.py')

     module.register = plugable.Registry()

-    for key, plugin_cls in (('commands', _SchemaCommandPlugin),

-                            ('classes', _SchemaObjectPlugin)):

-        for full_name in schema[key]:

-            plugin = plugin_cls(full_name)

+    for plugin_cls in (_SchemaCommandPlugin, _SchemaObjectPlugin):

+        for full_name in schema[plugin_cls.schema_key]:

+            plugin = plugin_cls(str(full_name))

             plugin = module.register()(plugin)

     sys.modules[module_name] = module

-- 

2.7.4