403b09
From 2618c2fbbd9e23f79a667ac373b0a828cdd5d643 Mon Sep 17 00:00:00 2001
403b09
From: David Kupka <dkupka@redhat.com>
403b09
Date: Mon, 22 Aug 2016 13:34:30 +0200
403b09
Subject: [PATCH] schema cache: Store and check info for pre-schema servers
403b09
403b09
Cache CommandError answer to schema command to avoid sending the command
403b09
to pre-schema servers every time. This information expires after some
403b09
time (1 hour) in order to start using schema as soon as the server is
403b09
upgraded.
403b09
403b09
https://fedorahosted.org/freeipa/ticket/6095
403b09
403b09
Signed-off-by: Jan Cholasta <jcholast@redhat.com>
403b09
Signed-off-by: David Kupka <dkupka@redhat.com>
403b09
Reviewed-By: Tomas Krizek <tkrizek@redhat.com>
403b09
---
403b09
 ipaclient/remote_plugins/__init__.py |  80 +++++++++++++--------
403b09
 ipaclient/remote_plugins/compat.py   |   9 ++-
403b09
 ipaclient/remote_plugins/schema.py   | 130 ++++++++++++++++++-----------------
403b09
 3 files changed, 128 insertions(+), 91 deletions(-)
403b09
403b09
diff --git a/ipaclient/remote_plugins/__init__.py b/ipaclient/remote_plugins/__init__.py
403b09
index 2be9222be693a5c4a04a735c216f590d75c1ecfe..b783c32819e58f49532531b6d7f3a594c17bae16 100644
403b09
--- a/ipaclient/remote_plugins/__init__.py
403b09
+++ b/ipaclient/remote_plugins/__init__.py
403b09
@@ -5,7 +5,9 @@
403b09
 import collections
403b09
 import errno
403b09
 import json
403b09
+import locale
403b09
 import os
403b09
+import time
403b09
 
403b09
 from . import compat
403b09
 from . import schema
403b09
@@ -23,20 +25,18 @@ class ServerInfo(collections.MutableMapping):
403b09
     def __init__(self, api):
403b09
         hostname = DNSName(api.env.server).ToASCII()
403b09
         self._path = os.path.join(self._DIR, hostname)
403b09
+        self._force_check = api.env.force_schema_check
403b09
         self._dict = {}
403b09
-        self._dirty = False
403b09
 
403b09
-        self._read()
403b09
-
403b09
-    def __enter__(self):
403b09
-        return self
403b09
-
403b09
-    def __exit__(self, *_exc_info):
403b09
-        self.flush()
403b09
+        # copy-paste from ipalib/rpc.py
403b09
+        try:
403b09
+            self._language = (
403b09
+                 locale.setlocale(locale.LC_ALL, '').split('.')[0].lower()
403b09
+            )
403b09
+        except locale.Error:
403b09
+            self._language = 'en_us'
403b09
 
403b09
-    def flush(self):
403b09
-        if self._dirty:
403b09
-            self._write()
403b09
+        self._read()
403b09
 
403b09
     def _read(self):
403b09
         try:
403b09
@@ -62,13 +62,10 @@ class ServerInfo(collections.MutableMapping):
403b09
         return self._dict[key]
403b09
 
403b09
     def __setitem__(self, key, value):
403b09
-        if key not in self._dict or self._dict[key] != value:
403b09
-            self._dirty = True
403b09
         self._dict[key] = value
403b09
 
403b09
     def __delitem__(self, key):
403b09
         del self._dict[key]
403b09
-        self._dirty = True
403b09
 
403b09
     def __iter__(self):
403b09
         return iter(self._dict)
403b09
@@ -76,26 +73,55 @@ class ServerInfo(collections.MutableMapping):
403b09
     def __len__(self):
403b09
         return len(self._dict)
403b09
 
403b09
+    def update_validity(self, ttl=None):
403b09
+        if ttl is None:
403b09
+            ttl = 3600
403b09
+        self['expiration'] = time.time() + ttl
403b09
+        self['language'] = self._language
403b09
+        self._write()
403b09
+
403b09
+    def is_valid(self):
403b09
+        if self._force_check:
403b09
+            return False
403b09
+
403b09
+        try:
403b09
+            expiration = self._dict['expiration']
403b09
+            language = self._dict['language']
403b09
+        except KeyError:
403b09
+            # if any of these is missing consider the entry expired
403b09
+            return False
403b09
+
403b09
+        if expiration < time.time():
403b09
+            # validity passed
403b09
+            return False
403b09
+
403b09
+        if language != self._language:
403b09
+            # language changed since last check
403b09
+            return False
403b09
+
403b09
+        return True
403b09
+
403b09
 
403b09
 def get_package(api):
403b09
     if api.env.in_tree:
403b09
         from ipaserver import plugins
403b09
     else:
403b09
-        client = rpcclient(api)
403b09
-        client.finalize()
403b09
-
403b09
         try:
403b09
-            server_info = api._server_info
403b09
+            plugins = api._remote_plugins
403b09
         except AttributeError:
403b09
-            server_info = api._server_info = ServerInfo(api)
403b09
+            server_info = ServerInfo(api)
403b09
 
403b09
-        try:
403b09
-            plugins = schema.get_package(api, server_info, client)
403b09
-        except schema.NotAvailable:
403b09
-            plugins = compat.get_package(api, server_info, client)
403b09
-        finally:
403b09
-            server_info.flush()
403b09
-            if client.isconnected():
403b09
-                client.disconnect()
403b09
+            client = rpcclient(api)
403b09
+            client.finalize()
403b09
+
403b09
+            try:
403b09
+                plugins = schema.get_package(server_info, client)
403b09
+            except schema.NotAvailable:
403b09
+                plugins = compat.get_package(server_info, client)
403b09
+            finally:
403b09
+                if client.isconnected():
403b09
+                    client.disconnect()
403b09
+
403b09
+            object.__setattr__(api, '_remote_plugins', plugins)
403b09
 
403b09
     return plugins
403b09
diff --git a/ipaclient/remote_plugins/compat.py b/ipaclient/remote_plugins/compat.py
403b09
index 5e08cb0ed73becbc17e724864d1a853142a5ef6f..984eecd3f86fada96084d70bbbeb81c3730346e8 100644
403b09
--- a/ipaclient/remote_plugins/compat.py
403b09
+++ b/ipaclient/remote_plugins/compat.py
403b09
@@ -31,10 +31,15 @@ class CompatObject(Object):
403b09
     pass
403b09
 
403b09
 
403b09
-def get_package(api, server_info, client):
403b09
+def get_package(server_info, client):
403b09
     try:
403b09
         server_version = server_info['version']
403b09
     except KeyError:
403b09
+        is_valid = False
403b09
+    else:
403b09
+        is_valid = server_info.is_valid()
403b09
+
403b09
+    if not is_valid:
403b09
         if not client.isconnected():
403b09
             client.connect(verbose=False)
403b09
         env = client.forward(u'env', u'api_version', version=u'2.0')
403b09
@@ -51,6 +56,8 @@ def get_package(api, server_info, client):
403b09
             else:
403b09
                 server_version = u'2.0'
403b09
         server_info['version'] = server_version
403b09
+        server_info.update_validity()
403b09
+
403b09
     server_version = LooseVersion(server_version)
403b09
 
403b09
     package_names = {}
403b09
diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py
403b09
index 553da35127188b1ae842a7a0b58433e632c82b9f..5634fd1c8fc9c4f9276b57eac2e4abecc8d7c792 100644
403b09
--- a/ipaclient/remote_plugins/schema.py
403b09
+++ b/ipaclient/remote_plugins/schema.py
403b09
@@ -7,10 +7,8 @@ import contextlib
403b09
 import errno
403b09
 import fcntl
403b09
 import json
403b09
-import locale
403b09
 import os
403b09
 import sys
403b09
-import time
403b09
 import types
403b09
 import zipfile
403b09
 
403b09
@@ -220,7 +218,7 @@ class _SchemaPlugin(object):
403b09
 
403b09
     def __call__(self, api):
403b09
         if self._class is None:
403b09
-            schema = api._schema[self.schema_key][self.full_name]
403b09
+            schema = self._schema[self.schema_key][self.full_name]
403b09
             name, bases, class_dict = self._create_class(api, schema)
403b09
             self._class = type(name, bases, class_dict)
403b09
 
403b09
@@ -361,7 +359,7 @@ class Schema(object):
403b09
     namespaces = {'classes', 'commands', 'topics'}
403b09
     _DIR = os.path.join(paths.USER_CACHE_PATH, 'ipa', 'schema', FORMAT)
403b09
 
403b09
-    def __init__(self, api, server_info, client):
403b09
+    def __init__(self, client, fingerprint=None):
403b09
         self._dict = {}
403b09
         self._namespaces = {}
403b09
         self._help = None
403b09
@@ -371,48 +369,29 @@ class Schema(object):
403b09
             self._dict[ns] = {}
403b09
             self._namespaces[ns] = _SchemaNameSpace(self, ns)
403b09
 
403b09
-        # copy-paste from ipalib/rpc.py
403b09
-        try:
403b09
-            self._language = (
403b09
-                locale.setlocale(locale.LC_ALL, '').split('.')[0].lower()
403b09
-            )
403b09
-        except locale.Error:
403b09
-            # fallback to default locale
403b09
-            self._language = 'en_us'
403b09
-
403b09
-        try:
403b09
-            self._fingerprint = server_info['fingerprint']
403b09
-            self._expiration = server_info['expiration']
403b09
-            language = server_info['language']
403b09
-        except KeyError:
403b09
-            is_known = False
403b09
-        else:
403b09
-            is_known = (not api.env.force_schema_check and
403b09
-                        self._expiration > time.time() and
403b09
-                        self._language == language)
403b09
+        ttl = None
403b09
+        read_failed = False
403b09
 
403b09
-        if is_known:
403b09
+        if fingerprint is not None:
403b09
             try:
403b09
-                self._read_schema()
403b09
-            except Exception:
403b09
-                pass
403b09
-            else:
403b09
-                return
403b09
-
403b09
-        try:
403b09
-            self._fetch(client)
403b09
-        except NotAvailable:
403b09
-            raise
403b09
-        except SchemaUpToDate as e:
403b09
-            self._fingerprint = e.fingerprint
403b09
-            self._expiration = time.time() + e.ttl
403b09
-            self._read_schema()
403b09
-        else:
403b09
-            self._write_schema()
403b09
+                self._read_schema(fingerprint)
403b09
+            except Exception as e:
403b09
+                # Failed to read the schema from cache. There may be a lot of
403b09
+                # causes and not much we can do about it. Just ensure we will
403b09
+                # ignore the cache and fetch the schema from server.
403b09
+                logger.warning("Failed to read schema: {}".format(e))
403b09
+                fingerprint = None
403b09
+                read_failed = True
403b09
+
403b09
+        if fingerprint is None:
403b09
+            fingerprint, ttl = self._fetch(client, ignore_cache=read_failed)
403b09
+            try:
403b09
+                self._write_schema(fingerprint)
403b09
+            except Exception as e:
403b09
+                logger.warning("Failed to write schema: {}".format(e))
403b09
 
403b09
-        server_info['fingerprint'] = self._fingerprint
403b09
-        server_info['expiration'] = self._expiration
403b09
-        server_info['language'] = self._language
403b09
+        self.fingerprint = fingerprint
403b09
+        self.ttl = ttl
403b09
 
403b09
     @contextlib.contextmanager
403b09
     def _open(self, filename, mode):
403b09
@@ -429,14 +408,16 @@ class Schema(object):
403b09
             finally:
403b09
                 fcntl.flock(f, fcntl.LOCK_UN)
403b09
 
403b09
-    def _fetch(self, client):
403b09
+    def _fetch(self, client, ignore_cache=False):
403b09
         if not client.isconnected():
403b09
             client.connect(verbose=False)
403b09
 
403b09
-        try:
403b09
-            fps = [fsdecode(f) for f in os.listdir(self._DIR)]
403b09
-        except EnvironmentError:
403b09
-            fps = []
403b09
+        fps = []
403b09
+        if not ignore_cache:
403b09
+            try:
403b09
+                fps = [fsdecode(f) for f in os.listdir(self._DIR)]
403b09
+            except EnvironmentError:
403b09
+                pass
403b09
 
403b09
         kwargs = {u'version': u'2.170'}
403b09
         if fps:
403b09
@@ -459,12 +440,11 @@ class Schema(object):
403b09
             logger.warning("Failed to fetch schema: %s", e)
403b09
             raise NotAvailable()
403b09
 
403b09
-        self._fingerprint = fp
403b09
-        self._expiration = time.time() + ttl
403b09
+        return (fp, ttl,)
403b09
 
403b09
-    def _read_schema(self):
403b09
+    def _read_schema(self, fingerprint):
403b09
         self._file.truncate(0)
403b09
-        with self._open(self._fingerprint, 'r') as f:
403b09
+        with self._open(fingerprint, 'r') as f:
403b09
             self._file.write(f.read())
403b09
 
403b09
         with zipfile.ZipFile(self._file, 'r') as schema:
403b09
@@ -500,13 +480,12 @@ class Schema(object):
403b09
 
403b09
         return halp
403b09
 
403b09
-    def _write_schema(self):
403b09
+    def _write_schema(self, fingerprint):
403b09
         try:
403b09
             os.makedirs(self._DIR)
403b09
         except EnvironmentError as e:
403b09
             if e.errno != errno.EEXIST:
403b09
-                logger.warning("Failed to write schema: {}".format(e))
403b09
-                return
403b09
+                raise
403b09
 
403b09
         self._file.truncate(0)
403b09
         with zipfile.ZipFile(self._file, 'w', zipfile.ZIP_DEFLATED) as schema:
403b09
@@ -523,7 +502,7 @@ class Schema(object):
403b09
                             json.dumps(self._generate_help(self._dict)))
403b09
 
403b09
         self._file.seek(0)
403b09
-        with self._open(self._fingerprint, 'w') as f:
403b09
+        with self._open(fingerprint, 'w') as f:
403b09
             f.truncate(0)
403b09
             f.write(self._file.read())
403b09
 
403b09
@@ -550,14 +529,39 @@ class Schema(object):
403b09
         return self._help[namespace][member]
403b09
 
403b09
 
403b09
-def get_package(api, server_info, client):
403b09
-    try:
403b09
-        schema = api._schema
403b09
-    except AttributeError:
403b09
-        schema = Schema(api, server_info, client)
403b09
-        object.__setattr__(api, '_schema', schema)
403b09
+def get_package(server_info, client):
403b09
+    NO_FINGERPRINT = object()
403b09
+
403b09
+    fingerprint = NO_FINGERPRINT
403b09
+    if server_info.is_valid():
403b09
+        fingerprint = server_info.get('fingerprint', fingerprint)
403b09
+
403b09
+    if fingerprint is not None:
403b09
+        try:
403b09
+            try:
403b09
+                if fingerprint is NO_FINGERPRINT:
403b09
+                    schema = Schema(client)
403b09
+                else:
403b09
+                    schema = Schema(client, fingerprint)
403b09
+            except SchemaUpToDate as e:
403b09
+                schema = Schema(client, e.fingerprint)
403b09
+        except NotAvailable:
403b09
+            fingerprint = None
403b09
+            ttl = None
403b09
+        except SchemaUpToDate as e:
403b09
+            fingerprint = e.fingerprint
403b09
+            ttl = e.ttl
403b09
+        else:
403b09
+            fingerprint = schema.fingerprint
403b09
+            ttl = schema.ttl
403b09
+
403b09
+        server_info['fingerprint'] = fingerprint
403b09
+        server_info.update_validity(ttl)
403b09
+
403b09
+    if fingerprint is None:
403b09
+        raise NotAvailable()
403b09
 
403b09
-    fingerprint = str(server_info['fingerprint'])
403b09
+    fingerprint = str(fingerprint)
403b09
     package_name = '{}${}'.format(__name__, fingerprint)
403b09
     package_dir = '{}${}'.format(os.path.splitext(__file__)[0], fingerprint)
403b09
 
403b09
-- 
403b09
2.7.4
403b09