From 2618c2fbbd9e23f79a667ac373b0a828cdd5d643 Mon Sep 17 00:00:00 2001

From: David Kupka <dkupka@redhat.com>

Date: Mon, 22 Aug 2016 13:34:30 +0200

Subject: [PATCH] schema cache: Store and check info for pre-schema servers

Cache CommandError answer to schema command to avoid sending the command

to pre-schema servers every time. This information expires after some

time (1 hour) in order to start using schema as soon as the server is

upgraded.

https://fedorahosted.org/freeipa/ticket/6095

Signed-off-by: Jan Cholasta <jcholast@redhat.com>

Signed-off-by: David Kupka <dkupka@redhat.com>

Reviewed-By: Tomas Krizek <tkrizek@redhat.com>

---

 ipaclient/remote_plugins/__init__.py |  80 +++++++++++++--------

 ipaclient/remote_plugins/compat.py   |   9 ++-

 ipaclient/remote_plugins/schema.py   | 130 ++++++++++++++++++-----------------

 3 files changed, 128 insertions(+), 91 deletions(-)

diff --git a/ipaclient/remote_plugins/__init__.py b/ipaclient/remote_plugins/__init__.py

index 2be9222be693a5c4a04a735c216f590d75c1ecfe..b783c32819e58f49532531b6d7f3a594c17bae16 100644

--- a/ipaclient/remote_plugins/__init__.py

+++ b/ipaclient/remote_plugins/__init__.py

@@ -5,7 +5,9 @@

 import collections

 import errno

 import json

+import locale

 import os

+import time

 from . import compat

 from . import schema

@@ -23,20 +25,18 @@ class ServerInfo(collections.MutableMapping):

     def __init__(self, api):

         hostname = DNSName(api.env.server).ToASCII()

         self._path = os.path.join(self._DIR, hostname)

+        self._force_check = api.env.force_schema_check

         self._dict = {}

-        self._dirty = False

-        self._read()

-

-    def __enter__(self):

-        return self

-

-    def __exit__(self, *_exc_info):

-        self.flush()

+        # copy-paste from ipalib/rpc.py

+        try:

+            self._language = (

+                 locale.setlocale(locale.LC_ALL, '').split('.')[0].lower()

+            )

+        except locale.Error:

+            self._language = 'en_us'

-    def flush(self):

-        if self._dirty:

-            self._write()

+        self._read()

     def _read(self):

         try:

@@ -62,13 +62,10 @@ class ServerInfo(collections.MutableMapping):

         return self._dict[key]

     def __setitem__(self, key, value):

-        if key not in self._dict or self._dict[key] != value:

-            self._dirty = True

         self._dict[key] = value

     def __delitem__(self, key):

         del self._dict[key]

-        self._dirty = True

     def __iter__(self):

         return iter(self._dict)

@@ -76,26 +73,55 @@ class ServerInfo(collections.MutableMapping):

     def __len__(self):

         return len(self._dict)

+    def update_validity(self, ttl=None):

+        if ttl is None:

+            ttl = 3600

+        self['expiration'] = time.time() + ttl

+        self['language'] = self._language

+        self._write()

+

+    def is_valid(self):

+        if self._force_check:

+            return False

+

+        try:

+            expiration = self._dict['expiration']

+            language = self._dict['language']

+        except KeyError:

+            # if any of these is missing consider the entry expired

+            return False

+

+        if expiration < time.time():

+            # validity passed

+            return False

+

+        if language != self._language:

+            # language changed since last check

+            return False

+

+        return True

+

 def get_package(api):

     if api.env.in_tree:

         from ipaserver import plugins

     else:

-        client = rpcclient(api)

-        client.finalize()

-

         try:

-            server_info = api._server_info

+            plugins = api._remote_plugins

         except AttributeError:

-            server_info = api._server_info = ServerInfo(api)

+            server_info = ServerInfo(api)

-        try:

-            plugins = schema.get_package(api, server_info, client)

-        except schema.NotAvailable:

-            plugins = compat.get_package(api, server_info, client)

-        finally:

-            server_info.flush()

-            if client.isconnected():

-                client.disconnect()

+            client = rpcclient(api)

+            client.finalize()

+

+            try:

+                plugins = schema.get_package(server_info, client)

+            except schema.NotAvailable:

+                plugins = compat.get_package(server_info, client)

+            finally:

+                if client.isconnected():

+                    client.disconnect()

+

+            object.__setattr__(api, '_remote_plugins', plugins)

     return plugins

diff --git a/ipaclient/remote_plugins/compat.py b/ipaclient/remote_plugins/compat.py

index 5e08cb0ed73becbc17e724864d1a853142a5ef6f..984eecd3f86fada96084d70bbbeb81c3730346e8 100644

--- a/ipaclient/remote_plugins/compat.py

+++ b/ipaclient/remote_plugins/compat.py

@@ -31,10 +31,15 @@ class CompatObject(Object):

     pass

-def get_package(api, server_info, client):

+def get_package(server_info, client):

     try:

         server_version = server_info['version']

     except KeyError:

+        is_valid = False

+    else:

+        is_valid = server_info.is_valid()

+

+    if not is_valid:

         if not client.isconnected():

             client.connect(verbose=False)

         env = client.forward(u'env', u'api_version', version=u'2.0')

@@ -51,6 +56,8 @@ def get_package(api, server_info, client):

             else:

                 server_version = u'2.0'

         server_info['version'] = server_version

+        server_info.update_validity()

+

     server_version = LooseVersion(server_version)

     package_names = {}

diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py

index 553da35127188b1ae842a7a0b58433e632c82b9f..5634fd1c8fc9c4f9276b57eac2e4abecc8d7c792 100644

--- a/ipaclient/remote_plugins/schema.py

+++ b/ipaclient/remote_plugins/schema.py

@@ -7,10 +7,8 @@ import contextlib

 import errno

 import fcntl

 import json

-import locale

 import os

 import sys

-import time

 import types

 import zipfile

@@ -220,7 +218,7 @@ class _SchemaPlugin(object):

     def __call__(self, api):

         if self._class is None:

-            schema = api._schema[self.schema_key][self.full_name]

+            schema = self._schema[self.schema_key][self.full_name]

             name, bases, class_dict = self._create_class(api, schema)

             self._class = type(name, bases, class_dict)

@@ -361,7 +359,7 @@ class Schema(object):

     namespaces = {'classes', 'commands', 'topics'}

     _DIR = os.path.join(paths.USER_CACHE_PATH, 'ipa', 'schema', FORMAT)

-    def __init__(self, api, server_info, client):

+    def __init__(self, client, fingerprint=None):

         self._dict = {}

         self._namespaces = {}

         self._help = None

@@ -371,48 +369,29 @@ class Schema(object):

             self._dict[ns] = {}

             self._namespaces[ns] = _SchemaNameSpace(self, ns)

-        # copy-paste from ipalib/rpc.py

-        try:

-            self._language = (

-                locale.setlocale(locale.LC_ALL, '').split('.')[0].lower()

-            )

-        except locale.Error:

-            # fallback to default locale

-            self._language = 'en_us'

-

-        try:

-            self._fingerprint = server_info['fingerprint']

-            self._expiration = server_info['expiration']

-            language = server_info['language']

-        except KeyError:

-            is_known = False

-        else:

-            is_known = (not api.env.force_schema_check and

-                        self._expiration > time.time() and

-                        self._language == language)

+        ttl = None

+        read_failed = False

-        if is_known:

+        if fingerprint is not None:

             try:

-                self._read_schema()

-            except Exception:

-                pass

-            else:

-                return

-

-        try:

-            self._fetch(client)

-        except NotAvailable:

-            raise

-        except SchemaUpToDate as e:

-            self._fingerprint = e.fingerprint

-            self._expiration = time.time() + e.ttl

-            self._read_schema()

-        else:

-            self._write_schema()

+                self._read_schema(fingerprint)

+            except Exception as e:

+                # Failed to read the schema from cache. There may be a lot of

+                # causes and not much we can do about it. Just ensure we will

+                # ignore the cache and fetch the schema from server.

+                logger.warning("Failed to read schema: {}".format(e))

+                fingerprint = None

+                read_failed = True

+

+        if fingerprint is None:

+            fingerprint, ttl = self._fetch(client, ignore_cache=read_failed)

+            try:

+                self._write_schema(fingerprint)

+            except Exception as e:

+                logger.warning("Failed to write schema: {}".format(e))

-        server_info['fingerprint'] = self._fingerprint

-        server_info['expiration'] = self._expiration

-        server_info['language'] = self._language

+        self.fingerprint = fingerprint

+        self.ttl = ttl

     @contextlib.contextmanager

     def _open(self, filename, mode):

@@ -429,14 +408,16 @@ class Schema(object):

             finally:

                 fcntl.flock(f, fcntl.LOCK_UN)

-    def _fetch(self, client):

+    def _fetch(self, client, ignore_cache=False):

         if not client.isconnected():

             client.connect(verbose=False)

-        try:

-            fps = [fsdecode(f) for f in os.listdir(self._DIR)]

-        except EnvironmentError:

-            fps = []

+        fps = []

+        if not ignore_cache:

+            try:

+                fps = [fsdecode(f) for f in os.listdir(self._DIR)]

+            except EnvironmentError:

+                pass

         kwargs = {u'version': u'2.170'}

         if fps:

@@ -459,12 +440,11 @@ class Schema(object):

             logger.warning("Failed to fetch schema: %s", e)

             raise NotAvailable()

-        self._fingerprint = fp

-        self._expiration = time.time() + ttl

+        return (fp, ttl,)

-    def _read_schema(self):

+    def _read_schema(self, fingerprint):

         self._file.truncate(0)

-        with self._open(self._fingerprint, 'r') as f:

+        with self._open(fingerprint, 'r') as f:

             self._file.write(f.read())

         with zipfile.ZipFile(self._file, 'r') as schema:

@@ -500,13 +480,12 @@ class Schema(object):

         return halp

-    def _write_schema(self):

+    def _write_schema(self, fingerprint):

         try:

             os.makedirs(self._DIR)

         except EnvironmentError as e:

             if e.errno != errno.EEXIST:

-                logger.warning("Failed to write schema: {}".format(e))

-                return

+                raise

         self._file.truncate(0)

         with zipfile.ZipFile(self._file, 'w', zipfile.ZIP_DEFLATED) as schema:

@@ -523,7 +502,7 @@ class Schema(object):

                             json.dumps(self._generate_help(self._dict)))

         self._file.seek(0)

-        with self._open(self._fingerprint, 'w') as f:

+        with self._open(fingerprint, 'w') as f:

             f.truncate(0)

             f.write(self._file.read())

@@ -550,14 +529,39 @@ class Schema(object):

         return self._help[namespace][member]

-def get_package(api, server_info, client):

-    try:

-        schema = api._schema

-    except AttributeError:

-        schema = Schema(api, server_info, client)

-        object.__setattr__(api, '_schema', schema)

+def get_package(server_info, client):

+    NO_FINGERPRINT = object()

+

+    fingerprint = NO_FINGERPRINT

+    if server_info.is_valid():

+        fingerprint = server_info.get('fingerprint', fingerprint)

+

+    if fingerprint is not None:

+        try:

+            try:

+                if fingerprint is NO_FINGERPRINT:

+                    schema = Schema(client)

+                else:

+                    schema = Schema(client, fingerprint)

+            except SchemaUpToDate as e:

+                schema = Schema(client, e.fingerprint)

+        except NotAvailable:

+            fingerprint = None

+            ttl = None

+        except SchemaUpToDate as e:

+            fingerprint = e.fingerprint

+            ttl = e.ttl

+        else:

+            fingerprint = schema.fingerprint

+            ttl = schema.ttl

+

+        server_info['fingerprint'] = fingerprint

+        server_info.update_validity(ttl)

+

+    if fingerprint is None:

+        raise NotAvailable()

-    fingerprint = str(server_info['fingerprint'])

+    fingerprint = str(fingerprint)

     package_name = '{}${}'.format(__name__, fingerprint)

     package_dir = '{}${}'.format(os.path.splitext(__file__)[0], fingerprint)

-- 

2.7.4