From 2618c2fbbd9e23f79a667ac373b0a828cdd5d643 Mon Sep 17 00:00:00 2001 From: David Kupka Date: Mon, 22 Aug 2016 13:34:30 +0200 Subject: [PATCH] schema cache: Store and check info for pre-schema servers Cache CommandError answer to schema command to avoid sending the command to pre-schema servers every time. This information expires after some time (1 hour) in order to start using schema as soon as the server is upgraded. https://fedorahosted.org/freeipa/ticket/6095 Signed-off-by: Jan Cholasta Signed-off-by: David Kupka Reviewed-By: Tomas Krizek --- ipaclient/remote_plugins/__init__.py | 80 +++++++++++++-------- ipaclient/remote_plugins/compat.py | 9 ++- ipaclient/remote_plugins/schema.py | 130 ++++++++++++++++++----------------- 3 files changed, 128 insertions(+), 91 deletions(-) diff --git a/ipaclient/remote_plugins/__init__.py b/ipaclient/remote_plugins/__init__.py index 2be9222be693a5c4a04a735c216f590d75c1ecfe..b783c32819e58f49532531b6d7f3a594c17bae16 100644 --- a/ipaclient/remote_plugins/__init__.py +++ b/ipaclient/remote_plugins/__init__.py @@ -5,7 +5,9 @@ import collections import errno import json +import locale import os +import time from . import compat from . import schema @@ -23,20 +25,18 @@ class ServerInfo(collections.MutableMapping): def __init__(self, api): hostname = DNSName(api.env.server).ToASCII() self._path = os.path.join(self._DIR, hostname) + self._force_check = api.env.force_schema_check self._dict = {} - self._dirty = False - self._read() - - def __enter__(self): - return self - - def __exit__(self, *_exc_info): - self.flush() + # copy-paste from ipalib/rpc.py + try: + self._language = ( + locale.setlocale(locale.LC_ALL, '').split('.')[0].lower() + ) + except locale.Error: + self._language = 'en_us' - def flush(self): - if self._dirty: - self._write() + self._read() def _read(self): try: @@ -62,13 +62,10 @@ class ServerInfo(collections.MutableMapping): return self._dict[key] def __setitem__(self, key, value): - if key not in self._dict or self._dict[key] != value: - self._dirty = True self._dict[key] = value def __delitem__(self, key): del self._dict[key] - self._dirty = True def __iter__(self): return iter(self._dict) @@ -76,26 +73,55 @@ class ServerInfo(collections.MutableMapping): def __len__(self): return len(self._dict) + def update_validity(self, ttl=None): + if ttl is None: + ttl = 3600 + self['expiration'] = time.time() + ttl + self['language'] = self._language + self._write() + + def is_valid(self): + if self._force_check: + return False + + try: + expiration = self._dict['expiration'] + language = self._dict['language'] + except KeyError: + # if any of these is missing consider the entry expired + return False + + if expiration < time.time(): + # validity passed + return False + + if language != self._language: + # language changed since last check + return False + + return True + def get_package(api): if api.env.in_tree: from ipaserver import plugins else: - client = rpcclient(api) - client.finalize() - try: - server_info = api._server_info + plugins = api._remote_plugins except AttributeError: - server_info = api._server_info = ServerInfo(api) + server_info = ServerInfo(api) - try: - plugins = schema.get_package(api, server_info, client) - except schema.NotAvailable: - plugins = compat.get_package(api, server_info, client) - finally: - server_info.flush() - if client.isconnected(): - client.disconnect() + client = rpcclient(api) + client.finalize() + + try: + plugins = schema.get_package(server_info, client) + except schema.NotAvailable: + plugins = compat.get_package(server_info, client) + finally: + if client.isconnected(): + client.disconnect() + + object.__setattr__(api, '_remote_plugins', plugins) return plugins diff --git a/ipaclient/remote_plugins/compat.py b/ipaclient/remote_plugins/compat.py index 5e08cb0ed73becbc17e724864d1a853142a5ef6f..984eecd3f86fada96084d70bbbeb81c3730346e8 100644 --- a/ipaclient/remote_plugins/compat.py +++ b/ipaclient/remote_plugins/compat.py @@ -31,10 +31,15 @@ class CompatObject(Object): pass -def get_package(api, server_info, client): +def get_package(server_info, client): try: server_version = server_info['version'] except KeyError: + is_valid = False + else: + is_valid = server_info.is_valid() + + if not is_valid: if not client.isconnected(): client.connect(verbose=False) env = client.forward(u'env', u'api_version', version=u'2.0') @@ -51,6 +56,8 @@ def get_package(api, server_info, client): else: server_version = u'2.0' server_info['version'] = server_version + server_info.update_validity() + server_version = LooseVersion(server_version) package_names = {} diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py index 553da35127188b1ae842a7a0b58433e632c82b9f..5634fd1c8fc9c4f9276b57eac2e4abecc8d7c792 100644 --- a/ipaclient/remote_plugins/schema.py +++ b/ipaclient/remote_plugins/schema.py @@ -7,10 +7,8 @@ import contextlib import errno import fcntl import json -import locale import os import sys -import time import types import zipfile @@ -220,7 +218,7 @@ class _SchemaPlugin(object): def __call__(self, api): if self._class is None: - schema = api._schema[self.schema_key][self.full_name] + schema = self._schema[self.schema_key][self.full_name] name, bases, class_dict = self._create_class(api, schema) self._class = type(name, bases, class_dict) @@ -361,7 +359,7 @@ class Schema(object): namespaces = {'classes', 'commands', 'topics'} _DIR = os.path.join(paths.USER_CACHE_PATH, 'ipa', 'schema', FORMAT) - def __init__(self, api, server_info, client): + def __init__(self, client, fingerprint=None): self._dict = {} self._namespaces = {} self._help = None @@ -371,48 +369,29 @@ class Schema(object): self._dict[ns] = {} self._namespaces[ns] = _SchemaNameSpace(self, ns) - # copy-paste from ipalib/rpc.py - try: - self._language = ( - locale.setlocale(locale.LC_ALL, '').split('.')[0].lower() - ) - except locale.Error: - # fallback to default locale - self._language = 'en_us' - - try: - self._fingerprint = server_info['fingerprint'] - self._expiration = server_info['expiration'] - language = server_info['language'] - except KeyError: - is_known = False - else: - is_known = (not api.env.force_schema_check and - self._expiration > time.time() and - self._language == language) + ttl = None + read_failed = False - if is_known: + if fingerprint is not None: try: - self._read_schema() - except Exception: - pass - else: - return - - try: - self._fetch(client) - except NotAvailable: - raise - except SchemaUpToDate as e: - self._fingerprint = e.fingerprint - self._expiration = time.time() + e.ttl - self._read_schema() - else: - self._write_schema() + self._read_schema(fingerprint) + except Exception as e: + # Failed to read the schema from cache. There may be a lot of + # causes and not much we can do about it. Just ensure we will + # ignore the cache and fetch the schema from server. + logger.warning("Failed to read schema: {}".format(e)) + fingerprint = None + read_failed = True + + if fingerprint is None: + fingerprint, ttl = self._fetch(client, ignore_cache=read_failed) + try: + self._write_schema(fingerprint) + except Exception as e: + logger.warning("Failed to write schema: {}".format(e)) - server_info['fingerprint'] = self._fingerprint - server_info['expiration'] = self._expiration - server_info['language'] = self._language + self.fingerprint = fingerprint + self.ttl = ttl @contextlib.contextmanager def _open(self, filename, mode): @@ -429,14 +408,16 @@ class Schema(object): finally: fcntl.flock(f, fcntl.LOCK_UN) - def _fetch(self, client): + def _fetch(self, client, ignore_cache=False): if not client.isconnected(): client.connect(verbose=False) - try: - fps = [fsdecode(f) for f in os.listdir(self._DIR)] - except EnvironmentError: - fps = [] + fps = [] + if not ignore_cache: + try: + fps = [fsdecode(f) for f in os.listdir(self._DIR)] + except EnvironmentError: + pass kwargs = {u'version': u'2.170'} if fps: @@ -459,12 +440,11 @@ class Schema(object): logger.warning("Failed to fetch schema: %s", e) raise NotAvailable() - self._fingerprint = fp - self._expiration = time.time() + ttl + return (fp, ttl,) - def _read_schema(self): + def _read_schema(self, fingerprint): self._file.truncate(0) - with self._open(self._fingerprint, 'r') as f: + with self._open(fingerprint, 'r') as f: self._file.write(f.read()) with zipfile.ZipFile(self._file, 'r') as schema: @@ -500,13 +480,12 @@ class Schema(object): return halp - def _write_schema(self): + def _write_schema(self, fingerprint): try: os.makedirs(self._DIR) except EnvironmentError as e: if e.errno != errno.EEXIST: - logger.warning("Failed to write schema: {}".format(e)) - return + raise self._file.truncate(0) with zipfile.ZipFile(self._file, 'w', zipfile.ZIP_DEFLATED) as schema: @@ -523,7 +502,7 @@ class Schema(object): json.dumps(self._generate_help(self._dict))) self._file.seek(0) - with self._open(self._fingerprint, 'w') as f: + with self._open(fingerprint, 'w') as f: f.truncate(0) f.write(self._file.read()) @@ -550,14 +529,39 @@ class Schema(object): return self._help[namespace][member] -def get_package(api, server_info, client): - try: - schema = api._schema - except AttributeError: - schema = Schema(api, server_info, client) - object.__setattr__(api, '_schema', schema) +def get_package(server_info, client): + NO_FINGERPRINT = object() + + fingerprint = NO_FINGERPRINT + if server_info.is_valid(): + fingerprint = server_info.get('fingerprint', fingerprint) + + if fingerprint is not None: + try: + try: + if fingerprint is NO_FINGERPRINT: + schema = Schema(client) + else: + schema = Schema(client, fingerprint) + except SchemaUpToDate as e: + schema = Schema(client, e.fingerprint) + except NotAvailable: + fingerprint = None + ttl = None + except SchemaUpToDate as e: + fingerprint = e.fingerprint + ttl = e.ttl + else: + fingerprint = schema.fingerprint + ttl = schema.ttl + + server_info['fingerprint'] = fingerprint + server_info.update_validity(ttl) + + if fingerprint is None: + raise NotAvailable() - fingerprint = str(server_info['fingerprint']) + fingerprint = str(fingerprint) package_name = '{}${}'.format(__name__, fingerprint) package_dir = '{}${}'.format(os.path.splitext(__file__)[0], fingerprint) -- 2.7.4