From b226448134b5182ba685702e7b7a486db772d956 Mon Sep 17 00:00:00 2001

From: Emanuele Giuseppe Esposito <eesposit@redhat.com>

Date: Fri, 4 Mar 2022 11:21:16 +0100

Subject: [PATCH 1/2] - Detect a Python version change and clear the cache

 (#857)

RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>

RH-MergeRequest: 54: - Detect a Python version change and clear the cache (#857)

RH-Commit: [1/2] c562cd802eabae9dc14079de0b26d471d2229ca8

RH-Bugzilla: 1935826

RH-Acked-by: Eduardo Otubo <otubo@redhat.com>

RH-Acked-by: Vitaly Kuznetsov <vkuznets@redhat.com>

RH-Acked-by: Mohamed Gamal Morsy <mmorsy@redhat.com>

commit 78e89b03ecb29e7df3181b1219a0b5f44b9d7532

Author: Robert Schweikert <rjschwei@suse.com>

Date:   Thu Jul 1 12:35:40 2021 -0400

    - Detect a Python version change and clear the cache (#857)

    summary: Clear cache when a Python version change is detected

    When a distribution gets updated it is possible that the Python version

    changes. Python makes no guarantee that pickle is consistent across

    versions as such we need to purge the cache and start over.

    Co-authored-by: James Falcon <therealfalcon@gmail.com>

Conflicts:

   tests/integration_tests/util.py: test is not present downstream

Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

---

 cloudinit/cmd/main.py                         |  30 ++++++++++

 cloudinit/cmd/tests/test_main.py              |   2 +

 .../assets/test_version_change.pkl            | Bin 0 -> 21 bytes

 .../modules/test_ssh_auth_key_fingerprints.py |   2 +-

 .../modules/test_version_change.py            |  56 ++++++++++++++++++

 5 files changed, 89 insertions(+), 1 deletion(-)

 create mode 100644 tests/integration_tests/assets/test_version_change.pkl

 create mode 100644 tests/integration_tests/modules/test_version_change.py

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py

index baf1381f..21213a4a 100644

--- a/cloudinit/cmd/main.py

+++ b/cloudinit/cmd/main.py

@@ -210,6 +210,35 @@ def attempt_cmdline_url(path, network=True, cmdline=None):

             (cmdline_name, url, path))

+def purge_cache_on_python_version_change(init):

+    """Purge the cache if python version changed on us.

+

+    There could be changes not represented in our cache (obj.pkl) after we

+    upgrade to a new version of python, so at that point clear the cache

+    """

+    current_python_version = '%d.%d' % (

+        sys.version_info.major, sys.version_info.minor

+    )

+    python_version_path = os.path.join(

+        init.paths.get_cpath('data'), 'python-version'

+    )

+    if os.path.exists(python_version_path):

+        cached_python_version = open(python_version_path).read()

+        # The Python version has changed out from under us, anything that was

+        # pickled previously is likely useless due to API changes.

+        if cached_python_version != current_python_version:

+            LOG.debug('Python version change detected. Purging cache')

+            init.purge_cache(True)

+            util.write_file(python_version_path, current_python_version)

+    else:

+        if os.path.exists(init.paths.get_ipath_cur('obj_pkl')):

+            LOG.info(

+                'Writing python-version file. '

+                'Cache compatibility status is currently unknown.'

+            )

+        util.write_file(python_version_path, current_python_version)

+

+

 def main_init(name, args):

     deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]

     if args.local:

@@ -276,6 +305,7 @@ def main_init(name, args):

         util.logexc(LOG, "Failed to initialize, likely bad things to come!")

     # Stage 4

     path_helper = init.paths

+    purge_cache_on_python_version_change(init)

     mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK

     if mode == sources.DSMODE_NETWORK:

diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py

index 78b27441..1f5975b0 100644

--- a/cloudinit/cmd/tests/test_main.py

+++ b/cloudinit/cmd/tests/test_main.py

@@ -17,6 +17,8 @@ myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand')

 class TestMain(FilesystemMockingTestCase):

+    with_logs = True

+    allowed_subp = False

     def setUp(self):

         super(TestMain, self).setUp()

diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py

index b9b0d85e..e1946cb1 100644

--- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py

+++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py

@@ -18,7 +18,7 @@ USER_DATA_SSH_AUTHKEY_DISABLE = """\

 no_ssh_fingerprints: true

 """

-USER_DATA_SSH_AUTHKEY_ENABLE="""\

+USER_DATA_SSH_AUTHKEY_ENABLE = """\

 #cloud-config

 ssh_genkeytypes:

   - ecdsa

diff --git a/tests/integration_tests/modules/test_version_change.py b/tests/integration_tests/modules/test_version_change.py

new file mode 100644

index 00000000..4e9ab63f

--- /dev/null

+++ b/tests/integration_tests/modules/test_version_change.py

@@ -0,0 +1,56 @@

+from pathlib import Path

+

+from tests.integration_tests.instances import IntegrationInstance

+from tests.integration_tests.util import ASSETS_DIR

+

+

+PICKLE_PATH = Path('/var/lib/cloud/instance/obj.pkl')

+TEST_PICKLE = ASSETS_DIR / 'test_version_change.pkl'

+

+

+def _assert_no_pickle_problems(log):

+    assert 'Failed loading pickled blob' not in log

+    assert 'Traceback' not in log

+    assert 'WARN' not in log

+

+

+def test_reboot_without_version_change(client: IntegrationInstance):

+    log = client.read_from_file('/var/log/cloud-init.log')

+    assert 'Python version change detected' not in log

+    assert 'Cache compatibility status is currently unknown.' not in log

+    _assert_no_pickle_problems(log)

+

+    client.restart()

+    log = client.read_from_file('/var/log/cloud-init.log')

+    assert 'Python version change detected' not in log

+    assert 'Could not determine Python version used to write cache' not in log

+    _assert_no_pickle_problems(log)

+

+    # Now ensure that loading a bad pickle gives us problems

+    client.push_file(TEST_PICKLE, PICKLE_PATH)

+    client.restart()

+    log = client.read_from_file('/var/log/cloud-init.log')

+    assert 'Failed loading pickled blob from {}'.format(PICKLE_PATH) in log

+

+

+def test_cache_purged_on_version_change(client: IntegrationInstance):

+    # Start by pushing the invalid pickle so we'll hit an error if the

+    # cache didn't actually get purged

+    client.push_file(TEST_PICKLE, PICKLE_PATH)

+    client.execute("echo '1.0' > /var/lib/cloud/data/python-version")

+    client.restart()

+    log = client.read_from_file('/var/log/cloud-init.log')

+    assert 'Python version change detected. Purging cache' in log

+    _assert_no_pickle_problems(log)

+

+

+def test_log_message_on_missing_version_file(client: IntegrationInstance):

+    # Start by pushing a pickle so we can see the log message

+    client.push_file(TEST_PICKLE, PICKLE_PATH)

+    client.execute("rm /var/lib/cloud/data/python-version")

+    client.restart()

+    log = client.read_from_file('/var/log/cloud-init.log')

+    assert (

+        'Writing python-version file. '

+        'Cache compatibility status is currently unknown.'

+    ) in log

-- 

2.31.1