From 04a4cc7b8da04ba4103118cf9d975d8e9548e0dc Mon Sep 17 00:00:00 2001

From: Emanuele Giuseppe Esposito <eesposit@redhat.com>

Date: Fri, 4 Mar 2022 11:23:22 +0100

Subject: [PATCH 2/2] Fix MIME policy failure on python version upgrade (#934)

RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>

RH-MergeRequest: 54: - Detect a Python version change and clear the cache (#857)

RH-Commit: [2/2] 05fc8c52a39b5ad464ad146488703467e39d73b1

RH-Bugzilla: 1935826

RH-Acked-by: Eduardo Otubo <otubo@redhat.com>

RH-Acked-by: Vitaly Kuznetsov <vkuznets@redhat.com>

RH-Acked-by: Mohamed Gamal Morsy <mmorsy@redhat.com>

commit eacb0353803263934aa2ac827c37e461c87cb107

Author: James Falcon <therealfalcon@gmail.com>

Date:   Thu Jul 15 17:52:21 2021 -0500

    Fix MIME policy failure on python version upgrade (#934)

    Python 3.6 added a new `policy` attribute to `MIMEMultipart`.

    MIMEMultipart may be part of the cached object pickle of a datasource.

    Upgrading from an old version of python to 3.6+ will cause the

    datasource to be invalid after pickle load.

    This commit uses the upgrade framework to attempt to access the mime

    message and fail early (thus discarding the cache) if we cannot.

    Commit 78e89b03 should fix this issue more generally.

Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

---

 cloudinit/sources/__init__.py                 |  18 +

 cloudinit/stages.py                           |   2 +

 .../assets/trusty_with_mime.pkl               | 572 ++++++++++++++++++

 .../modules/test_persistence.py               |  30 +

 4 files changed, 622 insertions(+)

 create mode 100644 tests/integration_tests/assets/trusty_with_mime.pkl

 create mode 100644 tests/integration_tests/modules/test_persistence.py

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py

index 7d74f8d9..338861e6 100644

--- a/cloudinit/sources/__init__.py

+++ b/cloudinit/sources/__init__.py

@@ -74,6 +74,10 @@ NetworkConfigSource = namedtuple('NetworkConfigSource',

                                  _NETCFG_SOURCE_NAMES)(*_NETCFG_SOURCE_NAMES)

+class DatasourceUnpickleUserDataError(Exception):

+    """Raised when userdata is unable to be unpickled due to python upgrades"""

+

+

 class DataSourceNotFoundException(Exception):

     pass

@@ -227,6 +231,20 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):

             self.vendordata2 = None

         if not hasattr(self, 'vendordata2_raw'):

             self.vendordata2_raw = None

+        if hasattr(self, 'userdata') and self.userdata is not None:

+            # If userdata stores MIME data, on < python3.6 it will be

+            # missing the 'policy' attribute that exists on >=python3.6.

+            # Calling str() on the userdata will attempt to access this

+            # policy attribute. This will raise an exception, causing

+            # the pickle load to fail, so cloud-init will discard the cache

+            try:

+                str(self.userdata)

+            except AttributeError as e:

+                LOG.debug(

+                    "Unable to unpickle datasource: %s."

+                    " Ignoring current cache.", e

+                )

+                raise DatasourceUnpickleUserDataError() from e

     def __str__(self):

         return type_utils.obj_name(self)

diff --git a/cloudinit/stages.py b/cloudinit/stages.py

index 83e25dd1..e709a5cf 100644

--- a/cloudinit/stages.py

+++ b/cloudinit/stages.py

@@ -980,6 +980,8 @@ def _pkl_load(fname):

         return None

     try:

         return pickle.loads(pickle_contents)

+    except sources.DatasourceUnpickleUserDataError:

+        return None

     except Exception:

         util.logexc(LOG, "Failed loading pickled blob from %s", fname)

         return None

diff --git a/tests/integration_tests/assets/trusty_with_mime.pkl b/tests/integration_tests/assets/trusty_with_mime.pkl

new file mode 100644

index 00000000..a4089ecf

--- /dev/null

+++ b/tests/integration_tests/assets/trusty_with_mime.pkl

@@ -0,0 +1,572 @@

+ccopy_reg

+_reconstructor

+p1

+(ccloudinit.sources.DataSourceNoCloud

+DataSourceNoCloudNet

+p2

+c__builtin__

+object

+p3

+NtRp4

+(dp5

+S'paths'

+p6

+g1

+(ccloudinit.helpers

+Paths

+p7

+g3

+NtRp8

+(dp9

+S'lookups'

+p10

+(dp11

+S'cloud_config'

+p12

+S'cloud-config.txt'

+p13

+sS'userdata'

+p14

+S'user-data.txt.i'

+p15

+sS'vendordata'

+p16

+S'vendor-data.txt.i'

+p17

+sS'userdata_raw'

+p18

+S'user-data.txt'

+p19

+sS'boothooks'

+p20

+g20

+sS'scripts'

+p21

+g21

+sS'sem'

+p22

+g22

+sS'data'

+p23

+g23

+sS'vendor_scripts'

+p24

+S'scripts/vendor'

+p25

+sS'handlers'

+p26

+g26

+sS'obj_pkl'

+p27

+S'obj.pkl'

+p28

+sS'vendordata_raw'

+p29

+S'vendor-data.txt'

+p30

+sS'vendor_cloud_config'

+p31

+S'vendor-cloud-config.txt'

+p32

+ssS'template_tpl'

+p33

+S'/etc/cloud/templates/%s.tmpl'

+p34

+sS'cfgs'

+p35

+(dp36

+S'cloud_dir'

+p37

+S'/var/lib/cloud/'

+p38

+sS'templates_dir'

+p39

+S'/etc/cloud/templates/'

+p40

+sS'upstart_dir'

+p41

+S'/etc/init/'

+p42

+ssS'cloud_dir'

+p43

+g38

+sS'datasource'

+p44

+NsS'upstart_conf_d'

+p45

+g42

+sS'boot_finished'

+p46

+S'/var/lib/cloud/instance/boot-finished'

+p47

+sS'instance_link'

+p48

+S'/var/lib/cloud/instance'

+p49

+sS'seed_dir'

+p50

+S'/var/lib/cloud/seed'

+p51

+sbsS'supported_seed_starts'

+p52

+(S'http://'

+p53

+S'https://'

+p54

+S'ftp://'

+p55

+tp56

+sS'sys_cfg'

+p57

+(dp58

+S'output'

+p59

+(dp60

+S'all'

+p61

+S'| tee -a /var/log/cloud-init-output.log'

+p62

+ssS'users'

+p63

+(lp64

+S'default'

+p65

+asS'def_log_file'

+p66

+S'/var/log/cloud-init.log'

+p67

+sS'cloud_final_modules'

+p68

+(lp69

+S'rightscale_userdata'

+p70

+aS'scripts-vendor'

+p71

+aS'scripts-per-once'

+p72

+aS'scripts-per-boot'

+p73

+aS'scripts-per-instance'

+p74

+aS'scripts-user'

+p75

+aS'ssh-authkey-fingerprints'

+p76

+aS'keys-to-console'

+p77

+aS'phone-home'

+p78

+aS'final-message'

+p79

+aS'power-state-change'

+p80

+asS'disable_root'

+p81

+I01

+sS'syslog_fix_perms'

+p82

+S'syslog:adm'

+p83

+sS'log_cfgs'

+p84

+(lp85

+(lp86

+S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n'

+p87

+aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n'

+p88

+aa(lp89

+g87

+aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n"

+p90

+aasS'cloud_init_modules'

+p91

+(lp92

+S'migrator'

+p93

+aS'seed_random'

+p94

+aS'bootcmd'

+p95

+aS'write-files'

+p96

+aS'growpart'

+p97

+aS'resizefs'

+p98

+aS'set_hostname'

+p99

+aS'update_hostname'

+p100

+aS'update_etc_hosts'

+p101

+aS'ca-certs'

+p102

+aS'rsyslog'

+p103

+aS'users-groups'

+p104

+aS'ssh'

+p105

+asS'preserve_hostname'

+p106

+I00

+sS'_log'

+p107

+(lp108

+g87

+ag90

+ag88

+asS'datasource_list'

+p109

+(lp110

+S'NoCloud'

+p111

+aS'ConfigDrive'

+p112

+aS'OpenNebula'

+p113

+aS'Azure'

+p114

+aS'AltCloud'

+p115

+aS'OVF'

+p116

+aS'MAAS'

+p117

+aS'GCE'

+p118

+aS'OpenStack'

+p119

+aS'CloudSigma'

+p120

+aS'Ec2'

+p121

+aS'CloudStack'

+p122

+aS'SmartOS'

+p123

+aS'None'

+p124

+asS'vendor_data'

+p125

+(dp126

+S'prefix'

+p127

+(lp128

+sS'enabled'

+p129

+I01

+ssS'cloud_config_modules'

+p130

+(lp131

+S'emit_upstart'

+p132

+aS'disk_setup'

+p133

+aS'mounts'

+p134

+aS'ssh-import-id'

+p135

+aS'locale'

+p136

+aS'set-passwords'

+p137

+aS'grub-dpkg'

+p138

+aS'apt-pipelining'

+p139

+aS'apt-configure'

+p140

+aS'package-update-upgrade-install'

+p141

+aS'landscape'

+p142

+aS'timezone'

+p143

+aS'puppet'

+p144

+aS'chef'

+p145

+aS'salt-minion'

+p146

+aS'mcollective'

+p147

+aS'disable-ec2-metadata'

+p148

+aS'runcmd'

+p149

+aS'byobu'

+p150

+assg14

+(iemail.mime.multipart

+MIMEMultipart

+p151

+(dp152

+S'_headers'

+p153

+(lp154

+(S'Content-Type'

+p155

+S'multipart/mixed; boundary="===============4291038100093149247=="'

+tp156

+a(S'MIME-Version'

+p157

+S'1.0'

+p158

+tp159

+a(S'Number-Attachments'

+p160

+S'1'

+tp161

+asS'_payload'

+p162

+(lp163

+(iemail.mime.base

+MIMEBase

+p164

+(dp165

+g153

+(lp166

+(g157

+g158

+tp167

+a(S'Content-Type'

+p168

+S'text/x-not-multipart'

+tp169

+a(S'Content-Disposition'

+p170

+S'attachment; filename="part-001"'

+tp171

+asg162

+S''

+sS'_charset'

+p172

+NsS'_default_type'

+p173

+S'text/plain'

+p174

+sS'preamble'

+p175

+NsS'defects'

+p176

+(lp177

+sS'_unixfrom'

+p178

+NsS'epilogue'

+p179

+Nsbasg172

+Nsg173

+g174

+sg175

+Nsg176

+(lp180

+sg178

+Nsg179

+Nsbsg16

+S'#cloud-config\n{}\n\n'

+p181

+sg18

+S'Content-Type: multipart/mixed; boundary="===============1378281702283945349=="\nMIME-Version: 1.0\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script1.sh"\n\nIyEvYmluL3NoCgplY2hvICdoaScgPiAvdmFyL3RtcC9oaQo=\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script2.sh"\n\nIyEvYmluL2Jhc2gKCmVjaG8gJ2hpMicgPiAvdmFyL3RtcC9oaTIK\n\n--===============1378281702283945349==--\n\n#cloud-config\n# final_message: |\n#   This is my final message!\n#   $version\n#   $timestamp\n#   $datasource\n#   $uptime\n# updates:\n#   network:\n#     when: [\'hotplug\']\n'

+p182

+sg29

+NsS'dsmode'

+p183

+S'net'

+p184

+sS'seed'

+p185

+S'/var/lib/cloud/seed/nocloud-net'

+p186

+sS'cmdline_id'

+p187

+S'ds=nocloud-net'

+p188

+sS'ud_proc'

+p189

+g1

+(ccloudinit.user_data

+UserDataProcessor

+p190

+g3

+NtRp191

+(dp192

+g6

+g8

+sS'ssl_details'

+p193

+(dp194

+sbsg50

+g186

+sS'ds_cfg'

+p195

+(dp196

+sS'distro'

+p197

+g1

+(ccloudinit.distros.ubuntu

+Distro

+p198

+g3

+NtRp199

+(dp200

+S'osfamily'

+p201

+S'debian'

+p202

+sS'_paths'

+p203

+g8

+sS'name'

+p204

+S'ubuntu'

+p205

+sS'_runner'

+p206

+g1

+(ccloudinit.helpers

+Runners

+p207

+g3

+NtRp208

+(dp209

+g6

+g8

+sS'sems'

+p210

+(dp211

+sbsS'_cfg'

+p212

+(dp213

+S'paths'

+p214

+(dp215

+g37

+g38

+sg39

+g40

+sg41

+g42

+ssS'default_user'

+p216

+(dp217

+S'shell'

+p218

+S'/bin/bash'

+p219

+sS'name'

+p220

+S'ubuntu'

+p221

+sS'sudo'

+p222

+(lp223

+S'ALL=(ALL) NOPASSWD:ALL'

+p224

+asS'lock_passwd'

+p225

+I01

+sS'gecos'

+p226

+S'Ubuntu'

+p227

+sS'groups'

+p228

+(lp229

+S'adm'

+p230

+aS'audio'

+p231

+aS'cdrom'

+p232

+aS'dialout'

+p233

+aS'dip'

+p234

+aS'floppy'

+p235

+aS'netdev'

+p236

+aS'plugdev'

+p237

+aS'sudo'

+p238

+aS'video'

+p239

+assS'package_mirrors'

+p240

+(lp241

+(dp242

+S'arches'

+p243

+(lp244

+S'i386'

+p245

+aS'amd64'

+p246

+asS'failsafe'

+p247

+(dp248

+S'security'

+p249

+S'http://security.ubuntu.com/ubuntu'

+p250

+sS'primary'

+p251

+S'http://archive.ubuntu.com/ubuntu'

+p252

+ssS'search'

+p253

+(dp254

+S'security'

+p255

+(lp256

+sS'primary'

+p257

+(lp258

+S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/'

+p259

+aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/'

+p260

+aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/'

+p261

+assa(dp262

+S'arches'

+p263

+(lp264

+S'armhf'

+p265

+aS'armel'

+p266

+aS'default'

+p267

+asS'failsafe'

+p268

+(dp269

+S'security'

+p270

+S'http://ports.ubuntu.com/ubuntu-ports'

+p271

+sS'primary'

+p272

+S'http://ports.ubuntu.com/ubuntu-ports'

+p273

+ssasS'ssh_svcname'

+p274

+S'ssh'

+p275

+ssbsS'metadata'

+p276

+(dp277

+g183

+g184

+sS'local-hostname'

+p278

+S'me'

+p279

+sS'instance-id'

+p280

+S'me'

+p281

+ssb.

\ No newline at end of file

diff --git a/tests/integration_tests/modules/test_persistence.py b/tests/integration_tests/modules/test_persistence.py

new file mode 100644

index 00000000..00fdeaea

--- /dev/null

+++ b/tests/integration_tests/modules/test_persistence.py

@@ -0,0 +1,30 @@

+# This file is part of cloud-init. See LICENSE file for license information.

+"""Test the behavior of loading/discarding pickle data"""

+from pathlib import Path

+

+import pytest

+

+from tests.integration_tests.instances import IntegrationInstance

+from tests.integration_tests.util import (

+    ASSETS_DIR,

+    verify_ordered_items_in_text,

+)

+

+

+PICKLE_PATH = Path('/var/lib/cloud/instance/obj.pkl')

+TEST_PICKLE = ASSETS_DIR / 'trusty_with_mime.pkl'

+

+

+@pytest.mark.lxd_container

+def test_log_message_on_missing_version_file(client: IntegrationInstance):

+    client.push_file(TEST_PICKLE, PICKLE_PATH)

+    client.restart()

+    assert client.execute('cloud-init status --wait').ok

+    log = client.read_from_file('/var/log/cloud-init.log')

+    verify_ordered_items_in_text([

+        "Unable to unpickle datasource: 'MIMEMultipart' object has no "

+        "attribute 'policy'. Ignoring current cache.",

+        'no cache found',

+        'Searching for local data source',

+        'SUCCESS: found local data from DataSourceNoCloud'

+    ], log)

-- 

2.31.1