diff --git a/SOURCES/ci-Adding-disk_setup-to-rhel-cloud.cfg.patch b/SOURCES/ci-Adding-disk_setup-to-rhel-cloud.cfg.patch new file mode 100644 index 0000000..b5ffd0b --- /dev/null +++ b/SOURCES/ci-Adding-disk_setup-to-rhel-cloud.cfg.patch @@ -0,0 +1,42 @@ +From a735a0e95143e39f5d63ec86f5a41737c5782822 Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Wed, 26 Sep 2018 13:57:42 +0200 +Subject: [PATCH 4/4] Adding disk_setup to rhel/cloud.cfg + +RH-Author: Eduardo Otubo +Message-id: <20180926135742.11140-5-otubo@redhat.com> +Patchwork-id: 82299 +O-Subject: [RHEL-7.6 cloud-init PATCHv2 4/4] Adding disk_setup to rhel/cloud.cfg +Bugzilla: 1560415 +RH-Acked-by: Vitaly Kuznetsov +RH-Acked-by: Miroslav Rezanina + +When Azure VM is de-allocated and started again its resource disk +needs to be re-partitioned and a RHEL supported filesystem needs to be +created on top. Include disk_setup module in the default RHEL config +which does the job. + +X-downstream-only: yes +Resolves: rhbz#1560415 + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + rhel/cloud.cfg | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/rhel/cloud.cfg b/rhel/cloud.cfg +index bb6bc4d..4a73981 100644 +--- a/rhel/cloud.cfg ++++ b/rhel/cloud.cfg +@@ -11,6 +11,7 @@ ssh_genkeytypes: ~ + syslog_fix_perms: ~ + + cloud_init_modules: ++ - disk_setup + - migrator + - bootcmd + - write-files +-- +1.8.3.1 + diff --git a/SOURCES/ci-Adding-systemd-mount-options-to-wait-for-cloud-init.patch b/SOURCES/ci-Adding-systemd-mount-options-to-wait-for-cloud-init.patch new file mode 100644 index 0000000..8a417cf --- /dev/null +++ b/SOURCES/ci-Adding-systemd-mount-options-to-wait-for-cloud-init.patch @@ -0,0 +1,42 @@ +From c533a99fd7e3f78027c74a889e931604c222db0f Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Wed, 26 Sep 2018 13:57:39 +0200 +Subject: [PATCH 1/4] Adding systemd mount options to wait for cloud-init + +RH-Author: Eduardo Otubo +Message-id: <20180926135742.11140-2-otubo@redhat.com> +Patchwork-id: 82297 +O-Subject: [RHEL-7.6 cloud-init PATCHv2 1/4] Adding systemd mount options to wait for cloud-init +Bugzilla: 1560415 +RH-Acked-by: Vitaly Kuznetsov +RH-Acked-by: Miroslav Rezanina + +This patch adds systemd mount options to wait for cloud-init. On Azure, +cloud-init needs to format ephemeral disk before we are able to mount +it. + +X-downstream-only: yes +Resolves: rhbz#1560415 + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + rhel/cloud.cfg | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/rhel/cloud.cfg b/rhel/cloud.cfg +index 8644872..bb6bc4d 100644 +--- a/rhel/cloud.cfg ++++ b/rhel/cloud.cfg +@@ -4,7 +4,7 @@ users: + disable_root: 1 + ssh_pwauth: 0 + +-mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] ++mount_default_fields: [~, ~, 'auto', 'defaults,nofail,x-systemd.requires=cloud-init.service', '0', '2'] + resize_rootfs_tmp: /dev + ssh_deletekeys: 0 + ssh_genkeytypes: ~ +-- +1.8.3.1 + diff --git a/SOURCES/ci-Azure-Ignore-NTFS-mount-errors-when-checking-ephemer.patch b/SOURCES/ci-Azure-Ignore-NTFS-mount-errors-when-checking-ephemer.patch new file mode 100644 index 0000000..1079cda --- /dev/null +++ b/SOURCES/ci-Azure-Ignore-NTFS-mount-errors-when-checking-ephemer.patch @@ -0,0 +1,422 @@ +From 1c985230cd8559c3fc4af33f9bff6e2c103ce5e9 Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Wed, 26 Sep 2018 13:57:40 +0200 +Subject: [PATCH 2/4] Azure: Ignore NTFS mount errors when checking ephemeral + drive + +RH-Author: Eduardo Otubo +Message-id: <20180926135742.11140-3-otubo@redhat.com> +Patchwork-id: 82300 +O-Subject: [RHEL-7.6 cloud-init PATCHv2 2/4] Azure: Ignore NTFS mount errors when checking ephemeral drive +Bugzilla: 1560415 +RH-Acked-by: Vitaly Kuznetsov +RH-Acked-by: Miroslav Rezanina + +commit aa4eeb80839382117e1813e396dc53aa634fd7ba +Author: Paul Meyer +Date: Wed May 23 15:45:39 2018 -0400 + + Azure: Ignore NTFS mount errors when checking ephemeral drive + + The Azure data source provides a method to check whether a NTFS partition + on the ephemeral disk is safe for reformatting to ext4. The method checks + to see if there are customer data files on the disk. However, mounting + the partition fails on systems that do not have the capability of + mounting NTFS. Note that in this case, it is also very unlikely that the + NTFS partition would have been used by the system (since it can't mount + it). The only case would be where an update to the system removed the + capability to mount NTFS, the likelihood of which is also very small. + This change allows the reformatting of the ephemeral disk to ext4 on + systems where mounting NTFS is not supported. + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + cloudinit/sources/DataSourceAzure.py | 63 ++++++++++++---- + cloudinit/util.py | 5 +- + tests/unittests/test_datasource/test_azure.py | 105 +++++++++++++++++++++----- + 3 files changed, 138 insertions(+), 35 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 23b4d53..7e49455 100644 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -214,6 +214,7 @@ BUILTIN_CLOUD_CONFIG = { + } + + DS_CFG_PATH = ['datasource', DS_NAME] ++DS_CFG_KEY_PRESERVE_NTFS = 'never_destroy_ntfs' + DEF_EPHEMERAL_LABEL = 'Temporary Storage' + + # The redacted password fails to meet password complexity requirements +@@ -400,14 +401,9 @@ class DataSourceAzure(sources.DataSource): + if found == ddir: + LOG.debug("using files cached in %s", ddir) + +- # azure / hyper-v provides random data here +- # TODO. find the seed on FreeBSD platform +- # now update ds_cfg to reflect contents pass in config +- if not util.is_FreeBSD(): +- seed = util.load_file("/sys/firmware/acpi/tables/OEM0", +- quiet=True, decode=False) +- if seed: +- self.metadata['random_seed'] = seed ++ seed = _get_random_seed() ++ if seed: ++ self.metadata['random_seed'] = seed + + user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) + self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) +@@ -537,7 +533,9 @@ class DataSourceAzure(sources.DataSource): + return fabric_data + + def activate(self, cfg, is_new_instance): +- address_ephemeral_resize(is_new_instance=is_new_instance) ++ address_ephemeral_resize(is_new_instance=is_new_instance, ++ preserve_ntfs=self.ds_cfg.get( ++ DS_CFG_KEY_PRESERVE_NTFS, False)) + return + + @property +@@ -581,17 +579,29 @@ def _has_ntfs_filesystem(devpath): + return os.path.realpath(devpath) in ntfs_devices + + +-def can_dev_be_reformatted(devpath): +- """Determine if block device devpath is newly formatted ephemeral. ++def can_dev_be_reformatted(devpath, preserve_ntfs): ++ """Determine if the ephemeral drive at devpath should be reformatted. + +- A newly formatted disk will: ++ A fresh ephemeral disk is formatted by Azure and will: + a.) have a partition table (dos or gpt) + b.) have 1 partition that is ntfs formatted, or + have 2 partitions with the second partition ntfs formatted. + (larger instances with >2TB ephemeral disk have gpt, and will + have a microsoft reserved partition as part 1. LP: #1686514) + c.) the ntfs partition will have no files other than possibly +- 'dataloss_warning_readme.txt'""" ++ 'dataloss_warning_readme.txt' ++ ++ User can indicate that NTFS should never be destroyed by setting ++ DS_CFG_KEY_PRESERVE_NTFS in dscfg. ++ If data is found on NTFS, user is warned to set DS_CFG_KEY_PRESERVE_NTFS ++ to make sure cloud-init does not accidentally wipe their data. ++ If cloud-init cannot mount the disk to check for data, destruction ++ will be allowed, unless the dscfg key is set.""" ++ if preserve_ntfs: ++ msg = ('config says to never destroy NTFS (%s.%s), skipping checks' % ++ (".".join(DS_CFG_PATH), DS_CFG_KEY_PRESERVE_NTFS)) ++ return False, msg ++ + if not os.path.exists(devpath): + return False, 'device %s does not exist' % devpath + +@@ -624,18 +634,27 @@ def can_dev_be_reformatted(devpath): + bmsg = ('partition %s (%s) on device %s was ntfs formatted' % + (cand_part, cand_path, devpath)) + try: +- file_count = util.mount_cb(cand_path, count_files) ++ file_count = util.mount_cb(cand_path, count_files, mtype="ntfs", ++ update_env_for_mount={'LANG': 'C'}) + except util.MountFailedError as e: ++ if "mount: unknown filesystem type 'ntfs'" in str(e): ++ return True, (bmsg + ' but this system cannot mount NTFS,' ++ ' assuming there are no important files.' ++ ' Formatting allowed.') + return False, bmsg + ' but mount of %s failed: %s' % (cand_part, e) + + if file_count != 0: ++ LOG.warning("it looks like you're using NTFS on the ephemeral disk, " ++ 'to ensure that filesystem does not get wiped, set ' ++ '%s.%s in config', '.'.join(DS_CFG_PATH), ++ DS_CFG_KEY_PRESERVE_NTFS) + return False, bmsg + ' but had %d files on it.' % file_count + + return True, bmsg + ' and had no important files. Safe for reformatting.' + + + def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, +- is_new_instance=False): ++ is_new_instance=False, preserve_ntfs=False): + # wait for ephemeral disk to come up + naplen = .2 + missing = util.wait_for_files([devpath], maxwait=maxwait, naplen=naplen, +@@ -651,7 +670,7 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, + if is_new_instance: + result, msg = (True, "First instance boot.") + else: +- result, msg = can_dev_be_reformatted(devpath) ++ result, msg = can_dev_be_reformatted(devpath, preserve_ntfs) + + LOG.debug("reformattable=%s: %s", result, msg) + if not result: +@@ -965,6 +984,18 @@ def _check_freebsd_cdrom(cdrom_dev): + return False + + ++def _get_random_seed(): ++ """Return content random seed file if available, otherwise, ++ return None.""" ++ # azure / hyper-v provides random data here ++ # TODO. find the seed on FreeBSD platform ++ # now update ds_cfg to reflect contents pass in config ++ if util.is_FreeBSD(): ++ return None ++ return util.load_file("/sys/firmware/acpi/tables/OEM0", ++ quiet=True, decode=False) ++ ++ + def list_possible_azure_ds_devs(): + devlist = [] + if util.is_FreeBSD(): +diff --git a/cloudinit/util.py b/cloudinit/util.py +index 0ab2c48..c8e14ba 100644 +--- a/cloudinit/util.py ++++ b/cloudinit/util.py +@@ -1608,7 +1608,8 @@ def mounts(): + return mounted + + +-def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): ++def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True, ++ update_env_for_mount=None): + """ + Mount the device, call method 'callback' passing the directory + in which it was mounted, then unmount. Return whatever 'callback' +@@ -1670,7 +1671,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) +- subp(mountcmd) ++ subp(mountcmd, update_env=update_env_for_mount) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + break +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index 3e8b791..af2c93a 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -1,10 +1,10 @@ + # This file is part of cloud-init. See LICENSE file for license information. + + from cloudinit import helpers +-from cloudinit.util import b64e, decode_binary, load_file, write_file + from cloudinit.sources import DataSourceAzure as dsaz +-from cloudinit.util import find_freebsd_part +-from cloudinit.util import get_path_dev_freebsd ++from cloudinit.util import (b64e, decode_binary, load_file, write_file, ++ find_freebsd_part, get_path_dev_freebsd, ++ MountFailedError) + from cloudinit.version import version_string as vs + from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock, + ExitStack, PY26, SkipTest) +@@ -95,6 +95,8 @@ class TestAzureDataSource(CiTestCase): + self.patches = ExitStack() + self.addCleanup(self.patches.close) + ++ self.patches.enter_context(mock.patch.object(dsaz, '_get_random_seed')) ++ + super(TestAzureDataSource, self).setUp() + + def apply_patches(self, patches): +@@ -335,6 +337,18 @@ fdescfs /dev/fd fdescfs rw 0 0 + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], '_COMMAND') + ++ def test_sys_cfg_set_never_destroy_ntfs(self): ++ sys_cfg = {'datasource': {'Azure': { ++ 'never_destroy_ntfs': 'user-supplied-value'}}} ++ data = {'ovfcontent': construct_valid_ovf_env(data={}), ++ 'sys_cfg': sys_cfg} ++ ++ dsrc = self._get_ds(data) ++ ret = self._get_and_setup(dsrc) ++ self.assertTrue(ret) ++ self.assertEqual(dsrc.ds_cfg.get(dsaz.DS_CFG_KEY_PRESERVE_NTFS), ++ 'user-supplied-value') ++ + def test_username_used(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} +@@ -676,6 +690,8 @@ class TestAzureBounce(CiTestCase): + mock.MagicMock(return_value={}))) + self.patches.enter_context( + mock.patch.object(dsaz.util, 'which', lambda x: True)) ++ self.patches.enter_context( ++ mock.patch.object(dsaz, '_get_random_seed')) + + def _dmi_mocks(key): + if key == 'system-uuid': +@@ -957,7 +973,9 @@ class TestCanDevBeReformatted(CiTestCase): + # return sorted by partition number + return sorted(ret, key=lambda d: d[0]) + +- def mount_cb(device, callback): ++ def mount_cb(device, callback, mtype, update_env_for_mount): ++ self.assertEqual('ntfs', mtype) ++ self.assertEqual('C', update_env_for_mount.get('LANG')) + p = self.tmp_dir() + for f in bypath.get(device).get('files', []): + write_file(os.path.join(p, f), content=f) +@@ -988,14 +1006,16 @@ class TestCanDevBeReformatted(CiTestCase): + '/dev/sda2': {'num': 2}, + '/dev/sda3': {'num': 3}, + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertFalse(value) + self.assertIn("3 or more", msg.lower()) + + def test_no_partitions_is_false(self): + """A disk with no partitions can not be formatted.""" + self.patchup({'/dev/sda': {}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertFalse(value) + self.assertIn("not partitioned", msg.lower()) + +@@ -1007,7 +1027,8 @@ class TestCanDevBeReformatted(CiTestCase): + '/dev/sda1': {'num': 1}, + '/dev/sda2': {'num': 2, 'fs': 'ext4', 'files': []}, + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertFalse(value) + self.assertIn("not ntfs", msg.lower()) + +@@ -1020,7 +1041,8 @@ class TestCanDevBeReformatted(CiTestCase): + '/dev/sda2': {'num': 2, 'fs': 'ntfs', + 'files': ['secret.txt']}, + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertFalse(value) + self.assertIn("files on it", msg.lower()) + +@@ -1032,7 +1054,8 @@ class TestCanDevBeReformatted(CiTestCase): + '/dev/sda1': {'num': 1}, + '/dev/sda2': {'num': 2, 'fs': 'ntfs', 'files': []}, + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertTrue(value) + self.assertIn("safe for", msg.lower()) + +@@ -1043,7 +1066,8 @@ class TestCanDevBeReformatted(CiTestCase): + 'partitions': { + '/dev/sda1': {'num': 1, 'fs': 'zfs'}, + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertFalse(value) + self.assertIn("not ntfs", msg.lower()) + +@@ -1055,9 +1079,14 @@ class TestCanDevBeReformatted(CiTestCase): + '/dev/sda1': {'num': 1, 'fs': 'ntfs', + 'files': ['file1.txt', 'file2.exe']}, + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") +- self.assertFalse(value) +- self.assertIn("files on it", msg.lower()) ++ with mock.patch.object(dsaz.LOG, 'warning') as warning: ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) ++ wmsg = warning.call_args[0][0] ++ self.assertIn("looks like you're using NTFS on the ephemeral disk", ++ wmsg) ++ self.assertFalse(value) ++ self.assertIn("files on it", msg.lower()) + + def test_one_partition_ntfs_empty_is_true(self): + """1 mountable ntfs partition and no files can be formatted.""" +@@ -1066,7 +1095,8 @@ class TestCanDevBeReformatted(CiTestCase): + 'partitions': { + '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []} + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertTrue(value) + self.assertIn("safe for", msg.lower()) + +@@ -1078,7 +1108,8 @@ class TestCanDevBeReformatted(CiTestCase): + '/dev/sda1': {'num': 1, 'fs': 'ntfs', + 'files': ['dataloss_warning_readme.txt']} + }}}) +- value, msg = dsaz.can_dev_be_reformatted("/dev/sda") ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=False) + self.assertTrue(value) + self.assertIn("safe for", msg.lower()) + +@@ -1093,7 +1124,8 @@ class TestCanDevBeReformatted(CiTestCase): + 'num': 1, 'fs': 'ntfs', 'files': [self.warning_file], + 'realpath': '/dev/sdb1'} + }}}) +- value, msg = dsaz.can_dev_be_reformatted(epath) ++ value, msg = dsaz.can_dev_be_reformatted(epath, ++ preserve_ntfs=False) + self.assertTrue(value) + self.assertIn("safe for", msg.lower()) + +@@ -1112,10 +1144,49 @@ class TestCanDevBeReformatted(CiTestCase): + epath + '-part3': {'num': 3, 'fs': 'ext', + 'realpath': '/dev/sdb3'} + }}}) +- value, msg = dsaz.can_dev_be_reformatted(epath) ++ value, msg = dsaz.can_dev_be_reformatted(epath, ++ preserve_ntfs=False) + self.assertFalse(value) + self.assertIn("3 or more", msg.lower()) + ++ def test_ntfs_mount_errors_true(self): ++ """can_dev_be_reformatted does not fail if NTFS is unknown fstype.""" ++ self.patchup({ ++ '/dev/sda': { ++ 'partitions': { ++ '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []} ++ }}}) ++ ++ err = ("Unexpected error while running command.\n", ++ "Command: ['mount', '-o', 'ro,sync', '-t', 'auto', ", ++ "'/dev/sda1', '/fake-tmp/dir']\n" ++ "Exit code: 32\n" ++ "Reason: -\n" ++ "Stdout: -\n" ++ "Stderr: mount: unknown filesystem type 'ntfs'") ++ self.m_mount_cb.side_effect = MountFailedError( ++ 'Failed mounting %s to %s due to: %s' % ++ ('/dev/sda', '/fake-tmp/dir', err)) ++ ++ value, msg = dsaz.can_dev_be_reformatted('/dev/sda', ++ preserve_ntfs=False) ++ self.assertTrue(value) ++ self.assertIn('cannot mount NTFS, assuming', msg) ++ ++ def test_never_destroy_ntfs_config_false(self): ++ """Normally formattable situation with never_destroy_ntfs set.""" ++ self.patchup({ ++ '/dev/sda': { ++ 'partitions': { ++ '/dev/sda1': {'num': 1, 'fs': 'ntfs', ++ 'files': ['dataloss_warning_readme.txt']} ++ }}}) ++ value, msg = dsaz.can_dev_be_reformatted("/dev/sda", ++ preserve_ntfs=True) ++ self.assertFalse(value) ++ self.assertIn("config says to never destroy NTFS " ++ "(datasource.Azure.never_destroy_ntfs)", msg) ++ + + class TestAzureNetExists(CiTestCase): + +-- +1.8.3.1 + diff --git a/SOURCES/ci-azure-Add-reported-ready-marker-file.patch b/SOURCES/ci-azure-Add-reported-ready-marker-file.patch new file mode 100644 index 0000000..3759bb5 --- /dev/null +++ b/SOURCES/ci-azure-Add-reported-ready-marker-file.patch @@ -0,0 +1,329 @@ +From b11937faf800b0ae8054dfd64ce50dc92bbb7f80 Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Wed, 26 Sep 2018 13:57:41 +0200 +Subject: [PATCH 3/4] azure: Add reported ready marker file. + +RH-Author: Eduardo Otubo +Message-id: <20180926135742.11140-4-otubo@redhat.com> +Patchwork-id: 82298 +O-Subject: [RHEL-7.6 cloud-init PATCHv2 3/4] azure: Add reported ready marker file. +Bugzilla: 1560415 +RH-Acked-by: Vitaly Kuznetsov +RH-Acked-by: Miroslav Rezanina + +commit aae494c39f4c6f625e7409ca262e657d085dd5d1 +Author: Joshua Chan +Date: Thu May 3 14:50:16 2018 -0600 + + azure: Add reported ready marker file. + + This change is for Azure VM Preprovisioning. A bug was found when after + azure VMs report ready the first time, during the time when VM is polling + indefinitely for the new ovf-env.xml from Instance Metadata Service + (IMDS), if a reboot happens, we send another report ready signal to the + fabric, which deletes the reprovisioning data on the node. + + This marker file is used to fix this issue so that we will only send a + report ready signal to the fabric when no marker file is present. Then, + create a marker file so that when a reboot does occur, we check if a + marker file has been created and decide whether we would like to send the + repot ready signal. + + LP: #1765214 + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + cloudinit/sources/DataSourceAzure.py | 21 +++- + tests/unittests/test_datasource/test_azure.py | 170 ++++++++++++++++++-------- + 2 files changed, 134 insertions(+), 57 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 7e49455..46d5744 100644 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -48,6 +48,7 @@ DEFAULT_FS = 'ext4' + # DMI chassis-asset-tag is set static for all azure instances + AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77' + REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds" ++REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready" + IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata" + + +@@ -439,11 +440,12 @@ class DataSourceAzure(sources.DataSource): + LOG.debug("negotiating already done for %s", + self.get_instance_id()) + +- def _poll_imds(self, report_ready=True): ++ def _poll_imds(self): + """Poll IMDS for the new provisioning data until we get a valid + response. Then return the returned JSON object.""" + url = IMDS_URL + "?api-version=2017-04-02" + headers = {"Metadata": "true"} ++ report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE)) + LOG.debug("Start polling IMDS") + + def exc_cb(msg, exception): +@@ -453,13 +455,17 @@ class DataSourceAzure(sources.DataSource): + # call DHCP and setup the ephemeral network to acquire the new IP. + return False + +- need_report = report_ready + while True: + try: + with EphemeralDHCPv4() as lease: +- if need_report: ++ if report_ready: ++ path = REPORTED_READY_MARKER_FILE ++ LOG.info( ++ "Creating a marker file to report ready: %s", path) ++ util.write_file(path, "{pid}: {time}\n".format( ++ pid=os.getpid(), time=time())) + self._report_ready(lease=lease) +- need_report = False ++ report_ready = False + return readurl(url, timeout=1, headers=headers, + exception_cb=exc_cb, infinite=True).contents + except UrlError: +@@ -493,8 +499,10 @@ class DataSourceAzure(sources.DataSource): + if (cfg.get('PreprovisionedVm') is True or + os.path.isfile(path)): + if not os.path.isfile(path): +- LOG.info("Creating a marker file to poll imds") +- util.write_file(path, "%s: %s\n" % (os.getpid(), time())) ++ LOG.info("Creating a marker file to poll imds: %s", ++ path) ++ util.write_file(path, "{pid}: {time}\n".format( ++ pid=os.getpid(), time=time())) + return True + return False + +@@ -529,6 +537,7 @@ class DataSourceAzure(sources.DataSource): + "Error communicating with Azure fabric; You may experience." + "connectivity issues.", exc_info=True) + return False ++ util.del_file(REPORTED_READY_MARKER_FILE) + util.del_file(REPROVISION_MARKER_FILE) + return fabric_data + +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index af2c93a..ed810d2 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -1196,19 +1196,9 @@ class TestAzureNetExists(CiTestCase): + self.assertTrue(hasattr(dsaz, "DataSourceAzureNet")) + + +-@mock.patch('cloudinit.sources.DataSourceAzure.util.subp') +-@mock.patch.object(dsaz, 'get_hostname') +-@mock.patch.object(dsaz, 'set_hostname') +-class TestAzureDataSourcePreprovisioning(CiTestCase): +- +- def setUp(self): +- super(TestAzureDataSourcePreprovisioning, self).setUp() +- tmp = self.tmp_dir() +- self.waagent_d = self.tmp_path('/var/lib/waagent', tmp) +- self.paths = helpers.Paths({'cloud_dir': tmp}) +- dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d ++class TestPreprovisioningReadAzureOvfFlag(CiTestCase): + +- def test_read_azure_ovf_with_true_flag(self, *args): ++ def test_read_azure_ovf_with_true_flag(self): + """The read_azure_ovf method should set the PreprovisionedVM + cfg flag if the proper setting is present.""" + content = construct_valid_ovf_env( +@@ -1217,7 +1207,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): + cfg = ret[2] + self.assertTrue(cfg['PreprovisionedVm']) + +- def test_read_azure_ovf_with_false_flag(self, *args): ++ def test_read_azure_ovf_with_false_flag(self): + """The read_azure_ovf method should set the PreprovisionedVM + cfg flag to false if the proper setting is false.""" + content = construct_valid_ovf_env( +@@ -1226,7 +1216,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): + cfg = ret[2] + self.assertFalse(cfg['PreprovisionedVm']) + +- def test_read_azure_ovf_without_flag(self, *args): ++ def test_read_azure_ovf_without_flag(self): + """The read_azure_ovf method should not set the + PreprovisionedVM cfg flag.""" + content = construct_valid_ovf_env() +@@ -1234,12 +1224,121 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): + cfg = ret[2] + self.assertFalse(cfg['PreprovisionedVm']) + +- @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD') +- @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') +- @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') +- @mock.patch('requests.Session.request') ++ ++@mock.patch('os.path.isfile') ++class TestPreprovisioningShouldReprovision(CiTestCase): ++ ++ def setUp(self): ++ super(TestPreprovisioningShouldReprovision, self).setUp() ++ tmp = self.tmp_dir() ++ self.waagent_d = self.tmp_path('/var/lib/waagent', tmp) ++ self.paths = helpers.Paths({'cloud_dir': tmp}) ++ dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d ++ ++ @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') ++ def test__should_reprovision_with_true_cfg(self, isfile, write_f): ++ """The _should_reprovision method should return true with config ++ flag present.""" ++ isfile.return_value = False ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ self.assertTrue(dsa._should_reprovision( ++ (None, None, {'PreprovisionedVm': True}, None))) ++ ++ def test__should_reprovision_with_file_existing(self, isfile): ++ """The _should_reprovision method should return True if the sentinal ++ exists.""" ++ isfile.return_value = True ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ self.assertTrue(dsa._should_reprovision( ++ (None, None, {'preprovisionedvm': False}, None))) ++ ++ def test__should_reprovision_returns_false(self, isfile): ++ """The _should_reprovision method should return False ++ if config and sentinal are not present.""" ++ isfile.return_value = False ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ self.assertFalse(dsa._should_reprovision((None, None, {}, None))) ++ ++ @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') ++ def test_reprovision_calls__poll_imds(self, _poll_imds, isfile): ++ """_reprovision will poll IMDS.""" ++ isfile.return_value = False ++ hostname = "myhost" ++ username = "myuser" ++ odata = {'HostName': hostname, 'UserName': username} ++ _poll_imds.return_value = construct_valid_ovf_env(data=odata) ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ dsa._reprovision() ++ _poll_imds.assert_called_with() ++ ++ ++@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') ++@mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') ++@mock.patch('requests.Session.request') ++@mock.patch( ++ 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') ++class TestPreprovisioningPollIMDS(CiTestCase): ++ ++ def setUp(self): ++ super(TestPreprovisioningPollIMDS, self).setUp() ++ self.tmp = self.tmp_dir() ++ self.waagent_d = self.tmp_path('/var/lib/waagent', self.tmp) ++ self.paths = helpers.Paths({'cloud_dir': self.tmp}) ++ dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d ++ ++ @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') ++ def test_poll_imds_calls_report_ready(self, write_f, report_ready_func, ++ fake_resp, m_dhcp, m_net): ++ """The poll_imds will call report_ready after creating marker file.""" ++ report_marker = self.tmp_path('report_marker', self.tmp) ++ lease = { ++ 'interface': 'eth9', 'fixed-address': '192.168.2.9', ++ 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', ++ 'unknown-245': '624c3620'} ++ m_dhcp.return_value = [lease] ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ mock_path = ( ++ 'cloudinit.sources.DataSourceAzure.REPORTED_READY_MARKER_FILE') ++ with mock.patch(mock_path, report_marker): ++ dsa._poll_imds() ++ self.assertEqual(report_ready_func.call_count, 1) ++ report_ready_func.assert_called_with(lease=lease) ++ ++ def test_poll_imds_report_ready_false(self, report_ready_func, ++ fake_resp, m_dhcp, m_net): ++ """The poll_imds should not call reporting ready ++ when flag is false""" ++ report_marker = self.tmp_path('report_marker', self.tmp) ++ write_file(report_marker, content='dont run report_ready :)') ++ m_dhcp.return_value = [{ ++ 'interface': 'eth9', 'fixed-address': '192.168.2.9', ++ 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', ++ 'unknown-245': '624c3620'}] ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ mock_path = ( ++ 'cloudinit.sources.DataSourceAzure.REPORTED_READY_MARKER_FILE') ++ with mock.patch(mock_path, report_marker): ++ dsa._poll_imds() ++ self.assertEqual(report_ready_func.call_count, 0) ++ ++ ++@mock.patch('cloudinit.sources.DataSourceAzure.util.subp') ++@mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') ++@mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD') ++@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') ++@mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') ++@mock.patch('requests.Session.request') ++class TestAzureDataSourcePreprovisioning(CiTestCase): ++ ++ def setUp(self): ++ super(TestAzureDataSourcePreprovisioning, self).setUp() ++ tmp = self.tmp_dir() ++ self.waagent_d = self.tmp_path('/var/lib/waagent', tmp) ++ self.paths = helpers.Paths({'cloud_dir': tmp}) ++ dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d ++ + def test_poll_imds_returns_ovf_env(self, fake_resp, m_dhcp, m_net, +- m_is_bsd, *args): ++ m_is_bsd, write_f, subp): + """The _poll_imds method should return the ovf_env.xml.""" + m_is_bsd.return_value = False + m_dhcp.return_value = [{ +@@ -1265,12 +1364,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): + prefix_or_mask='255.255.255.0', router='192.168.2.1') + self.assertEqual(m_net.call_count, 1) + +- @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD') +- @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') +- @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') +- @mock.patch('requests.Session.request') + def test__reprovision_calls__poll_imds(self, fake_resp, m_dhcp, m_net, +- m_is_bsd, *args): ++ m_is_bsd, write_f, subp): + """The _reprovision method should call poll IMDS.""" + m_is_bsd.return_value = False + m_dhcp.return_value = [{ +@@ -1302,32 +1397,5 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): + prefix_or_mask='255.255.255.0', router='192.168.2.1') + self.assertEqual(m_net.call_count, 1) + +- @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file') +- @mock.patch('os.path.isfile') +- def test__should_reprovision_with_true_cfg(self, isfile, write_f, *args): +- """The _should_reprovision method should return true with config +- flag present.""" +- isfile.return_value = False +- dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) +- self.assertTrue(dsa._should_reprovision( +- (None, None, {'PreprovisionedVm': True}, None))) +- +- @mock.patch('os.path.isfile') +- def test__should_reprovision_with_file_existing(self, isfile, *args): +- """The _should_reprovision method should return True if the sentinal +- exists.""" +- isfile.return_value = True +- dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) +- self.assertTrue(dsa._should_reprovision( +- (None, None, {'preprovisionedvm': False}, None))) +- +- @mock.patch('os.path.isfile') +- def test__should_reprovision_returns_false(self, isfile, *args): +- """The _should_reprovision method should return False +- if config and sentinal are not present.""" +- isfile.return_value = False +- dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) +- self.assertFalse(dsa._should_reprovision((None, None, {}, None))) +- + + # vi: ts=4 expandtab +-- +1.8.3.1 + diff --git a/SPECS/cloud-init.spec b/SPECS/cloud-init.spec index 892c244..54ddb89 100644 --- a/SPECS/cloud-init.spec +++ b/SPECS/cloud-init.spec @@ -7,7 +7,7 @@ Name: cloud-init Version: 18.2 -Release: 1%{?dist} +Release: 1%{?dist}.1 Summary: Cloud instance init scripts Group: System Environment/Base @@ -25,6 +25,14 @@ Patch0006: 0006-azure-ensure-that-networkmanager-hook-script-runs.patch Patch0007: 0007-sysconfig-Don-t-write-BOOTPROTO-dhcp-for-ipv6-dhcp.patch Patch0008: 0008-DataSourceAzure.py-use-hostnamectl-to-set-hostname.patch Patch0009: 0009-sysconfig-Don-t-disable-IPV6_AUTOCONF.patch +# For bz#1633282 - [Azure] cloud-init fails to mount /dev/sdb1 after stop(deallocate)&&start VM +Patch10: ci-Adding-systemd-mount-options-to-wait-for-cloud-init.patch +# For bz#1633282 - [Azure] cloud-init fails to mount /dev/sdb1 after stop(deallocate)&&start VM +Patch11: ci-Azure-Ignore-NTFS-mount-errors-when-checking-ephemer.patch +# For bz#1633282 - [Azure] cloud-init fails to mount /dev/sdb1 after stop(deallocate)&&start VM +Patch12: ci-azure-Add-reported-ready-marker-file.patch +# For bz#1633282 - [Azure] cloud-init fails to mount /dev/sdb1 after stop(deallocate)&&start VM +Patch13: ci-Adding-disk_setup-to-rhel-cloud.cfg.patch # Deal with noarch -> arch # https://bugzilla.redhat.com/show_bug.cgi?id=1067089 @@ -177,6 +185,14 @@ fi %config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf %changelog +* Thu Sep 27 2018 Miroslav Rezanina - 18.2-1.el7_6.1 +- ci-Adding-systemd-mount-options-to-wait-for-cloud-init.patch [bz#1633282] +- ci-Azure-Ignore-NTFS-mount-errors-when-checking-ephemer.patch [bz#1633282] +- ci-azure-Add-reported-ready-marker-file.patch [bz#1633282] +- ci-Adding-disk_setup-to-rhel-cloud.cfg.patch [bz#1633282] +- Resolves: bz#1633282 + ([Azure] cloud-init fails to mount /dev/sdb1 after stop(deallocate)&&start VM) + * Thu Jun 21 2018 Miroslav Rezanina - Rebase to 18.2 Resolves: rhbz#1525267