diff --git a/SOURCES/ci-Azure-Return-static-fallback-address-as-if-failed-to.patch b/SOURCES/ci-Azure-Return-static-fallback-address-as-if-failed-to.patch new file mode 100644 index 0000000..32d3734 --- /dev/null +++ b/SOURCES/ci-Azure-Return-static-fallback-address-as-if-failed-to.patch @@ -0,0 +1,107 @@ +From f54ebeac5b95c7481718e09c4598a86bc1a8dcfb Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Wed, 3 Jul 2019 13:14:53 +0200 +Subject: [PATCH] Azure: Return static fallback address as if failed to find + endpoint + +RH-Author: Eduardo Otubo +Message-id: <20190703131453.15811-1-otubo@redhat.com> +Patchwork-id: 89354 +O-Subject: [RHEL-7.8 cloud-init PATCH] Azure: Return static fallback address as if failed to find endpoint +Bugzilla: 1726701 +RH-Acked-by: Bandan Das +RH-Acked-by: Mohammed Gamal + +BZ: 1687565 +BRANCH: rhel7/master-18.5 +UPSTREAM: baa478546d8cac98a706010699d64f8c2f70b5bf +BREW: 22476988 + +commit aefb0f1c281740ef307116509057770062d61375 +Author: Jason Zions (MSFT) +Date: Fri May 10 18:38:55 2019 +0000 + + Azure: Return static fallback address as if failed to find endpoint + + The Azure data source helper attempts to use information in the dhcp + lease to find the Wireserver endpoint (IP address). Under some unusual + circumstances, those attempts will fail. This change uses a static + address, known to be always correct in the Azure public and sovereign + clouds, when the helper fails to locate a valid dhcp lease. This + address is not guaranteed to be correct in Azure Stack environments; + it's still best to use the information from the lease whenever possible. + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + cloudinit/sources/helpers/azure.py | 14 +++++++++++--- + tests/unittests/test_datasource/test_azure_helper.py | 9 +++++++-- + 2 files changed, 18 insertions(+), 5 deletions(-) + +diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py +index d3af05e..82c4c8c 100755 +--- a/cloudinit/sources/helpers/azure.py ++++ b/cloudinit/sources/helpers/azure.py +@@ -20,6 +20,9 @@ from cloudinit.reporting import events + + LOG = logging.getLogger(__name__) + ++# This endpoint matches the format as found in dhcp lease files, since this ++# value is applied if the endpoint can't be found within a lease file ++DEFAULT_WIRESERVER_ENDPOINT = "a8:3f:81:10" + + azure_ds_reporter = events.ReportEventStack( + name="azure-ds", +@@ -297,7 +300,12 @@ class WALinuxAgentShim(object): + @azure_ds_telemetry_reporter + def _get_value_from_leases_file(fallback_lease_file): + leases = [] +- content = util.load_file(fallback_lease_file) ++ try: ++ content = util.load_file(fallback_lease_file) ++ except IOError as ex: ++ LOG.error("Failed to read %s: %s", fallback_lease_file, ex) ++ return None ++ + LOG.debug("content is %s", content) + option_name = _get_dhcp_endpoint_option_name() + for line in content.splitlines(): +@@ -372,9 +380,9 @@ class WALinuxAgentShim(object): + fallback_lease_file) + value = WALinuxAgentShim._get_value_from_leases_file( + fallback_lease_file) +- + if value is None: +- raise ValueError('No endpoint found.') ++ LOG.warning("No lease found; using default endpoint") ++ value = DEFAULT_WIRESERVER_ENDPOINT + + endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) + LOG.debug('Azure endpoint found at %s', endpoint_ip_address) +diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py +index 0255616..bd006ab 100644 +--- a/tests/unittests/test_datasource/test_azure_helper.py ++++ b/tests/unittests/test_datasource/test_azure_helper.py +@@ -67,12 +67,17 @@ class TestFindEndpoint(CiTestCase): + self.networkd_leases.return_value = None + + def test_missing_file(self): +- self.assertRaises(ValueError, wa_shim.find_endpoint) ++ """wa_shim find_endpoint uses default endpoint if leasefile not found ++ """ ++ self.assertEqual(wa_shim.find_endpoint(), "168.63.129.16") + + def test_missing_special_azure_line(self): ++ """wa_shim find_endpoint uses default endpoint if leasefile is found ++ but does not contain DHCP Option 245 (whose value is the endpoint) ++ """ + self.load_file.return_value = '' + self.dhcp_options.return_value = {'eth0': {'key': 'value'}} +- self.assertRaises(ValueError, wa_shim.find_endpoint) ++ self.assertEqual(wa_shim.find_endpoint(), "168.63.129.16") + + @staticmethod + def _build_lease_content(encoded_address): +-- +1.8.3.1 + diff --git a/SOURCES/ci-Fix-for-network-configuration-not-persisting-after-r.patch b/SOURCES/ci-Fix-for-network-configuration-not-persisting-after-r.patch new file mode 100644 index 0000000..8e4a8e3 --- /dev/null +++ b/SOURCES/ci-Fix-for-network-configuration-not-persisting-after-r.patch @@ -0,0 +1,107 @@ +From 9969cf3eaa23398816d140b319b3277465aa4bb8 Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Fri, 6 Sep 2019 12:12:11 +0200 +Subject: [PATCH] Fix for network configuration not persisting after reboot + +RH-Author: Eduardo Otubo +Message-id: <20190906121211.23172-1-otubo@redhat.com> +Patchwork-id: 90300 +O-Subject: [RHEL-7.8/RHEL-8.1.0 cloud-init PATCH] Fix for network configuration not persisting after reboot +Bugzilla: 1593010 +RH-Acked-by: Mohammed Gamal +RH-Acked-by: Miroslav Rezanina + +The reasons the configuration does not persist after reboot includes +different aspects and they're all fixed on this patch: + + 1) The rpm package doesn't include the systemd-generator and +ds-identify. The systemd-generator is called early in the boot process +that calls ds-identify to check if there's any Data Source available in +the current boot. In the current use case, the Data Source is removed +from the VM on the second boot, this means cloud-init should disable +itself in order to keep the configuration it did in the first boot. + + 2) Even after adding those scripts, cloud-init was still being +executed and the configurations were being lost. The reason for this is +that the cloud-init systemd units had a wrong dependency + + WantedBy: multi-user.target + + Which would start them every time no matter the return of +ds-identify. The fix is to replace the dependency by the systemd unit to +cloud-init.target, which is the main cloud-init target enabled - or in +this case, disabled by ds-identify. The file cloud-init.target was also +missing on rpm package. + +After adding both scripts, the main cloud-init systemd target and +adjusting the systemd dependencies the configuration persists after +reboots and shutdowns. + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + redhat/cloud-init.spec.template | 15 +++++++++++++++ + rhel/systemd/cloud-config.service | 2 +- + rhel/systemd/cloud-final.service | 2 +- + rhel/systemd/cloud-init-local.service | 2 +- + rhel/systemd/cloud-init.service | 2 +- + rhel/systemd/cloud-init.target | 7 +++++++ + 6 files changed, 26 insertions(+), 4 deletions(-) + create mode 100644 rhel/systemd/cloud-init.target + +diff --git a/rhel/systemd/cloud-config.service b/rhel/systemd/cloud-config.service +index 12ca9df..f3dcd4b 100644 +--- a/rhel/systemd/cloud-config.service ++++ b/rhel/systemd/cloud-config.service +@@ -15,4 +15,4 @@ TimeoutSec=0 + StandardOutput=journal+console + + [Install] +-WantedBy=multi-user.target ++WantedBy=cloud-init.target +diff --git a/rhel/systemd/cloud-final.service b/rhel/systemd/cloud-final.service +index 32a83d8..739b7e3 100644 +--- a/rhel/systemd/cloud-final.service ++++ b/rhel/systemd/cloud-final.service +@@ -16,4 +16,4 @@ KillMode=process + StandardOutput=journal+console + + [Install] +-WantedBy=multi-user.target ++WantedBy=cloud-init.target +diff --git a/rhel/systemd/cloud-init-local.service b/rhel/systemd/cloud-init-local.service +index 656eddb..8f9f6c9 100644 +--- a/rhel/systemd/cloud-init-local.service ++++ b/rhel/systemd/cloud-init-local.service +@@ -28,4 +28,4 @@ TimeoutSec=0 + StandardOutput=journal+console + + [Install] +-WantedBy=multi-user.target ++WantedBy=cloud-init.target +diff --git a/rhel/systemd/cloud-init.service b/rhel/systemd/cloud-init.service +index 68fc5f1..d0023a0 100644 +--- a/rhel/systemd/cloud-init.service ++++ b/rhel/systemd/cloud-init.service +@@ -22,4 +22,4 @@ TimeoutSec=0 + StandardOutput=journal+console + + [Install] +-WantedBy=multi-user.target ++WantedBy=cloud-init.target +diff --git a/rhel/systemd/cloud-init.target b/rhel/systemd/cloud-init.target +new file mode 100644 +index 0000000..083c3b6 +--- /dev/null ++++ b/rhel/systemd/cloud-init.target +@@ -0,0 +1,7 @@ ++# cloud-init target is enabled by cloud-init-generator ++# To disable it you can either: ++# a.) boot with kernel cmdline of 'cloud-init=disabled' ++# b.) touch a file /etc/cloud/cloud-init.disabled ++[Unit] ++Description=Cloud-init target ++After=multi-user.target +-- +1.8.3.1 + diff --git a/SOURCES/ci-util-json.dumps-on-python-2.7-will-handle-UnicodeDec.patch b/SOURCES/ci-util-json.dumps-on-python-2.7-will-handle-UnicodeDec.patch new file mode 100644 index 0000000..a3919e4 --- /dev/null +++ b/SOURCES/ci-util-json.dumps-on-python-2.7-will-handle-UnicodeDec.patch @@ -0,0 +1,144 @@ +From 5a3cd50df652e4a70f85ccc712dc11bf9726adda Mon Sep 17 00:00:00 2001 +From: Eduardo Otubo +Date: Wed, 16 Oct 2019 12:10:24 +0200 +Subject: [PATCH] util: json.dumps on python 2.7 will handle UnicodeDecodeError + on binary + +RH-Author: Eduardo Otubo +Message-id: <20191016121024.23694-1-otubo@redhat.com> +Patchwork-id: 91812 +O-Subject: [RHEL-7.8/RHEL-8.1.0 cloud-init PATCH] util: json.dumps on python 2.7 will handle UnicodeDecodeError on binary +Bugzilla: 1744526 +RH-Acked-by: Vitaly Kuznetsov +RH-Acked-by: Mohammed Gamal + +commit 067516d7bc917e4921b9f1424b7a64e92cae0ad2 +Author: Chad Smith +Date: Fri Sep 27 20:46:00 2019 +0000 + + util: json.dumps on python 2.7 will handle UnicodeDecodeError on binary + + Since python 2.7 doesn't handle UnicodeDecodeErrors with the default + handler + + LP: #1801364 + +Signed-off-by: Eduardo Otubo +Signed-off-by: Miroslav Rezanina +--- + cloudinit/sources/tests/test_init.py | 12 +++++------- + cloudinit/tests/test_util.py | 20 ++++++++++++++++++++ + cloudinit/util.py | 27 +++++++++++++++++++++++++-- + 3 files changed, 50 insertions(+), 9 deletions(-) + +diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py +index 6378e98..9698261 100644 +--- a/cloudinit/sources/tests/test_init.py ++++ b/cloudinit/sources/tests/test_init.py +@@ -457,19 +457,17 @@ class TestDataSource(CiTestCase): + instance_json['ds']['meta_data']) + + @skipIf(not six.PY2, "Only python2 hits UnicodeDecodeErrors on non-utf8") +- def test_non_utf8_encoding_logs_warning(self): +- """When non-utf-8 values exist in py2 instance-data is not written.""" ++ def test_non_utf8_encoding_gets_b64encoded(self): ++ """When non-utf-8 values exist in py2 instance-data is b64encoded.""" + tmp = self.tmp_dir() + datasource = DataSourceTestSubclassNet( + self.sys_cfg, self.distro, Paths({'run_dir': tmp}), + custom_metadata={'key1': 'val1', 'key2': {'key2.1': b'ab\xaadef'}}) + self.assertTrue(datasource.get_data()) + json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) +- self.assertFalse(os.path.exists(json_file)) +- self.assertIn( +- "WARNING: Error persisting instance-data.json: 'utf8' codec can't" +- " decode byte 0xaa in position 2: invalid start byte", +- self.logs.getvalue()) ++ instance_json = util.load_json(util.load_file(json_file)) ++ key21_value = instance_json['ds']['meta_data']['key2']['key2.1'] ++ self.assertEqual('ci-b64:' + util.b64e(b'ab\xaadef'), key21_value) + + def test_get_hostname_subclass_support(self): + """Validate get_hostname signature on all subclasses of DataSource.""" +diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py +index e3d2dba..f4f95e9 100644 +--- a/cloudinit/tests/test_util.py ++++ b/cloudinit/tests/test_util.py +@@ -2,7 +2,9 @@ + + """Tests for cloudinit.util""" + ++import base64 + import logging ++import json + import platform + + import cloudinit.util as util +@@ -528,6 +530,24 @@ class TestGetLinuxDistro(CiTestCase): + self.assertEqual(('foo', '1.1', 'aarch64'), dist) + + ++class TestJsonDumps(CiTestCase): ++ def test_is_str(self): ++ """json_dumps should return a string.""" ++ self.assertTrue(isinstance(util.json_dumps({'abc': '123'}), str)) ++ ++ def test_utf8(self): ++ smiley = '\\ud83d\\ude03' ++ self.assertEqual( ++ {'smiley': smiley}, ++ json.loads(util.json_dumps({'smiley': smiley}))) ++ ++ def test_non_utf8(self): ++ blob = b'\xba\x03Qx-#y\xea' ++ self.assertEqual( ++ {'blob': 'ci-b64:' + base64.b64encode(blob).decode('utf-8')}, ++ json.loads(util.json_dumps({'blob': blob}))) ++ ++ + @mock.patch('os.path.exists') + class TestIsLXD(CiTestCase): + +diff --git a/cloudinit/util.py b/cloudinit/util.py +index a84112a..2c9ac66 100644 +--- a/cloudinit/util.py ++++ b/cloudinit/util.py +@@ -1590,10 +1590,33 @@ def json_serialize_default(_obj): + return 'Warning: redacted unserializable type {0}'.format(type(_obj)) + + ++def json_preserialize_binary(data): ++ """Preserialize any discovered binary values to avoid json.dumps issues. ++ ++ Used only on python 2.7 where default type handling is not honored for ++ failure to encode binary data. LP: #1801364. ++ TODO(Drop this function when py2.7 support is dropped from cloud-init) ++ """ ++ data = obj_copy.deepcopy(data) ++ for key, value in data.items(): ++ if isinstance(value, (dict)): ++ data[key] = json_preserialize_binary(value) ++ if isinstance(value, bytes): ++ data[key] = 'ci-b64:{0}'.format(b64e(value)) ++ return data ++ ++ + def json_dumps(data): + """Return data in nicely formatted json.""" +- return json.dumps(data, indent=1, sort_keys=True, +- separators=(',', ': '), default=json_serialize_default) ++ try: ++ return json.dumps( ++ data, indent=1, sort_keys=True, separators=(',', ': '), ++ default=json_serialize_default) ++ except UnicodeDecodeError: ++ if sys.version_info[:2] == (2, 7): ++ data = json_preserialize_binary(data) ++ return json.dumps(data) ++ raise + + + def yaml_dumps(obj, explicit_start=True, explicit_end=True): +-- +1.8.3.1 + diff --git a/SOURCES/cloud-init-centos-user.patch b/SOURCES/cloud-init-centos-user.patch deleted file mode 100644 index 3ebaa88..0000000 --- a/SOURCES/cloud-init-centos-user.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -uNrp cloud-init-18.2.orig/rhel/cloud.cfg cloud-init-18.2/rhel/cloud.cfg ---- cloud-init-18.2.orig/rhel/cloud.cfg 2018-11-04 15:38:13.763701007 +0000 -+++ cloud-init-18.2/rhel/cloud.cfg 2018-11-04 15:41:06.934576619 +0000 -@@ -52,7 +52,7 @@ cloud_final_modules: - - system_info: - default_user: -- name: cloud-user -+ name: centos - lock_passwd: true - gecos: Cloud User - groups: [wheel, adm, systemd-journal] diff --git a/SPECS/cloud-init.spec b/SPECS/cloud-init.spec index 810391a..7f60664 100644 --- a/SPECS/cloud-init.spec +++ b/SPECS/cloud-init.spec @@ -7,7 +7,7 @@ Name: cloud-init Version: 18.5 -Release: 3%{?dist} +Release: 6%{?dist} Summary: Cloud instance init scripts Group: System Environment/Base @@ -40,8 +40,13 @@ Patch16: ci-DataSourceAzure-Adjust-timeout-for-polling-IMDS.patch Patch17: ci-cc_mounts-check-if-mount-a-on-no-change-fstab-path.patch # For bz#1707725 - [WALA][cloud] cloud-init dhclient-hook script has some unexpected side-effects on Azure Patch18: ci-Revert-azure-ensure-that-networkmanager-hook-script-.patch +# For bz#1726701 - [Azure] [RHEL 7.8] Cloud-init fixes to support fast provisioning for Azure +Patch19: ci-Azure-Return-static-fallback-address-as-if-failed-to.patch +# For bz#1593010 - [cloud-init][RHVM]cloud-init network configuration does not persist reboot [RHEL 7.8] +Patch20: ci-Fix-for-network-configuration-not-persisting-after-r.patch +# For bz#1744526 - [cloud-init][OpenStack] cloud-init can't persist instance-data.json +Patch21: ci-util-json.dumps-on-python-2.7-will-handle-UnicodeDec.patch -Patch9999: cloud-init-centos-user.patch # Deal with noarch -> arch # https://bugzilla.redhat.com/show_bug.cgi?id=1067089 Obsoletes: cloud-init < 0.7.5-3 @@ -121,6 +126,12 @@ mv $RPM_BUILD_ROOT/etc/NetworkManager/dispatcher.d/hook-network-manager \ mkdir -p $RPM_BUILD_ROOT%{_unitdir} cp rhel/systemd/* $RPM_BUILD_ROOT%{_unitdir}/ +[ ! -d $RPM_BUILD_ROOT/usr/lib/systemd/system-generators ] && mkdir -p $RPM_BUILD_ROOT/usr/lib/systemd/system-generators +cp -p systemd/cloud-init-generator $RPM_BUILD_ROOT/usr/lib/systemd/system-generators + +[ ! -d $RPM_BUILD_ROOT/usr/lib/%{name} ] && mkdir -p $RPM_BUILD_ROOT/usr/lib/%{name} +cp -p tools/ds-identify $RPM_BUILD_ROOT/usr/lib/%{name}/ds-identify + %clean rm -rf $RPM_BUILD_ROOT @@ -134,6 +145,7 @@ if [ $1 -eq 1 ] ; then /bin/systemctl enable cloud-final.service >/dev/null 2>&1 || : /bin/systemctl enable cloud-init.service >/dev/null 2>&1 || : /bin/systemctl enable cloud-init-local.service >/dev/null 2>&1 || : + /bin/systemctl enable cloud-init.target >/dev/null 2>&1 || : elif [ $1 -eq 2 ]; then # Upgrade. If the upgrade is from a version older than 0.7.9-8, # there will be stale systemd config @@ -148,6 +160,9 @@ elif [ $1 -eq 2 ]; then /bin/systemctl is-enabled cloud-init-local.service >/dev/null 2>&1 && /bin/systemctl reenable cloud-init-local.service >/dev/null 2>&1 || : + + /bin/systemctl is-enabled cloud-init.target >/dev/null 2>&1 && + /bin/systemctl reenable cloud-init.target >/dev/null 2>&1 || : fi %preun @@ -157,6 +172,7 @@ if [ $1 -eq 0 ] ; then /bin/systemctl --no-reload disable cloud-final.service >/dev/null 2>&1 || : /bin/systemctl --no-reload disable cloud-init.service >/dev/null 2>&1 || : /bin/systemctl --no-reload disable cloud-init-local.service >/dev/null 2>&1 || : + /bin/systemctl --no-reload disable cloud-init.target >/dev/null 2>&1 || : # One-shot services -> no need to stop fi @@ -179,6 +195,7 @@ fi %{_unitdir}/cloud-final.service %{_unitdir}/cloud-init-local.service %{_unitdir}/cloud-init.service +%{_unitdir}/cloud-init.target %{_tmpfilesdir}/%{name}.conf %{python_sitelib}/* %{_libexecdir}/%{name} @@ -190,11 +207,29 @@ fi %{_udevrulesdir}/66-azure-ephemeral.rules %{_sysconfdir}/bash_completion.d/cloud-init %{_bindir}/cloud-id +/usr/lib/%{name}/ds-identify +/usr/lib/systemd/system-generators/cloud-init-generator + %dir %{_sysconfdir}/rsyslog.d %config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf %changelog +* Thu Oct 24 2019 Miroslav Rezanina - 18.5-6.el7 +- ci-util-json.dumps-on-python-2.7-will-handle-UnicodeDec.patch [bz#1744526] +- Resolves: bz#1744526 + ([cloud-init][OpenStack] cloud-init can't persist instance-data.json) + +* Tue Sep 10 2019 Miroslav Rezanina - 18.5-5.el7 +- ci-Fix-for-network-configuration-not-persisting-after-r.patch [bz#1593010] +- Resolves: bz#1593010 + ([cloud-init][RHVM]cloud-init network configuration does not persist reboot [RHEL 7.8]) + +* Tue Aug 20 2019 Miroslav Rezanina - 18.5-4.el7 +- ci-Azure-Return-static-fallback-address-as-if-failed-to.patch [bz#1726701] +- Resolves: bz#1726701 + ([Azure] [RHEL 7.8] Cloud-init fixes to support fast provisioning for Azure) + * Tue May 28 2019 Miroslav Rezanina - 18.5-3.el7 - ci-Revert-azure-ensure-that-networkmanager-hook-script-.patch [bz#1707725] - Resolves: bz#1707725