diff --git a/SOURCES/ci-Add-flexibility-to-IMDS-api-version-793.patch b/SOURCES/ci-Add-flexibility-to-IMDS-api-version-793.patch
new file mode 100644
index 0000000..4c3dbc3
--- /dev/null
+++ b/SOURCES/ci-Add-flexibility-to-IMDS-api-version-793.patch
@@ -0,0 +1,295 @@
+From f844e9c263e59a623ca8c647bd87bf4f91374d54 Mon Sep 17 00:00:00 2001
+From: Thomas Stringer <thstring@microsoft.com>
+Date: Wed, 3 Mar 2021 11:07:43 -0500
+Subject: [PATCH 1/7] Add flexibility to IMDS api-version (#793)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [1/7] 99a3db20e3f277a2f12ea21e937e06939434a2ca (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+
+Add flexibility to IMDS api-version by having both a desired IMDS
+api-version and a minimum api-version. The desired api-version will
+be used first, and if that fails it will fall back to the minimum
+api-version.
+---
+ cloudinit/sources/DataSourceAzure.py | 113 ++++++++++++++----
+ tests/unittests/test_datasource/test_azure.py | 42 ++++++-
+ 2 files changed, 129 insertions(+), 26 deletions(-)
+
+diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
+index 553b5a7e..de1452ce 100755
+--- a/cloudinit/sources/DataSourceAzure.py
++++ b/cloudinit/sources/DataSourceAzure.py
+@@ -78,17 +78,15 @@ AGENT_SEED_DIR = '/var/lib/waagent'
+ # In the event where the IMDS primary server is not
+ # available, it takes 1s to fallback to the secondary one
+ IMDS_TIMEOUT_IN_SECONDS = 2
+-IMDS_URL = "http://169.254.169.254/metadata/"
+-IMDS_VER = "2019-06-01"
+-IMDS_VER_PARAM = "api-version={}".format(IMDS_VER)
++IMDS_URL = "http://169.254.169.254/metadata"
++IMDS_VER_MIN = "2019-06-01"
++IMDS_VER_WANT = "2020-09-01"
+
+
+ class metadata_type(Enum):
+- compute = "{}instance?{}".format(IMDS_URL, IMDS_VER_PARAM)
+- network = "{}instance/network?{}".format(IMDS_URL,
+- IMDS_VER_PARAM)
+- reprovisiondata = "{}reprovisiondata?{}".format(IMDS_URL,
+- IMDS_VER_PARAM)
++ compute = "{}/instance".format(IMDS_URL)
++ network = "{}/instance/network".format(IMDS_URL)
++ reprovisiondata = "{}/reprovisiondata".format(IMDS_URL)
+
+
+ PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0"
+@@ -349,6 +347,8 @@ class DataSourceAzure(sources.DataSource):
+ self.update_events['network'].add(EventType.BOOT)
+ self._ephemeral_dhcp_ctx = None
+
++ self.failed_desired_api_version = False
++
+ def __str__(self):
+ root = sources.DataSource.__str__(self)
+ return "%s [seed=%s]" % (root, self.seed)
+@@ -520,8 +520,10 @@ class DataSourceAzure(sources.DataSource):
+ self._wait_for_all_nics_ready()
+ ret = self._reprovision()
+
+- imds_md = get_metadata_from_imds(
+- self.fallback_interface, retries=10)
++ imds_md = self.get_imds_data_with_api_fallback(
++ self.fallback_interface,
++ retries=10
++ )
+ (md, userdata_raw, cfg, files) = ret
+ self.seed = cdev
+ crawled_data.update({
+@@ -652,6 +654,57 @@ class DataSourceAzure(sources.DataSource):
+ self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700)
+ return True
+
++ @azure_ds_telemetry_reporter
++ def get_imds_data_with_api_fallback(
++ self,
++ fallback_nic,
++ retries,
++ md_type=metadata_type.compute):
++ """
++ Wrapper for get_metadata_from_imds so that we can have flexibility
++ in which IMDS api-version we use. If a particular instance of IMDS
++ does not have the api version that is desired, we want to make
++ this fault tolerant and fall back to a good known minimum api
++ version.
++ """
++
++ if not self.failed_desired_api_version:
++ for _ in range(retries):
++ try:
++ LOG.info(
++ "Attempting IMDS api-version: %s",
++ IMDS_VER_WANT
++ )
++ return get_metadata_from_imds(
++ fallback_nic=fallback_nic,
++ retries=0,
++ md_type=md_type,
++ api_version=IMDS_VER_WANT
++ )
++ except UrlError as err:
++ LOG.info(
++ "UrlError with IMDS api-version: %s",
++ IMDS_VER_WANT
++ )
++ if err.code == 400:
++ log_msg = "Fall back to IMDS api-version: {}".format(
++ IMDS_VER_MIN
++ )
++ report_diagnostic_event(
++ log_msg,
++ logger_func=LOG.info
++ )
++ self.failed_desired_api_version = True
++ break
++
++ LOG.info("Using IMDS api-version: %s", IMDS_VER_MIN)
++ return get_metadata_from_imds(
++ fallback_nic=fallback_nic,
++ retries=retries,
++ md_type=md_type,
++ api_version=IMDS_VER_MIN
++ )
++
+ def device_name_to_device(self, name):
+ return self.ds_cfg['disk_aliases'].get(name)
+
+@@ -880,10 +933,11 @@ class DataSourceAzure(sources.DataSource):
+ # primary nic is being attached first helps here. Otherwise each nic
+ # could add several seconds of delay.
+ try:
+- imds_md = get_metadata_from_imds(
++ imds_md = self.get_imds_data_with_api_fallback(
+ ifname,
+ 5,
+- metadata_type.network)
++ metadata_type.network
++ )
+ except Exception as e:
+ LOG.warning(
+ "Failed to get network metadata using nic %s. Attempt to "
+@@ -1017,7 +1071,10 @@ class DataSourceAzure(sources.DataSource):
+ def _poll_imds(self):
+ """Poll IMDS for the new provisioning data until we get a valid
+ response. Then return the returned JSON object."""
+- url = metadata_type.reprovisiondata.value
++ url = "{}?api-version={}".format(
++ metadata_type.reprovisiondata.value,
++ IMDS_VER_MIN
++ )
+ headers = {"Metadata": "true"}
+ nl_sock = None
+ report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
+@@ -2059,7 +2116,8 @@ def _generate_network_config_from_fallback_config() -> dict:
+ @azure_ds_telemetry_reporter
+ def get_metadata_from_imds(fallback_nic,
+ retries,
+- md_type=metadata_type.compute):
++ md_type=metadata_type.compute,
++ api_version=IMDS_VER_MIN):
+ """Query Azure's instance metadata service, returning a dictionary.
+
+ If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
+@@ -2069,13 +2127,16 @@ def get_metadata_from_imds(fallback_nic,
+ @param fallback_nic: String. The name of the nic which requires active
+ network in order to query IMDS.
+ @param retries: The number of retries of the IMDS_URL.
++ @param md_type: Metadata type for IMDS request.
++ @param api_version: IMDS api-version to use in the request.
+
+ @return: A dict of instance metadata containing compute and network
+ info.
+ """
+ kwargs = {'logfunc': LOG.debug,
+ 'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
+- 'func': _get_metadata_from_imds, 'args': (retries, md_type,)}
++ 'func': _get_metadata_from_imds,
++ 'args': (retries, md_type, api_version,)}
+ if net.is_up(fallback_nic):
+ return util.log_time(**kwargs)
+ else:
+@@ -2091,20 +2152,26 @@ def get_metadata_from_imds(fallback_nic,
+
+
+ @azure_ds_telemetry_reporter
+-def _get_metadata_from_imds(retries, md_type=metadata_type.compute):
+-
+- url = md_type.value
++def _get_metadata_from_imds(
++ retries,
++ md_type=metadata_type.compute,
++ api_version=IMDS_VER_MIN):
++ url = "{}?api-version={}".format(md_type.value, api_version)
+ headers = {"Metadata": "true"}
+ try:
+ response = readurl(
+ url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers,
+ retries=retries, exception_cb=retry_on_url_exc)
+ except Exception as e:
+- report_diagnostic_event(
+- 'Ignoring IMDS instance metadata. '
+- 'Get metadata from IMDS failed: %s' % e,
+- logger_func=LOG.warning)
+- return {}
++ # pylint:disable=no-member
++ if isinstance(e, UrlError) and e.code == 400:
++ raise
++ else:
++ report_diagnostic_event(
++ 'Ignoring IMDS instance metadata. '
++ 'Get metadata from IMDS failed: %s' % e,
++ logger_func=LOG.warning)
++ return {}
+ try:
+ from json.decoder import JSONDecodeError
+ json_decode_error = JSONDecodeError
+diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
+index f597c723..dedebeb1 100644
+--- a/tests/unittests/test_datasource/test_azure.py
++++ b/tests/unittests/test_datasource/test_azure.py
+@@ -408,7 +408,9 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
+
+ def setUp(self):
+ super(TestGetMetadataFromIMDS, self).setUp()
+- self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2019-06-01"
++ self.network_md_url = "{}/instance?api-version=2019-06-01".format(
++ dsaz.IMDS_URL
++ )
+
+ @mock.patch(MOCKPATH + 'readurl')
+ @mock.patch(MOCKPATH + 'EphemeralDHCPv4', autospec=True)
+@@ -518,7 +520,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
+ """Return empty dict when IMDS network metadata is absent."""
+ httpretty.register_uri(
+ httpretty.GET,
+- dsaz.IMDS_URL + 'instance?api-version=2017-12-01',
++ dsaz.IMDS_URL + '/instance?api-version=2017-12-01',
+ body={}, status=404)
+
+ m_net_is_up.return_value = True # skips dhcp
+@@ -1877,6 +1879,40 @@ scbus-1 on xpt0 bus 0
+ ssh_keys = dsrc.get_public_ssh_keys()
+ self.assertEqual(ssh_keys, ['key2'])
+
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_imds_api_version_wanted_nonexistent(
++ self,
++ m_get_metadata_from_imds):
++ def get_metadata_from_imds_side_eff(*args, **kwargs):
++ if kwargs['api_version'] == dsaz.IMDS_VER_WANT:
++ raise url_helper.UrlError("No IMDS version", code=400)
++ return NETWORK_METADATA
++ m_get_metadata_from_imds.side_effect = get_metadata_from_imds_side_eff
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ dsrc = self._get_ds(data)
++ dsrc.get_data()
++ self.assertIsNotNone(dsrc.metadata)
++ self.assertTrue(dsrc.failed_desired_api_version)
++
++ @mock.patch(
++ MOCKPATH + 'get_metadata_from_imds', return_value=NETWORK_METADATA)
++ def test_imds_api_version_wanted_exists(self, m_get_metadata_from_imds):
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ dsrc = self._get_ds(data)
++ dsrc.get_data()
++ self.assertIsNotNone(dsrc.metadata)
++ self.assertFalse(dsrc.failed_desired_api_version)
++
+
+ class TestAzureBounce(CiTestCase):
+
+@@ -2657,7 +2693,7 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
+ @mock.patch(MOCKPATH + 'DataSourceAzure.wait_for_link_up')
+ @mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_attach_event')
+ @mock.patch('cloudinit.sources.net.find_fallback_nic')
+- @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback')
+ @mock.patch(MOCKPATH + 'EphemeralDHCPv4')
+ @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach')
+ @mock.patch('os.path.isfile')
+--
+2.27.0
+
diff --git a/SOURCES/ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch b/SOURCES/ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch
new file mode 100644
index 0000000..c30ed09
--- /dev/null
+++ b/SOURCES/ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch
@@ -0,0 +1,397 @@
+From 68f058e8d20a499f74bc78af8e0c6a90ca57ae20 Mon Sep 17 00:00:00 2001
+From: Thomas Stringer <thstring@microsoft.com>
+Date: Mon, 26 Apr 2021 09:41:38 -0400
+Subject: [PATCH 5/7] Azure: Retrieve username and hostname from IMDS (#865)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [5/7] 6a768d31e63e5f00dae0fad2712a7618d62b0879 (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+
+This change allows us to retrieve the username and hostname from
+IMDS instead of having to rely on the mounted OVF.
+---
+ cloudinit/sources/DataSourceAzure.py | 149 ++++++++++++++----
+ tests/unittests/test_datasource/test_azure.py | 87 +++++++++-
+ 2 files changed, 205 insertions(+), 31 deletions(-)
+
+diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
+index 39e67c4f..6d7954ee 100755
+--- a/cloudinit/sources/DataSourceAzure.py
++++ b/cloudinit/sources/DataSourceAzure.py
+@@ -5,6 +5,7 @@
+ # This file is part of cloud-init. See LICENSE file for license information.
+
+ import base64
++from collections import namedtuple
+ import contextlib
+ import crypt
+ from functools import partial
+@@ -25,6 +26,7 @@ from cloudinit.net import device_driver
+ from cloudinit.net.dhcp import EphemeralDHCPv4
+ from cloudinit import sources
+ from cloudinit.sources.helpers import netlink
++from cloudinit import ssh_util
+ from cloudinit import subp
+ from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc
+ from cloudinit import util
+@@ -80,7 +82,12 @@ AGENT_SEED_DIR = '/var/lib/waagent'
+ IMDS_TIMEOUT_IN_SECONDS = 2
+ IMDS_URL = "http://169.254.169.254/metadata"
+ IMDS_VER_MIN = "2019-06-01"
+-IMDS_VER_WANT = "2020-09-01"
++IMDS_VER_WANT = "2020-10-01"
++
++
++# This holds SSH key data including if the source was
++# from IMDS, as well as the SSH key data itself.
++SSHKeys = namedtuple("SSHKeys", ("keys_from_imds", "ssh_keys"))
+
+
+ class metadata_type(Enum):
+@@ -391,6 +398,8 @@ class DataSourceAzure(sources.DataSource):
+ """Return the subplatform metadata source details."""
+ if self.seed.startswith('/dev'):
+ subplatform_type = 'config-disk'
++ elif self.seed.lower() == 'imds':
++ subplatform_type = 'imds'
+ else:
+ subplatform_type = 'seed-dir'
+ return '%s (%s)' % (subplatform_type, self.seed)
+@@ -433,9 +442,11 @@ class DataSourceAzure(sources.DataSource):
+
+ found = None
+ reprovision = False
++ ovf_is_accessible = True
+ reprovision_after_nic_attach = False
+ for cdev in candidates:
+ try:
++ LOG.debug("cdev: %s", cdev)
+ if cdev == "IMDS":
+ ret = None
+ reprovision = True
+@@ -462,8 +473,18 @@ class DataSourceAzure(sources.DataSource):
+ raise sources.InvalidMetaDataException(msg)
+ except util.MountFailedError:
+ report_diagnostic_event(
+- '%s was not mountable' % cdev, logger_func=LOG.warning)
+- continue
++ '%s was not mountable' % cdev, logger_func=LOG.debug)
++ cdev = 'IMDS'
++ ovf_is_accessible = False
++ empty_md = {'local-hostname': ''}
++ empty_cfg = dict(
++ system_info=dict(
++ default_user=dict(
++ name=''
++ )
++ )
++ )
++ ret = (empty_md, '', empty_cfg, {})
+
+ report_diagnostic_event("Found provisioning metadata in %s" % cdev,
+ logger_func=LOG.debug)
+@@ -490,6 +511,10 @@ class DataSourceAzure(sources.DataSource):
+ self.fallback_interface,
+ retries=10
+ )
++ if not imds_md and not ovf_is_accessible:
++ msg = 'No OVF or IMDS available'
++ report_diagnostic_event(msg)
++ raise sources.InvalidMetaDataException(msg)
+ (md, userdata_raw, cfg, files) = ret
+ self.seed = cdev
+ crawled_data.update({
+@@ -498,6 +523,21 @@ class DataSourceAzure(sources.DataSource):
+ 'metadata': util.mergemanydict(
+ [md, {'imds': imds_md}]),
+ 'userdata_raw': userdata_raw})
++ imds_username = _username_from_imds(imds_md)
++ imds_hostname = _hostname_from_imds(imds_md)
++ imds_disable_password = _disable_password_from_imds(imds_md)
++ if imds_username:
++ LOG.debug('Username retrieved from IMDS: %s', imds_username)
++ cfg['system_info']['default_user']['name'] = imds_username
++ if imds_hostname:
++ LOG.debug('Hostname retrieved from IMDS: %s', imds_hostname)
++ crawled_data['metadata']['local-hostname'] = imds_hostname
++ if imds_disable_password:
++ LOG.debug(
++ 'Disable password retrieved from IMDS: %s',
++ imds_disable_password
++ )
++ crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501
+ found = cdev
+
+ report_diagnostic_event(
+@@ -676,6 +716,13 @@ class DataSourceAzure(sources.DataSource):
+
+ @azure_ds_telemetry_reporter
+ def get_public_ssh_keys(self):
++ """
++ Retrieve public SSH keys.
++ """
++
++ return self._get_public_ssh_keys_and_source().ssh_keys
++
++ def _get_public_ssh_keys_and_source(self):
+ """
+ Try to get the ssh keys from IMDS first, and if that fails
+ (i.e. IMDS is unavailable) then fallback to getting the ssh
+@@ -685,30 +732,50 @@ class DataSourceAzure(sources.DataSource):
+ advantage, so this is a strong preference. But we must keep
+ OVF as a second option for environments that don't have IMDS.
+ """
++
+ LOG.debug('Retrieving public SSH keys')
+ ssh_keys = []
++ keys_from_imds = True
++ LOG.debug('Attempting to get SSH keys from IMDS')
+ try:
+- raise KeyError(
+- "Not using public SSH keys from IMDS"
+- )
+- # pylint:disable=unreachable
+ ssh_keys = [
+ public_key['keyData']
+ for public_key
+ in self.metadata['imds']['compute']['publicKeys']
+ ]
+- LOG.debug('Retrieved SSH keys from IMDS')
++ for key in ssh_keys:
++ if not _key_is_openssh_formatted(key=key):
++ keys_from_imds = False
++ break
++
++ if not keys_from_imds:
++ log_msg = 'Keys not in OpenSSH format, using OVF'
++ else:
++ log_msg = 'Retrieved {} keys from IMDS'.format(
++ len(ssh_keys)
++ if ssh_keys is not None
++ else 0
++ )
+ except KeyError:
+ log_msg = 'Unable to get keys from IMDS, falling back to OVF'
++ keys_from_imds = False
++ finally:
+ report_diagnostic_event(log_msg, logger_func=LOG.debug)
++
++ if not keys_from_imds:
++ LOG.debug('Attempting to get SSH keys from OVF')
+ try:
+ ssh_keys = self.metadata['public-keys']
+- LOG.debug('Retrieved keys from OVF')
++ log_msg = 'Retrieved {} keys from OVF'.format(len(ssh_keys))
+ except KeyError:
+ log_msg = 'No keys available from OVF'
++ finally:
+ report_diagnostic_event(log_msg, logger_func=LOG.debug)
+
+- return ssh_keys
++ return SSHKeys(
++ keys_from_imds=keys_from_imds,
++ ssh_keys=ssh_keys
++ )
+
+ def get_config_obj(self):
+ return self.cfg
+@@ -1325,30 +1392,21 @@ class DataSourceAzure(sources.DataSource):
+ self.bounce_network_with_azure_hostname()
+
+ pubkey_info = None
+- try:
+- raise KeyError(
+- "Not using public SSH keys from IMDS"
+- )
+- # pylint:disable=unreachable
+- public_keys = self.metadata['imds']['compute']['publicKeys']
+- LOG.debug(
+- 'Successfully retrieved %s key(s) from IMDS',
+- len(public_keys)
+- if public_keys is not None
++ ssh_keys_and_source = self._get_public_ssh_keys_and_source()
++
++ if not ssh_keys_and_source.keys_from_imds:
++ pubkey_info = self.cfg.get('_pubkeys', None)
++ log_msg = 'Retrieved {} fingerprints from OVF'.format(
++ len(pubkey_info)
++ if pubkey_info is not None
+ else 0
+ )
+- except KeyError:
+- LOG.debug(
+- 'Unable to retrieve SSH keys from IMDS during '
+- 'negotiation, falling back to OVF'
+- )
+- pubkey_info = self.cfg.get('_pubkeys', None)
++ report_diagnostic_event(log_msg, logger_func=LOG.debug)
+
+ metadata_func = partial(get_metadata_from_fabric,
+ fallback_lease_file=self.
+ dhclient_lease_file,
+- pubkey_info=pubkey_info,
+- iso_dev=self.iso_dev)
++ pubkey_info=pubkey_info)
+
+ LOG.debug("negotiating with fabric via agent command %s",
+ self.ds_cfg['agent_command'])
+@@ -1404,6 +1462,41 @@ class DataSourceAzure(sources.DataSource):
+ return self.metadata.get('imds', {}).get('compute', {}).get('location')
+
+
++def _username_from_imds(imds_data):
++ try:
++ return imds_data['compute']['osProfile']['adminUsername']
++ except KeyError:
++ return None
++
++
++def _hostname_from_imds(imds_data):
++ try:
++ return imds_data['compute']['osProfile']['computerName']
++ except KeyError:
++ return None
++
++
++def _disable_password_from_imds(imds_data):
++ try:
++ return imds_data['compute']['osProfile']['disablePasswordAuthentication'] == 'true' # noqa: E501
++ except KeyError:
++ return None
++
++
++def _key_is_openssh_formatted(key):
++ """
++ Validate whether or not the key is OpenSSH-formatted.
++ """
++
++ parser = ssh_util.AuthKeyLineParser()
++ try:
++ akl = parser.parse(key)
++ except TypeError:
++ return False
++
++ return akl.keytype is not None
++
++
+ def _partitions_on_device(devpath, maxnum=16):
+ # return a list of tuples (ptnum, path) for each part on devpath
+ for suff in ("-part", "p", ""):
+diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
+index 320fa857..d9817d84 100644
+--- a/tests/unittests/test_datasource/test_azure.py
++++ b/tests/unittests/test_datasource/test_azure.py
+@@ -108,7 +108,7 @@ NETWORK_METADATA = {
+ "zone": "",
+ "publicKeys": [
+ {
+- "keyData": "key1",
++ "keyData": "ssh-rsa key1",
+ "path": "path1"
+ }
+ ]
+@@ -1761,8 +1761,29 @@ scbus-1 on xpt0 bus 0
+ dsrc.get_data()
+ dsrc.setup(True)
+ ssh_keys = dsrc.get_public_ssh_keys()
+- # Temporarily alter this test so that SSH public keys
+- # from IMDS are *not* going to be in use to fix a regression.
++ self.assertEqual(ssh_keys, ["ssh-rsa key1"])
++ self.assertEqual(m_parse_certificates.call_count, 0)
++
++ @mock.patch(
++ 'cloudinit.sources.helpers.azure.OpenSSLManager.parse_certificates')
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_get_public_ssh_keys_with_no_openssh_format(
++ self,
++ m_get_metadata_from_imds,
++ m_parse_certificates):
++ imds_data = copy.deepcopy(NETWORK_METADATA)
++ imds_data['compute']['publicKeys'][0]['keyData'] = 'no-openssh-format'
++ m_get_metadata_from_imds.return_value = imds_data
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ dsrc = self._get_ds(data)
++ dsrc.get_data()
++ dsrc.setup(True)
++ ssh_keys = dsrc.get_public_ssh_keys()
+ self.assertEqual(ssh_keys, [])
+ self.assertEqual(m_parse_certificates.call_count, 0)
+
+@@ -1818,6 +1839,66 @@ scbus-1 on xpt0 bus 0
+ self.assertIsNotNone(dsrc.metadata)
+ self.assertFalse(dsrc.failed_desired_api_version)
+
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_hostname_from_imds(self, m_get_metadata_from_imds):
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA)
++ imds_data_with_os_profile["compute"]["osProfile"] = dict(
++ adminUsername="username1",
++ computerName="hostname1",
++ disablePasswordAuthentication="true"
++ )
++ m_get_metadata_from_imds.return_value = imds_data_with_os_profile
++ dsrc = self._get_ds(data)
++ dsrc.get_data()
++ self.assertEqual(dsrc.metadata["local-hostname"], "hostname1")
++
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_username_from_imds(self, m_get_metadata_from_imds):
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA)
++ imds_data_with_os_profile["compute"]["osProfile"] = dict(
++ adminUsername="username1",
++ computerName="hostname1",
++ disablePasswordAuthentication="true"
++ )
++ m_get_metadata_from_imds.return_value = imds_data_with_os_profile
++ dsrc = self._get_ds(data)
++ dsrc.get_data()
++ self.assertEqual(
++ dsrc.cfg["system_info"]["default_user"]["name"],
++ "username1"
++ )
++
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_disable_password_from_imds(self, m_get_metadata_from_imds):
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA)
++ imds_data_with_os_profile["compute"]["osProfile"] = dict(
++ adminUsername="username1",
++ computerName="hostname1",
++ disablePasswordAuthentication="true"
++ )
++ m_get_metadata_from_imds.return_value = imds_data_with_os_profile
++ dsrc = self._get_ds(data)
++ dsrc.get_data()
++ self.assertTrue(dsrc.metadata["disable_password"])
++
+
+ class TestAzureBounce(CiTestCase):
+
+--
+2.27.0
+
diff --git a/SOURCES/ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch b/SOURCES/ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch
new file mode 100644
index 0000000..2d02d7d
--- /dev/null
+++ b/SOURCES/ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch
@@ -0,0 +1,315 @@
+From 816fe5c2e6d5dcc68f292092b00b2acfbc4c8e88 Mon Sep 17 00:00:00 2001
+From: aswinrajamannar <39812128+aswinrajamannar@users.noreply.github.com>
+Date: Mon, 26 Apr 2021 07:28:39 -0700
+Subject: [PATCH 6/7] Azure: Retry net metadata during nic attach for
+ non-timeout errs (#878)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [6/7] 794cd340644260bb43a7c8582a8067f403b9842d (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+
+When network interfaces are hot-attached to the VM, attempting to get
+network metadata might return 410 (or 500, 503 etc) because the info
+is not yet available. In those cases, we retry getting the metadata
+before giving up. The only case where we can move on to wait for more
+nic attach events is if the call times out despite retries, which
+means the interface is not likely a primary interface, and we should
+try for more nic attach events.
+---
+ cloudinit/sources/DataSourceAzure.py | 65 +++++++++++--
+ tests/unittests/test_datasource/test_azure.py | 95 ++++++++++++++++---
+ 2 files changed, 140 insertions(+), 20 deletions(-)
+
+diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
+index 6d7954ee..d0be6d84 100755
+--- a/cloudinit/sources/DataSourceAzure.py
++++ b/cloudinit/sources/DataSourceAzure.py
+@@ -17,6 +17,7 @@ from time import sleep
+ from xml.dom import minidom
+ import xml.etree.ElementTree as ET
+ from enum import Enum
++import requests
+
+ from cloudinit import dmi
+ from cloudinit import log as logging
+@@ -665,7 +666,9 @@ class DataSourceAzure(sources.DataSource):
+ self,
+ fallback_nic,
+ retries,
+- md_type=metadata_type.compute):
++ md_type=metadata_type.compute,
++ exc_cb=retry_on_url_exc,
++ infinite=False):
+ """
+ Wrapper for get_metadata_from_imds so that we can have flexibility
+ in which IMDS api-version we use. If a particular instance of IMDS
+@@ -685,7 +688,8 @@ class DataSourceAzure(sources.DataSource):
+ fallback_nic=fallback_nic,
+ retries=0,
+ md_type=md_type,
+- api_version=IMDS_VER_WANT
++ api_version=IMDS_VER_WANT,
++ exc_cb=exc_cb
+ )
+ except UrlError as err:
+ LOG.info(
+@@ -708,7 +712,9 @@ class DataSourceAzure(sources.DataSource):
+ fallback_nic=fallback_nic,
+ retries=retries,
+ md_type=md_type,
+- api_version=IMDS_VER_MIN
++ api_version=IMDS_VER_MIN,
++ exc_cb=exc_cb,
++ infinite=infinite
+ )
+
+ def device_name_to_device(self, name):
+@@ -938,6 +944,9 @@ class DataSourceAzure(sources.DataSource):
+ is_primary = False
+ expected_nic_count = -1
+ imds_md = None
++ metadata_poll_count = 0
++ metadata_logging_threshold = 1
++ metadata_timeout_count = 0
+
+ # For now, only a VM's primary NIC can contact IMDS and WireServer. If
+ # DHCP fails for a NIC, we have no mechanism to determine if the NIC is
+@@ -962,14 +971,48 @@ class DataSourceAzure(sources.DataSource):
+ % (ifname, e), logger_func=LOG.error)
+ raise
+
++ # Retry polling network metadata for a limited duration only when the
++ # calls fail due to timeout. This is because the platform drops packets
++ # going towards IMDS when it is not a primary nic. If the calls fail
++ # due to other issues like 410, 503 etc, then it means we are primary
++ # but IMDS service is unavailable at the moment. Retry indefinitely in
++ # those cases since we cannot move on without the network metadata.
++ def network_metadata_exc_cb(msg, exc):
++ nonlocal metadata_timeout_count, metadata_poll_count
++ nonlocal metadata_logging_threshold
++
++ metadata_poll_count = metadata_poll_count + 1
++
++ # Log when needed but back off exponentially to avoid exploding
++ # the log file.
++ if metadata_poll_count >= metadata_logging_threshold:
++ metadata_logging_threshold *= 2
++ report_diagnostic_event(
++ "Ran into exception when attempting to reach %s "
++ "after %d polls." % (msg, metadata_poll_count),
++ logger_func=LOG.error)
++
++ if isinstance(exc, UrlError):
++ report_diagnostic_event("poll IMDS with %s failed. "
++ "Exception: %s and code: %s" %
++ (msg, exc.cause, exc.code),
++ logger_func=LOG.error)
++
++ if exc.cause and isinstance(exc.cause, requests.Timeout):
++ metadata_timeout_count = metadata_timeout_count + 1
++ return (metadata_timeout_count <= 10)
++ return True
++
+ # Primary nic detection will be optimized in the future. The fact that
+ # primary nic is being attached first helps here. Otherwise each nic
+ # could add several seconds of delay.
+ try:
+ imds_md = self.get_imds_data_with_api_fallback(
+ ifname,
+- 5,
+- metadata_type.network
++ 0,
++ metadata_type.network,
++ network_metadata_exc_cb,
++ True
+ )
+ except Exception as e:
+ LOG.warning(
+@@ -2139,7 +2182,9 @@ def _generate_network_config_from_fallback_config() -> dict:
+ def get_metadata_from_imds(fallback_nic,
+ retries,
+ md_type=metadata_type.compute,
+- api_version=IMDS_VER_MIN):
++ api_version=IMDS_VER_MIN,
++ exc_cb=retry_on_url_exc,
++ infinite=False):
+ """Query Azure's instance metadata service, returning a dictionary.
+
+ If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
+@@ -2158,7 +2203,7 @@ def get_metadata_from_imds(fallback_nic,
+ kwargs = {'logfunc': LOG.debug,
+ 'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
+ 'func': _get_metadata_from_imds,
+- 'args': (retries, md_type, api_version,)}
++ 'args': (retries, exc_cb, md_type, api_version, infinite)}
+ if net.is_up(fallback_nic):
+ return util.log_time(**kwargs)
+ else:
+@@ -2176,14 +2221,16 @@ def get_metadata_from_imds(fallback_nic,
+ @azure_ds_telemetry_reporter
+ def _get_metadata_from_imds(
+ retries,
++ exc_cb,
+ md_type=metadata_type.compute,
+- api_version=IMDS_VER_MIN):
++ api_version=IMDS_VER_MIN,
++ infinite=False):
+ url = "{}?api-version={}".format(md_type.value, api_version)
+ headers = {"Metadata": "true"}
+ try:
+ response = readurl(
+ url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers,
+- retries=retries, exception_cb=retry_on_url_exc)
++ retries=retries, exception_cb=exc_cb, infinite=infinite)
+ except Exception as e:
+ # pylint:disable=no-member
+ if isinstance(e, UrlError) and e.code == 400:
+diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
+index d9817d84..c4a8e08d 100644
+--- a/tests/unittests/test_datasource/test_azure.py
++++ b/tests/unittests/test_datasource/test_azure.py
+@@ -448,7 +448,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
+ "http://169.254.169.254/metadata/instance?api-version="
+ "2019-06-01", exception_cb=mock.ANY,
+ headers=mock.ANY, retries=mock.ANY,
+- timeout=mock.ANY)
++ timeout=mock.ANY, infinite=False)
+
+ @mock.patch(MOCKPATH + 'readurl', autospec=True)
+ @mock.patch(MOCKPATH + 'EphemeralDHCPv4')
+@@ -467,7 +467,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
+ "http://169.254.169.254/metadata/instance/network?api-version="
+ "2019-06-01", exception_cb=mock.ANY,
+ headers=mock.ANY, retries=mock.ANY,
+- timeout=mock.ANY)
++ timeout=mock.ANY, infinite=False)
+
+ @mock.patch(MOCKPATH + 'readurl', autospec=True)
+ @mock.patch(MOCKPATH + 'EphemeralDHCPv4')
+@@ -486,7 +486,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
+ "http://169.254.169.254/metadata/instance?api-version="
+ "2019-06-01", exception_cb=mock.ANY,
+ headers=mock.ANY, retries=mock.ANY,
+- timeout=mock.ANY)
++ timeout=mock.ANY, infinite=False)
+
+ @mock.patch(MOCKPATH + 'readurl', autospec=True)
+ @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting', autospec=True)
+@@ -511,7 +511,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
+ m_readurl.assert_called_with(
+ self.network_md_url, exception_cb=mock.ANY,
+ headers={'Metadata': 'true'}, retries=2,
+- timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS)
++ timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, infinite=False)
+
+ @mock.patch('cloudinit.url_helper.time.sleep')
+ @mock.patch(MOCKPATH + 'net.is_up', autospec=True)
+@@ -2694,15 +2694,22 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
+
+ def nic_attach_ret(nl_sock, nics_found):
+ nonlocal m_attach_call_count
+- if m_attach_call_count == 0:
+- m_attach_call_count = m_attach_call_count + 1
++ m_attach_call_count = m_attach_call_count + 1
++ if m_attach_call_count == 1:
+ return "eth0"
+- return "eth1"
++ elif m_attach_call_count == 2:
++ return "eth1"
++ raise RuntimeError("Must have found primary nic by now.")
++
++ # Simulate two NICs by adding the same one twice.
++ md = {
++ "interface": [
++ IMDS_NETWORK_METADATA['interface'][0],
++ IMDS_NETWORK_METADATA['interface'][0]
++ ]
++ }
+
+- def network_metadata_ret(ifname, retries, type):
+- # Simulate two NICs by adding the same one twice.
+- md = IMDS_NETWORK_METADATA
+- md['interface'].append(md['interface'][0])
++ def network_metadata_ret(ifname, retries, type, exc_cb, infinite):
+ if ifname == "eth0":
+ return md
+ raise requests.Timeout('Fake connection timeout')
+@@ -2724,6 +2731,72 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
+ self.assertEqual(1, m_imds.call_count)
+ self.assertEqual(2, m_link_up.call_count)
+
++ @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback')
++ @mock.patch(MOCKPATH + 'EphemeralDHCPv4')
++ def test_check_if_nic_is_primary_retries_on_failures(
++ self, m_dhcpv4, m_imds):
++ """Retry polling for network metadata on all failures except timeout"""
++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
++ lease = {
++ 'interface': 'eth9', 'fixed-address': '192.168.2.9',
++ 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
++ 'unknown-245': '624c3620'}
++
++ eth0Retries = []
++ eth1Retries = []
++ # Simulate two NICs by adding the same one twice.
++ md = {
++ "interface": [
++ IMDS_NETWORK_METADATA['interface'][0],
++ IMDS_NETWORK_METADATA['interface'][0]
++ ]
++ }
++
++ def network_metadata_ret(ifname, retries, type, exc_cb, infinite):
++ nonlocal eth0Retries, eth1Retries
++
++ # Simulate readurl functionality with retries and
++ # exception callbacks so that the callback logic can be
++ # validated.
++ if ifname == "eth0":
++ cause = requests.HTTPError()
++ for _ in range(0, 15):
++ error = url_helper.UrlError(cause=cause, code=410)
++ eth0Retries.append(exc_cb("No goal state.", error))
++ else:
++ cause = requests.Timeout('Fake connection timeout')
++ for _ in range(0, 10):
++ error = url_helper.UrlError(cause=cause)
++ eth1Retries.append(exc_cb("Connection timeout", error))
++ # Should stop retrying after 10 retries
++ eth1Retries.append(exc_cb("Connection timeout", error))
++ raise cause
++ return md
++
++ m_imds.side_effect = network_metadata_ret
++
++ dhcp_ctx = mock.MagicMock(lease=lease)
++ dhcp_ctx.obtain_lease.return_value = lease
++ m_dhcpv4.return_value = dhcp_ctx
++
++ is_primary, expected_nic_count = dsa._check_if_nic_is_primary("eth0")
++ self.assertEqual(True, is_primary)
++ self.assertEqual(2, expected_nic_count)
++
++ # All Eth0 errors are non-timeout errors. So we should have been
++ # retrying indefinitely until success.
++ for i in eth0Retries:
++ self.assertTrue(i)
++
++ is_primary, expected_nic_count = dsa._check_if_nic_is_primary("eth1")
++ self.assertEqual(False, is_primary)
++
++ # All Eth1 errors are timeout errors. Retry happens for a max of 10 and
++ # then we should have moved on assuming it is not the primary nic.
++ for i in range(0, 10):
++ self.assertTrue(eth1Retries[i])
++ self.assertFalse(eth1Retries[10])
++
+ @mock.patch('cloudinit.distros.networking.LinuxNetworking.try_set_link_up')
+ def test_wait_for_link_up_returns_if_already_up(
+ self, m_is_link_up):
+--
+2.27.0
+
diff --git a/SOURCES/ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch b/SOURCES/ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch
new file mode 100644
index 0000000..0f17062
--- /dev/null
+++ b/SOURCES/ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch
@@ -0,0 +1,129 @@
+From 0def71378dc7abf682727c600b696f7313cdcf60 Mon Sep 17 00:00:00 2001
+From: Anh Vo <anhvo@microsoft.com>
+Date: Tue, 27 Apr 2021 13:40:59 -0400
+Subject: [PATCH 7/7] Azure: adding support for consuming userdata from IMDS
+ (#884)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [7/7] 1e7ab925162ed9ef2c9b5b9f5c6d5e6ec6e623dd (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+---
+ cloudinit/sources/DataSourceAzure.py | 23 ++++++++-
+ tests/unittests/test_datasource/test_azure.py | 50 +++++++++++++++++++
+ 2 files changed, 72 insertions(+), 1 deletion(-)
+
+diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
+index d0be6d84..a66f023d 100755
+--- a/cloudinit/sources/DataSourceAzure.py
++++ b/cloudinit/sources/DataSourceAzure.py
+@@ -83,7 +83,7 @@ AGENT_SEED_DIR = '/var/lib/waagent'
+ IMDS_TIMEOUT_IN_SECONDS = 2
+ IMDS_URL = "http://169.254.169.254/metadata"
+ IMDS_VER_MIN = "2019-06-01"
+-IMDS_VER_WANT = "2020-10-01"
++IMDS_VER_WANT = "2021-01-01"
+
+
+ # This holds SSH key data including if the source was
+@@ -539,6 +539,20 @@ class DataSourceAzure(sources.DataSource):
+ imds_disable_password
+ )
+ crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501
++
++ # only use userdata from imds if OVF did not provide custom data
++ # userdata provided by IMDS is always base64 encoded
++ if not userdata_raw:
++ imds_userdata = _userdata_from_imds(imds_md)
++ if imds_userdata:
++ LOG.debug("Retrieved userdata from IMDS")
++ try:
++ crawled_data['userdata_raw'] = base64.b64decode(
++ ''.join(imds_userdata.split()))
++ except Exception:
++ report_diagnostic_event(
++ "Bad userdata in IMDS",
++ logger_func=LOG.warning)
+ found = cdev
+
+ report_diagnostic_event(
+@@ -1512,6 +1526,13 @@ def _username_from_imds(imds_data):
+ return None
+
+
++def _userdata_from_imds(imds_data):
++ try:
++ return imds_data['compute']['userData']
++ except KeyError:
++ return None
++
++
+ def _hostname_from_imds(imds_data):
+ try:
+ return imds_data['compute']['osProfile']['computerName']
+diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
+index c4a8e08d..f8433690 100644
+--- a/tests/unittests/test_datasource/test_azure.py
++++ b/tests/unittests/test_datasource/test_azure.py
+@@ -1899,6 +1899,56 @@ scbus-1 on xpt0 bus 0
+ dsrc.get_data()
+ self.assertTrue(dsrc.metadata["disable_password"])
+
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_userdata_from_imds(self, m_get_metadata_from_imds):
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ odata = {'HostName': "myhost", 'UserName': "myuser"}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++ userdata = "userdataImds"
++ imds_data = copy.deepcopy(NETWORK_METADATA)
++ imds_data["compute"]["osProfile"] = dict(
++ adminUsername="username1",
++ computerName="hostname1",
++ disablePasswordAuthentication="true",
++ )
++ imds_data["compute"]["userData"] = b64e(userdata)
++ m_get_metadata_from_imds.return_value = imds_data
++ dsrc = self._get_ds(data)
++ ret = dsrc.get_data()
++ self.assertTrue(ret)
++ self.assertEqual(dsrc.userdata_raw, userdata.encode('utf-8'))
++
++ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
++ def test_userdata_from_imds_with_customdata_from_OVF(
++ self, m_get_metadata_from_imds):
++ userdataOVF = "userdataOVF"
++ odata = {
++ 'HostName': "myhost", 'UserName': "myuser",
++ 'UserData': {'text': b64e(userdataOVF), 'encoding': 'base64'}
++ }
++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
++ data = {
++ 'ovfcontent': construct_valid_ovf_env(data=odata),
++ 'sys_cfg': sys_cfg
++ }
++
++ userdataImds = "userdataImds"
++ imds_data = copy.deepcopy(NETWORK_METADATA)
++ imds_data["compute"]["osProfile"] = dict(
++ adminUsername="username1",
++ computerName="hostname1",
++ disablePasswordAuthentication="true",
++ )
++ imds_data["compute"]["userData"] = b64e(userdataImds)
++ m_get_metadata_from_imds.return_value = imds_data
++ dsrc = self._get_ds(data)
++ ret = dsrc.get_data()
++ self.assertTrue(ret)
++ self.assertEqual(dsrc.userdata_raw, userdataOVF.encode('utf-8'))
++
+
+ class TestAzureBounce(CiTestCase):
+
+--
+2.27.0
+
diff --git a/SOURCES/ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch b/SOURCES/ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch
new file mode 100644
index 0000000..947a035
--- /dev/null
+++ b/SOURCES/ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch
@@ -0,0 +1,177 @@
+From 2ece71923a37a5e1107c80f091a1cc620943fbf2 Mon Sep 17 00:00:00 2001
+From: Anh Vo <anhvo@microsoft.com>
+Date: Fri, 23 Apr 2021 10:18:05 -0400
+Subject: [PATCH 4/7] Azure: eject the provisioning iso before reporting ready
+ (#861)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [4/7] 63e379a4406530c0c15c733f8eee35421079508b (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+
+Due to hyper-v implementations, iso ejection is more efficient if performed
+from within the guest. The code will attempt to perform a best-effort ejection.
+Failure during ejection will not prevent reporting ready from happening. If iso
+ejection is successful, later iso ejection from the platform will be a no-op.
+In the event the iso ejection from the guest fails, iso ejection will still happen at
+the platform level.
+---
+ cloudinit/sources/DataSourceAzure.py | 22 +++++++++++++++---
+ cloudinit/sources/helpers/azure.py | 23 ++++++++++++++++---
+ .../test_datasource/test_azure_helper.py | 13 +++++++++--
+ 3 files changed, 50 insertions(+), 8 deletions(-)
+
+diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
+index 020b7006..39e67c4f 100755
+--- a/cloudinit/sources/DataSourceAzure.py
++++ b/cloudinit/sources/DataSourceAzure.py
+@@ -332,6 +332,7 @@ class DataSourceAzure(sources.DataSource):
+ dsname = 'Azure'
+ _negotiated = False
+ _metadata_imds = sources.UNSET
++ _ci_pkl_version = 1
+
+ def __init__(self, sys_cfg, distro, paths):
+ sources.DataSource.__init__(self, sys_cfg, distro, paths)
+@@ -346,8 +347,13 @@ class DataSourceAzure(sources.DataSource):
+ # Regenerate network config new_instance boot and every boot
+ self.update_events['network'].add(EventType.BOOT)
+ self._ephemeral_dhcp_ctx = None
+-
+ self.failed_desired_api_version = False
++ self.iso_dev = None
++
++ def _unpickle(self, ci_pkl_version: int) -> None:
++ super()._unpickle(ci_pkl_version)
++ if "iso_dev" not in self.__dict__:
++ self.iso_dev = None
+
+ def __str__(self):
+ root = sources.DataSource.__str__(self)
+@@ -459,6 +465,13 @@ class DataSourceAzure(sources.DataSource):
+ '%s was not mountable' % cdev, logger_func=LOG.warning)
+ continue
+
++ report_diagnostic_event("Found provisioning metadata in %s" % cdev,
++ logger_func=LOG.debug)
++
++ # save the iso device for ejection before reporting ready
++ if cdev.startswith("/dev"):
++ self.iso_dev = cdev
++
+ perform_reprovision = reprovision or self._should_reprovision(ret)
+ perform_reprovision_after_nic_attach = (
+ reprovision_after_nic_attach or
+@@ -1226,7 +1239,9 @@ class DataSourceAzure(sources.DataSource):
+ @return: The success status of sending the ready signal.
+ """
+ try:
+- get_metadata_from_fabric(None, lease['unknown-245'])
++ get_metadata_from_fabric(fallback_lease_file=None,
++ dhcp_opts=lease['unknown-245'],
++ iso_dev=self.iso_dev)
+ return True
+ except Exception as e:
+ report_diagnostic_event(
+@@ -1332,7 +1347,8 @@ class DataSourceAzure(sources.DataSource):
+ metadata_func = partial(get_metadata_from_fabric,
+ fallback_lease_file=self.
+ dhclient_lease_file,
+- pubkey_info=pubkey_info)
++ pubkey_info=pubkey_info,
++ iso_dev=self.iso_dev)
+
+ LOG.debug("negotiating with fabric via agent command %s",
+ self.ds_cfg['agent_command'])
+diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
+index 03e7156b..ad476076 100755
+--- a/cloudinit/sources/helpers/azure.py
++++ b/cloudinit/sources/helpers/azure.py
+@@ -865,7 +865,19 @@ class WALinuxAgentShim:
+ return endpoint_ip_address
+
+ @azure_ds_telemetry_reporter
+- def register_with_azure_and_fetch_data(self, pubkey_info=None) -> dict:
++ def eject_iso(self, iso_dev) -> None:
++ try:
++ LOG.debug("Ejecting the provisioning iso")
++ subp.subp(['eject', iso_dev])
++ except Exception as e:
++ report_diagnostic_event(
++ "Failed ejecting the provisioning iso: %s" % e,
++ logger_func=LOG.debug)
++
++ @azure_ds_telemetry_reporter
++ def register_with_azure_and_fetch_data(self,
++ pubkey_info=None,
++ iso_dev=None) -> dict:
+ """Gets the VM's GoalState from Azure, uses the GoalState information
+ to report ready/send the ready signal/provisioning complete signal to
+ Azure, and then uses pubkey_info to filter and obtain the user's
+@@ -891,6 +903,10 @@ class WALinuxAgentShim:
+ ssh_keys = self._get_user_pubkeys(goal_state, pubkey_info)
+ health_reporter = GoalStateHealthReporter(
+ goal_state, self.azure_endpoint_client, self.endpoint)
++
++ if iso_dev is not None:
++ self.eject_iso(iso_dev)
++
+ health_reporter.send_ready_signal()
+ return {'public-keys': ssh_keys}
+
+@@ -1046,11 +1062,12 @@ class WALinuxAgentShim:
+
+ @azure_ds_telemetry_reporter
+ def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None,
+- pubkey_info=None):
++ pubkey_info=None, iso_dev=None):
+ shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file,
+ dhcp_options=dhcp_opts)
+ try:
+- return shim.register_with_azure_and_fetch_data(pubkey_info=pubkey_info)
++ return shim.register_with_azure_and_fetch_data(
++ pubkey_info=pubkey_info, iso_dev=iso_dev)
+ finally:
+ shim.clean_up()
+
+diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
+index 63482c6c..552c7905 100644
+--- a/tests/unittests/test_datasource/test_azure_helper.py
++++ b/tests/unittests/test_datasource/test_azure_helper.py
+@@ -1009,6 +1009,14 @@ class TestWALinuxAgentShim(CiTestCase):
+ self.GoalState.return_value.container_id = self.test_container_id
+ self.GoalState.return_value.instance_id = self.test_instance_id
+
++ def test_eject_iso_is_called(self):
++ shim = wa_shim()
++ with mock.patch.object(
++ shim, 'eject_iso', autospec=True
++ ) as m_eject_iso:
++ shim.register_with_azure_and_fetch_data(iso_dev="/dev/sr0")
++ m_eject_iso.assert_called_once_with("/dev/sr0")
++
+ def test_http_client_does_not_use_certificate_for_report_ready(self):
+ shim = wa_shim()
+ shim.register_with_azure_and_fetch_data()
+@@ -1283,13 +1291,14 @@ class TestGetMetadataGoalStateXMLAndReportReadyToFabric(CiTestCase):
+
+ def test_calls_shim_register_with_azure_and_fetch_data(self):
+ m_pubkey_info = mock.MagicMock()
+- azure_helper.get_metadata_from_fabric(pubkey_info=m_pubkey_info)
++ azure_helper.get_metadata_from_fabric(
++ pubkey_info=m_pubkey_info, iso_dev="/dev/sr0")
+ self.assertEqual(
+ 1,
+ self.m_shim.return_value
+ .register_with_azure_and_fetch_data.call_count)
+ self.assertEqual(
+- mock.call(pubkey_info=m_pubkey_info),
++ mock.call(iso_dev="/dev/sr0", pubkey_info=m_pubkey_info),
+ self.m_shim.return_value
+ .register_with_azure_and_fetch_data.call_args)
+
+--
+2.27.0
+
diff --git a/SOURCES/ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch b/SOURCES/ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch
new file mode 100644
index 0000000..33a8acc
--- /dev/null
+++ b/SOURCES/ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch
@@ -0,0 +1,90 @@
+From 3ee42e6e6ca51b3fd0b6461f707d62c89d54e227 Mon Sep 17 00:00:00 2001
+From: Johnson Shi <Johnson.Shi@microsoft.com>
+Date: Thu, 25 Mar 2021 07:20:10 -0700
+Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between
+ retries (#842)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [2/7] 65672cdfe2265f32e6d3c440ba5a8accafdb6ca6 (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+
+Ensure that the Azure helper's http handler sleeps a fixed duration
+between retry failure attempts. The http handler will sleep a fixed
+duration between failed attempts regardless of whether the attempt
+failed due to (1) request timing out or (2) instant failure (no
+timeout).
+
+Due to certain platform issues, the http request to the Azure endpoint
+may instantly fail without reaching the http timeout duration. Without
+sleeping a fixed duration in between retry attempts, the http handler
+will loop through the max retry attempts quickly. This causes the
+communication between cloud-init and the Azure platform to be less
+resilient due to the short total duration if there is no sleep in
+between retries.
+---
+ cloudinit/sources/helpers/azure.py | 2 ++
+ tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++--
+ 2 files changed, 11 insertions(+), 2 deletions(-)
+
+diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
+index d3055d08..03e7156b 100755
+--- a/cloudinit/sources/helpers/azure.py
++++ b/cloudinit/sources/helpers/azure.py
+@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str:
+
+ max_readurl_attempts = 240
+ default_readurl_timeout = 5
++ sleep_duration_between_retries = 5
+ periodic_logging_attempts = 12
+
+ if 'timeout' not in kwargs:
+@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str:
+ 'attempt %d with exception: %s' %
+ (url, attempt, e),
+ logger_func=LOG.debug)
++ time.sleep(sleep_duration_between_retries)
+
+ raise exc
+
+diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
+index b8899807..63482c6c 100644
+--- a/tests/unittests/test_datasource/test_azure_helper.py
++++ b/tests/unittests/test_datasource/test_azure_helper.py
+@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
+
+ max_readurl_attempts = 240
+ default_readurl_timeout = 5
++ sleep_duration_between_retries = 5
+ periodic_logging_attempts = 12
+
+ def setUp(self):
+@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
+ self.m_readurl = patches.enter_context(
+ mock.patch.object(
+ azure_helper.url_helper, 'readurl', mock.MagicMock()))
+- patches.enter_context(
+- mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
++ self.m_sleep = patches.enter_context(
++ mock.patch.object(azure_helper.time, 'sleep', autospec=True))
+
+ def test_http_with_retries(self):
+ self.m_readurl.return_value = 'TestResp'
+@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
+ self.m_readurl.call_count,
+ self.periodic_logging_attempts + 1)
+
++ # Ensure that cloud-init did sleep between each failed request
++ self.assertEqual(
++ self.m_sleep.call_count,
++ self.periodic_logging_attempts)
++ self.m_sleep.assert_called_with(self.sleep_duration_between_retries)
++
+ def test_http_with_retries_long_delay_logs_periodic_failure_msg(self):
+ self.m_readurl.side_effect = \
+ [SentinelException] * self.periodic_logging_attempts + \
+--
+2.27.0
+
diff --git a/SOURCES/ci-Change-netifaces-dependency-to-0.10.4-965.patch b/SOURCES/ci-Change-netifaces-dependency-to-0.10.4-965.patch
new file mode 100644
index 0000000..666ef73
--- /dev/null
+++ b/SOURCES/ci-Change-netifaces-dependency-to-0.10.4-965.patch
@@ -0,0 +1,47 @@
+From 18138313e009a08592fe79c5e66d6eba8f027f19 Mon Sep 17 00:00:00 2001
+From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+Date: Fri, 14 Jan 2022 16:49:57 +0100
+Subject: [PATCH 2/5] Change netifaces dependency to 0.10.4 (#965)
+
+RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+RH-MergeRequest: 17: Datasource for VMware
+RH-Commit: [2/5] 8688e8b955a7ee15cf66de0b2a242c7c418b7630 (eesposit/cloud-init-centos-)
+RH-Bugzilla: 2040090
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
+
+commit b9d308b4d61d22bacc05bcae59819755975631f8
+Author: Andrew Kutz <101085+akutz@users.noreply.github.com>
+Date: Tue Aug 10 15:10:44 2021 -0500
+
+ Change netifaces dependency to 0.10.4 (#965)
+
+ Change netifaces dependency to 0.10.4
+
+ Currently versions Ubuntu <=20.10 use netifaces 0.10.4 By requiring
+ netifaces 0.10.9, the VMware datasource omitted itself from cloud-init
+ on Ubuntu <=20.10.
+
+ This patch changes the netifaces dependency to 0.10.4. While it is true
+ there are patches to netifaces post 0.10.4 that are desirable, testing
+ against the most common network configuration was performed to verify
+ the VMware datasource will still function with netifaces 0.10.4.
+
+Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+---
+ requirements.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/requirements.txt b/requirements.txt
+index 41d01d62..c4adc455 100644
+--- a/requirements.txt
++++ b/requirements.txt
+@@ -40,4 +40,4 @@ jsonschema
+ # and still participate in instance-data by gathering the network in detail at
+ # runtime and merge that information into the metadata and repersist that to
+ # disk.
+-netifaces>=0.10.9
++netifaces>=0.10.4
+--
+2.27.0
+
diff --git a/SOURCES/ci-Datasource-for-VMware-953.patch b/SOURCES/ci-Datasource-for-VMware-953.patch
new file mode 100644
index 0000000..74af6f6
--- /dev/null
+++ b/SOURCES/ci-Datasource-for-VMware-953.patch
@@ -0,0 +1,2201 @@
+From 078f3a218394eef3b28a2a061d836efe42b6c9ed Mon Sep 17 00:00:00 2001
+From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+Date: Fri, 14 Jan 2022 16:49:28 +0100
+Subject: [PATCH 1/5] Datasource for VMware (#953)
+
+RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+RH-MergeRequest: 17: Datasource for VMware
+RH-Commit: [1/5] 7b47334ec524dcf1b8edd02b65df7d0ff5a366e0 (eesposit/cloud-init-centos-)
+RH-Bugzilla: 2040090
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
+
+commit 8b4a9bc7b81e61943af873bad92e2133f8275b0b
+Author: Andrew Kutz <101085+akutz@users.noreply.github.com>
+Date: Mon Aug 9 21:24:07 2021 -0500
+
+ Datasource for VMware (#953)
+
+ This patch finally introduces the Cloud-Init Datasource for VMware
+ GuestInfo as a part of cloud-init proper. This datasource has existed
+ since 2018, and rapidly became the de facto datasource for developers
+ working with Packer, Terraform, for projects like kube-image-builder,
+ and the de jure datasource for Photon OS.
+
+ The major change to the datasource from its previous incarnation is
+ the name. Now named DatasourceVMware, this new version of the
+ datasource will allow multiple transport types in addition to
+ GuestInfo keys.
+
+ This datasource includes several unique features developed to address
+ real-world situations:
+
+ * Support for reading any key (metadata, userdata, vendordata) both
+ from the guestinfo table when running on a VM in vSphere as well as
+ from an environment variable when running inside of a container,
+ useful for rapid dev/test.
+
+ * Allows booting with DHCP while still providing full participation
+ in Cloud-Init instance data and Jinja queries. The netifaces library
+ provides the ability to inspect the network after it is online,
+ and the runtime network configuration is then merged into the
+ existing metadata and persisted to disk.
+
+ * Advertises the local_ipv4 and local_ipv6 addresses via guestinfo
+ as well. This is useful as Guest Tools is not always able to
+ identify what would be considered the local address.
+
+ The primary author and current steward of this datasource spoke at
+ Cloud-Init Con 2020 where there was interest in contributing this datasource
+ to the Cloud-Init codebase.
+
+ The datasource currently lives in its own GitHub repository at
+ https://github.com/vmware/cloud-init-vmware-guestinfo. Once the datasource
+ is merged into Cloud-Init, the old repository will be deprecated.
+
+Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+---
+ README.md | 2 +-
+ cloudinit/settings.py | 1 +
+ cloudinit/sources/DataSourceVMware.py | 871 ++++++++++++++++++
+ doc/rtd/topics/availability.rst | 1 +
+ doc/rtd/topics/datasources.rst | 2 +-
+ doc/rtd/topics/datasources/vmware.rst | 359 ++++++++
+ requirements.txt | 12 +
+ .../unittests/test_datasource/test_common.py | 3 +
+ .../unittests/test_datasource/test_vmware.py | 377 ++++++++
+ tests/unittests/test_ds_identify.py | 279 +++++-
+ tools/.github-cla-signers | 1 +
+ tools/ds-identify | 76 +-
+ 12 files changed, 1980 insertions(+), 4 deletions(-)
+ create mode 100644 cloudinit/sources/DataSourceVMware.py
+ create mode 100644 doc/rtd/topics/datasources/vmware.rst
+ create mode 100644 tests/unittests/test_datasource/test_vmware.py
+
+diff --git a/README.md b/README.md
+index 435405da..aa4fad63 100644
+--- a/README.md
++++ b/README.md
+@@ -39,7 +39,7 @@ get in contact with that distribution and send them our way!
+
+ | Supported OSes | Supported Public Clouds | Supported Private Clouds |
+ | --- | --- | --- |
+-| Alpine Linux<br />ArchLinux<br />Debian<br />Fedora<br />FreeBSD<br />Gentoo Linux<br />NetBSD<br />OpenBSD<br />RHEL/CentOS<br />SLES/openSUSE<br />Ubuntu<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /> | Amazon Web Services<br />Microsoft Azure<br />Google Cloud Platform<br />Oracle Cloud Infrastructure<br />Softlayer<br />Rackspace Public Cloud<br />IBM Cloud<br />Digital Ocean<br />Bigstep<br />Hetzner<br />Joyent<br />CloudSigma<br />Alibaba Cloud<br />OVH<br />OpenNebula<br />Exoscale<br />Scaleway<br />CloudStack<br />AltCloud<br />SmartOS<br />HyperOne<br />Rootbox<br /> | Bare metal installs<br />OpenStack<br />LXD<br />KVM<br />Metal-as-a-Service (MAAS)<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br />|
++| Alpine Linux<br />ArchLinux<br />Debian<br />Fedora<br />FreeBSD<br />Gentoo Linux<br />NetBSD<br />OpenBSD<br />RHEL/CentOS<br />SLES/openSUSE<br />Ubuntu<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /> | Amazon Web Services<br />Microsoft Azure<br />Google Cloud Platform<br />Oracle Cloud Infrastructure<br />Softlayer<br />Rackspace Public Cloud<br />IBM Cloud<br />Digital Ocean<br />Bigstep<br />Hetzner<br />Joyent<br />CloudSigma<br />Alibaba Cloud<br />OVH<br />OpenNebula<br />Exoscale<br />Scaleway<br />CloudStack<br />AltCloud<br />SmartOS<br />HyperOne<br />Rootbox<br /> | Bare metal installs<br />OpenStack<br />LXD<br />KVM<br />Metal-as-a-Service (MAAS)<br />VMware<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br />|
+
+ ## To start developing cloud-init
+
+diff --git a/cloudinit/settings.py b/cloudinit/settings.py
+index 2acf2615..d5f32dbb 100644
+--- a/cloudinit/settings.py
++++ b/cloudinit/settings.py
+@@ -42,6 +42,7 @@ CFG_BUILTIN = {
+ 'Exoscale',
+ 'RbxCloud',
+ 'UpCloud',
++ 'VMware',
+ # At the end to act as a 'catch' when none of the above work...
+ 'None',
+ ],
+diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py
+new file mode 100644
+index 00000000..22ca63de
+--- /dev/null
++++ b/cloudinit/sources/DataSourceVMware.py
+@@ -0,0 +1,871 @@
++# Cloud-Init DataSource for VMware
++#
++# Copyright (c) 2018-2021 VMware, Inc. All Rights Reserved.
++#
++# Authors: Anish Swaminathan <anishs@vmware.com>
++# Andrew Kutz <akutz@vmware.com>
++#
++# This file is part of cloud-init. See LICENSE file for license information.
++
++"""Cloud-Init DataSource for VMware
++
++This module provides a cloud-init datasource for VMware systems and supports
++multiple transports types, including:
++
++ * EnvVars
++ * GuestInfo
++
++Netifaces (https://github.com/al45tair/netifaces)
++
++ Please note this module relies on the netifaces project to introspect the
++ runtime, network configuration of the host on which this datasource is
++ running. This is in contrast to the rest of cloud-init which uses the
++ cloudinit/netinfo module.
++
++ The reasons for using netifaces include:
++
++ * Netifaces is built in C and is more portable across multiple systems
++ and more deterministic than shell exec'ing local network commands and
++ parsing their output.
++
++ * Netifaces provides a stable way to determine the view of the host's
++ network after DHCP has brought the network online. Unlike most other
++ datasources, this datasource still provides support for JINJA queries
++ based on networking information even when the network is based on a
++ DHCP lease. While this does not tie this datasource directly to
++ netifaces, it does mean the ability to consistently obtain the
++ correct information is paramount.
++
++ * It is currently possible to execute this datasource on macOS
++ (which many developers use today) to print the output of the
++ get_host_info function. This function calls netifaces to obtain
++ the same runtime network configuration that the datasource would
++ persist to the local system's instance data.
++
++ However, the netinfo module fails on macOS. The result is either a
++ hung operation that requires a SIGINT to return control to the user,
++ or, if brew is used to install iproute2mac, the ip commands are used
++ but produce output the netinfo module is unable to parse.
++
++ While macOS is not a target of cloud-init, this feature is quite
++ useful when working on this datasource.
++
++ For more information about this behavior, please see the following
++ PR comment, https://bit.ly/3fG7OVh.
++
++ The authors of this datasource are not opposed to moving away from
++ netifaces. The goal may be to eventually do just that. This proviso was
++ added to the top of this module as a way to remind future-us and others
++ why netifaces was used in the first place in order to either smooth the
++ transition away from netifaces or embrace it further up the cloud-init
++ stack.
++"""
++
++import collections
++import copy
++from distutils.spawn import find_executable
++import ipaddress
++import json
++import os
++import socket
++import time
++
++from cloudinit import dmi, log as logging
++from cloudinit import sources
++from cloudinit import util
++from cloudinit.subp import subp, ProcessExecutionError
++
++import netifaces
++
++
++PRODUCT_UUID_FILE_PATH = "/sys/class/dmi/id/product_uuid"
++
++LOG = logging.getLogger(__name__)
++NOVAL = "No value found"
++
++DATA_ACCESS_METHOD_ENVVAR = "envvar"
++DATA_ACCESS_METHOD_GUESTINFO = "guestinfo"
++
++VMWARE_RPCTOOL = find_executable("vmware-rpctool")
++REDACT = "redact"
++CLEANUP_GUESTINFO = "cleanup-guestinfo"
++VMX_GUESTINFO = "VMX_GUESTINFO"
++GUESTINFO_EMPTY_YAML_VAL = "---"
++
++LOCAL_IPV4 = "local-ipv4"
++LOCAL_IPV6 = "local-ipv6"
++WAIT_ON_NETWORK = "wait-on-network"
++WAIT_ON_NETWORK_IPV4 = "ipv4"
++WAIT_ON_NETWORK_IPV6 = "ipv6"
++
++
++class DataSourceVMware(sources.DataSource):
++ """
++ Setting the hostname:
++ The hostname is set by way of the metadata key "local-hostname".
++
++ Setting the instance ID:
++ The instance ID may be set by way of the metadata key "instance-id".
++ However, if this value is absent then the instance ID is read
++ from the file /sys/class/dmi/id/product_uuid.
++
++ Configuring the network:
++ The network is configured by setting the metadata key "network"
++ with a value consistent with Network Config Versions 1 or 2,
++ depending on the Linux distro's version of cloud-init:
++
++ Network Config Version 1 - http://bit.ly/cloudinit-net-conf-v1
++ Network Config Version 2 - http://bit.ly/cloudinit-net-conf-v2
++
++ For example, CentOS 7's official cloud-init package is version
++ 0.7.9 and does not support Network Config Version 2. However,
++ this datasource still supports supplying Network Config Version 2
++ data as long as the Linux distro's cloud-init package is new
++ enough to parse the data.
++
++ The metadata key "network.encoding" may be used to indicate the
++ format of the metadata key "network". Valid encodings are base64
++ and gzip+base64.
++ """
++
++ dsname = "VMware"
++
++ def __init__(self, sys_cfg, distro, paths, ud_proc=None):
++ sources.DataSource.__init__(self, sys_cfg, distro, paths, ud_proc)
++
++ self.data_access_method = None
++ self.vmware_rpctool = VMWARE_RPCTOOL
++
++ def _get_data(self):
++ """
++ _get_data loads the metadata, userdata, and vendordata from one of
++ the following locations in the given order:
++
++ * envvars
++ * guestinfo
++
++ Please note when updating this function with support for new data
++ transports, the order should match the order in the dscheck_VMware
++ function from the file ds-identify.
++ """
++
++ # Initialize the locally scoped metadata, userdata, and vendordata
++ # variables. They are assigned below depending on the detected data
++ # access method.
++ md, ud, vd = None, None, None
++
++ # First check to see if there is data via env vars.
++ if os.environ.get(VMX_GUESTINFO, ""):
++ md = guestinfo_envvar("metadata")
++ ud = guestinfo_envvar("userdata")
++ vd = guestinfo_envvar("vendordata")
++
++ if md or ud or vd:
++ self.data_access_method = DATA_ACCESS_METHOD_ENVVAR
++
++ # At this point, all additional data transports are valid only on
++ # a VMware platform.
++ if not self.data_access_method:
++ system_type = dmi.read_dmi_data("system-product-name")
++ if system_type is None:
++ LOG.debug("No system-product-name found")
++ return False
++ if "vmware" not in system_type.lower():
++ LOG.debug("Not a VMware platform")
++ return False
++
++ # If no data was detected, check the guestinfo transport next.
++ if not self.data_access_method:
++ if self.vmware_rpctool:
++ md = guestinfo("metadata", self.vmware_rpctool)
++ ud = guestinfo("userdata", self.vmware_rpctool)
++ vd = guestinfo("vendordata", self.vmware_rpctool)
++
++ if md or ud or vd:
++ self.data_access_method = DATA_ACCESS_METHOD_GUESTINFO
++
++ if not self.data_access_method:
++ LOG.error("failed to find a valid data access method")
++ return False
++
++ LOG.info("using data access method %s", self._get_subplatform())
++
++ # Get the metadata.
++ self.metadata = process_metadata(load_json_or_yaml(md))
++
++ # Get the user data.
++ self.userdata_raw = ud
++
++ # Get the vendor data.
++ self.vendordata_raw = vd
++
++ # Redact any sensitive information.
++ self.redact_keys()
++
++ # get_data returns true if there is any available metadata,
++ # userdata, or vendordata.
++ if self.metadata or self.userdata_raw or self.vendordata_raw:
++ return True
++ else:
++ return False
++
++ def setup(self, is_new_instance):
++ """setup(is_new_instance)
++
++ This is called before user-data and vendor-data have been processed.
++
++ Unless the datasource has set mode to 'local', then networking
++ per 'fallback' or per 'network_config' will have been written and
++ brought up the OS at this point.
++ """
++
++ host_info = wait_on_network(self.metadata)
++ LOG.info("got host-info: %s", host_info)
++
++ # Reflect any possible local IPv4 or IPv6 addresses in the guest
++ # info.
++ advertise_local_ip_addrs(host_info)
++
++ # Ensure the metadata gets updated with information about the
++ # host, including the network interfaces, default IP addresses,
++ # etc.
++ self.metadata = util.mergemanydict([self.metadata, host_info])
++
++ # Persist the instance data for versions of cloud-init that support
++ # doing so. This occurs here rather than in the get_data call in
++ # order to ensure that the network interfaces are up and can be
++ # persisted with the metadata.
++ self.persist_instance_data()
++
++ def _get_subplatform(self):
++ get_key_name_fn = None
++ if self.data_access_method == DATA_ACCESS_METHOD_ENVVAR:
++ get_key_name_fn = get_guestinfo_envvar_key_name
++ elif self.data_access_method == DATA_ACCESS_METHOD_GUESTINFO:
++ get_key_name_fn = get_guestinfo_key_name
++ else:
++ return sources.METADATA_UNKNOWN
++
++ return "%s (%s)" % (
++ self.data_access_method,
++ get_key_name_fn("metadata"),
++ )
++
++ @property
++ def network_config(self):
++ if "network" in self.metadata:
++ LOG.debug("using metadata network config")
++ else:
++ LOG.debug("using fallback network config")
++ self.metadata["network"] = {
++ "config": self.distro.generate_fallback_config(),
++ }
++ return self.metadata["network"]["config"]
++
++ def get_instance_id(self):
++ # Pull the instance ID out of the metadata if present. Otherwise
++ # read the file /sys/class/dmi/id/product_uuid for the instance ID.
++ if self.metadata and "instance-id" in self.metadata:
++ return self.metadata["instance-id"]
++ with open(PRODUCT_UUID_FILE_PATH, "r") as id_file:
++ self.metadata["instance-id"] = str(id_file.read()).rstrip().lower()
++ return self.metadata["instance-id"]
++
++ def get_public_ssh_keys(self):
++ for key_name in (
++ "public-keys-data",
++ "public_keys_data",
++ "public-keys",
++ "public_keys",
++ ):
++ if key_name in self.metadata:
++ return sources.normalize_pubkey_data(self.metadata[key_name])
++ return []
++
++ def redact_keys(self):
++ # Determine if there are any keys to redact.
++ keys_to_redact = None
++ if REDACT in self.metadata:
++ keys_to_redact = self.metadata[REDACT]
++ elif CLEANUP_GUESTINFO in self.metadata:
++ # This is for backwards compatibility.
++ keys_to_redact = self.metadata[CLEANUP_GUESTINFO]
++
++ if self.data_access_method == DATA_ACCESS_METHOD_GUESTINFO:
++ guestinfo_redact_keys(keys_to_redact, self.vmware_rpctool)
++
++
++def decode(key, enc_type, data):
++ """
++ decode returns the decoded string value of data
++ key is a string used to identify the data being decoded in log messages
++ """
++ LOG.debug("Getting encoded data for key=%s, enc=%s", key, enc_type)
++
++ raw_data = None
++ if enc_type in ["gzip+base64", "gz+b64"]:
++ LOG.debug("Decoding %s format %s", enc_type, key)
++ raw_data = util.decomp_gzip(util.b64d(data))
++ elif enc_type in ["base64", "b64"]:
++ LOG.debug("Decoding %s format %s", enc_type, key)
++ raw_data = util.b64d(data)
++ else:
++ LOG.debug("Plain-text data %s", key)
++ raw_data = data
++
++ return util.decode_binary(raw_data)
++
++
++def get_none_if_empty_val(val):
++ """
++ get_none_if_empty_val returns None if the provided value, once stripped
++ of its trailing whitespace, is empty or equal to GUESTINFO_EMPTY_YAML_VAL.
++
++ The return value is always a string, regardless of whether the input is
++ a bytes class or a string.
++ """
++
++ # If the provided value is a bytes class, convert it to a string to
++ # simplify the rest of this function's logic.
++ val = util.decode_binary(val)
++ val = val.rstrip()
++ if len(val) == 0 or val == GUESTINFO_EMPTY_YAML_VAL:
++ return None
++ return val
++
++
++def advertise_local_ip_addrs(host_info):
++ """
++ advertise_local_ip_addrs gets the local IP address information from
++ the provided host_info map and sets the addresses in the guestinfo
++ namespace
++ """
++ if not host_info:
++ return
++
++ # Reflect any possible local IPv4 or IPv6 addresses in the guest
++ # info.
++ local_ipv4 = host_info.get(LOCAL_IPV4)
++ if local_ipv4:
++ guestinfo_set_value(LOCAL_IPV4, local_ipv4)
++ LOG.info("advertised local ipv4 address %s in guestinfo", local_ipv4)
++
++ local_ipv6 = host_info.get(LOCAL_IPV6)
++ if local_ipv6:
++ guestinfo_set_value(LOCAL_IPV6, local_ipv6)
++ LOG.info("advertised local ipv6 address %s in guestinfo", local_ipv6)
++
++
++def handle_returned_guestinfo_val(key, val):
++ """
++ handle_returned_guestinfo_val returns the provided value if it is
++ not empty or set to GUESTINFO_EMPTY_YAML_VAL, otherwise None is
++ returned
++ """
++ val = get_none_if_empty_val(val)
++ if val:
++ return val
++ LOG.debug("No value found for key %s", key)
++ return None
++
++
++def get_guestinfo_key_name(key):
++ return "guestinfo." + key
++
++
++def get_guestinfo_envvar_key_name(key):
++ return ("vmx." + get_guestinfo_key_name(key)).upper().replace(".", "_", -1)
++
++
++def guestinfo_envvar(key):
++ val = guestinfo_envvar_get_value(key)
++ if not val:
++ return None
++ enc_type = guestinfo_envvar_get_value(key + ".encoding")
++ return decode(get_guestinfo_envvar_key_name(key), enc_type, val)
++
++
++def guestinfo_envvar_get_value(key):
++ env_key = get_guestinfo_envvar_key_name(key)
++ return handle_returned_guestinfo_val(key, os.environ.get(env_key, ""))
++
++
++def guestinfo(key, vmware_rpctool=VMWARE_RPCTOOL):
++ """
++ guestinfo returns the guestinfo value for the provided key, decoding
++ the value when required
++ """
++ val = guestinfo_get_value(key, vmware_rpctool)
++ if not val:
++ return None
++ enc_type = guestinfo_get_value(key + ".encoding", vmware_rpctool)
++ return decode(get_guestinfo_key_name(key), enc_type, val)
++
++
++def guestinfo_get_value(key, vmware_rpctool=VMWARE_RPCTOOL):
++ """
++ Returns a guestinfo value for the specified key.
++ """
++ LOG.debug("Getting guestinfo value for key %s", key)
++
++ try:
++ (stdout, stderr) = subp(
++ [
++ vmware_rpctool,
++ "info-get " + get_guestinfo_key_name(key),
++ ]
++ )
++ if stderr == NOVAL:
++ LOG.debug("No value found for key %s", key)
++ elif not stdout:
++ LOG.error("Failed to get guestinfo value for key %s", key)
++ return handle_returned_guestinfo_val(key, stdout)
++ except ProcessExecutionError as error:
++ if error.stderr == NOVAL:
++ LOG.debug("No value found for key %s", key)
++ else:
++ util.logexc(
++ LOG,
++ "Failed to get guestinfo value for key %s: %s",
++ key,
++ error,
++ )
++ except Exception:
++ util.logexc(
++ LOG,
++ "Unexpected error while trying to get "
++ + "guestinfo value for key %s",
++ key,
++ )
++
++ return None
++
++
++def guestinfo_set_value(key, value, vmware_rpctool=VMWARE_RPCTOOL):
++ """
++ Sets a guestinfo value for the specified key. Set value to an empty string
++ to clear an existing guestinfo key.
++ """
++
++ # If value is an empty string then set it to a single space as it is not
++ # possible to set a guestinfo key to an empty string. Setting a guestinfo
++ # key to a single space is as close as it gets to clearing an existing
++ # guestinfo key.
++ if value == "":
++ value = " "
++
++ LOG.debug("Setting guestinfo key=%s to value=%s", key, value)
++
++ try:
++ subp(
++ [
++ vmware_rpctool,
++ ("info-set %s %s" % (get_guestinfo_key_name(key), value)),
++ ]
++ )
++ return True
++ except ProcessExecutionError as error:
++ util.logexc(
++ LOG,
++ "Failed to set guestinfo key=%s to value=%s: %s",
++ key,
++ value,
++ error,
++ )
++ except Exception:
++ util.logexc(
++ LOG,
++ "Unexpected error while trying to set "
++ + "guestinfo key=%s to value=%s",
++ key,
++ value,
++ )
++
++ return None
++
++
++def guestinfo_redact_keys(keys, vmware_rpctool=VMWARE_RPCTOOL):
++ """
++ guestinfo_redact_keys redacts guestinfo of all of the keys in the given
++ list. each key will have its value set to "---". Since the value is valid
++ YAML, cloud-init can still read it if it tries.
++ """
++ if not keys:
++ return
++ if not type(keys) in (list, tuple):
++ keys = [keys]
++ for key in keys:
++ key_name = get_guestinfo_key_name(key)
++ LOG.info("clearing %s", key_name)
++ if not guestinfo_set_value(
++ key, GUESTINFO_EMPTY_YAML_VAL, vmware_rpctool
++ ):
++ LOG.error("failed to clear %s", key_name)
++ LOG.info("clearing %s.encoding", key_name)
++ if not guestinfo_set_value(key + ".encoding", "", vmware_rpctool):
++ LOG.error("failed to clear %s.encoding", key_name)
++
++
++def load_json_or_yaml(data):
++ """
++ load first attempts to unmarshal the provided data as JSON, and if
++ that fails then attempts to unmarshal the data as YAML. If data is
++ None then a new dictionary is returned.
++ """
++ if not data:
++ return {}
++ try:
++ return util.load_json(data)
++ except (json.JSONDecodeError, TypeError):
++ return util.load_yaml(data)
++
++
++def process_metadata(data):
++ """
++ process_metadata processes metadata and loads the optional network
++ configuration.
++ """
++ network = None
++ if "network" in data:
++ network = data["network"]
++ del data["network"]
++
++ network_enc = None
++ if "network.encoding" in data:
++ network_enc = data["network.encoding"]
++ del data["network.encoding"]
++
++ if network:
++ if isinstance(network, collections.abc.Mapping):
++ LOG.debug("network data copied to 'config' key")
++ network = {"config": copy.deepcopy(network)}
++ else:
++ LOG.debug("network data to be decoded %s", network)
++ dec_net = decode("metadata.network", network_enc, network)
++ network = {
++ "config": load_json_or_yaml(dec_net),
++ }
++
++ LOG.debug("network data %s", network)
++ data["network"] = network
++
++ return data
++
++
++# Used to match classes to dependencies
++datasources = [
++ (DataSourceVMware, (sources.DEP_FILESYSTEM,)), # Run at init-local
++ (DataSourceVMware, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
++]
++
++
++def get_datasource_list(depends):
++ """
++ Return a list of data sources that match this set of dependencies
++ """
++ return sources.list_from_depends(depends, datasources)
++
++
++def get_default_ip_addrs():
++ """
++ Returns the default IPv4 and IPv6 addresses based on the device(s) used for
++ the default route. Please note that None may be returned for either address
++ family if that family has no default route or if there are multiple
++ addresses associated with the device used by the default route for a given
++ address.
++ """
++ # TODO(promote and use netifaces in cloudinit.net* modules)
++ gateways = netifaces.gateways()
++ if "default" not in gateways:
++ return None, None
++
++ default_gw = gateways["default"]
++ if (
++ netifaces.AF_INET not in default_gw
++ and netifaces.AF_INET6 not in default_gw
++ ):
++ return None, None
++
++ ipv4 = None
++ ipv6 = None
++
++ gw4 = default_gw.get(netifaces.AF_INET)
++ if gw4:
++ _, dev4 = gw4
++ addr4_fams = netifaces.ifaddresses(dev4)
++ if addr4_fams:
++ af_inet4 = addr4_fams.get(netifaces.AF_INET)
++ if af_inet4:
++ if len(af_inet4) > 1:
++ LOG.warning(
++ "device %s has more than one ipv4 address: %s",
++ dev4,
++ af_inet4,
++ )
++ elif "addr" in af_inet4[0]:
++ ipv4 = af_inet4[0]["addr"]
++
++ # Try to get the default IPv6 address by first seeing if there is a default
++ # IPv6 route.
++ gw6 = default_gw.get(netifaces.AF_INET6)
++ if gw6:
++ _, dev6 = gw6
++ addr6_fams = netifaces.ifaddresses(dev6)
++ if addr6_fams:
++ af_inet6 = addr6_fams.get(netifaces.AF_INET6)
++ if af_inet6:
++ if len(af_inet6) > 1:
++ LOG.warning(
++ "device %s has more than one ipv6 address: %s",
++ dev6,
++ af_inet6,
++ )
++ elif "addr" in af_inet6[0]:
++ ipv6 = af_inet6[0]["addr"]
++
++ # If there is a default IPv4 address but not IPv6, then see if there is a
++ # single IPv6 address associated with the same device associated with the
++ # default IPv4 address.
++ if ipv4 and not ipv6:
++ af_inet6 = addr4_fams.get(netifaces.AF_INET6)
++ if af_inet6:
++ if len(af_inet6) > 1:
++ LOG.warning(
++ "device %s has more than one ipv6 address: %s",
++ dev4,
++ af_inet6,
++ )
++ elif "addr" in af_inet6[0]:
++ ipv6 = af_inet6[0]["addr"]
++
++ # If there is a default IPv6 address but not IPv4, then see if there is a
++ # single IPv4 address associated with the same device associated with the
++ # default IPv6 address.
++ if not ipv4 and ipv6:
++ af_inet4 = addr6_fams.get(netifaces.AF_INET)
++ if af_inet4:
++ if len(af_inet4) > 1:
++ LOG.warning(
++ "device %s has more than one ipv4 address: %s",
++ dev6,
++ af_inet4,
++ )
++ elif "addr" in af_inet4[0]:
++ ipv4 = af_inet4[0]["addr"]
++
++ return ipv4, ipv6
++
++
++# patched socket.getfqdn() - see https://bugs.python.org/issue5004
++
++
++def getfqdn(name=""):
++ """Get fully qualified domain name from name.
++ An empty argument is interpreted as meaning the local host.
++ """
++ # TODO(may want to promote this function to util.getfqdn)
++ # TODO(may want to extend util.get_hostname to accept fqdn=True param)
++ name = name.strip()
++ if not name or name == "0.0.0.0":
++ name = util.get_hostname()
++ try:
++ addrs = socket.getaddrinfo(
++ name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME
++ )
++ except socket.error:
++ pass
++ else:
++ for addr in addrs:
++ if addr[3]:
++ name = addr[3]
++ break
++ return name
++
++
++def is_valid_ip_addr(val):
++ """
++ Returns false if the address is loopback, link local or unspecified;
++ otherwise true is returned.
++ """
++ # TODO(extend cloudinit.net.is_ip_addr exclude link_local/loopback etc)
++ # TODO(migrate to use cloudinit.net.is_ip_addr)#
++
++ addr = None
++ try:
++ addr = ipaddress.ip_address(val)
++ except ipaddress.AddressValueError:
++ addr = ipaddress.ip_address(str(val))
++ except Exception:
++ return None
++
++ if addr.is_link_local or addr.is_loopback or addr.is_unspecified:
++ return False
++ return True
++
++
++def get_host_info():
++ """
++ Returns host information such as the host name and network interfaces.
++ """
++ # TODO(look to promote netifices use up in cloud-init netinfo funcs)
++ host_info = {
++ "network": {
++ "interfaces": {
++ "by-mac": collections.OrderedDict(),
++ "by-ipv4": collections.OrderedDict(),
++ "by-ipv6": collections.OrderedDict(),
++ },
++ },
++ }
++ hostname = getfqdn(util.get_hostname())
++ if hostname:
++ host_info["hostname"] = hostname
++ host_info["local-hostname"] = hostname
++ host_info["local_hostname"] = hostname
++
++ default_ipv4, default_ipv6 = get_default_ip_addrs()
++ if default_ipv4:
++ host_info[LOCAL_IPV4] = default_ipv4
++ if default_ipv6:
++ host_info[LOCAL_IPV6] = default_ipv6
++
++ by_mac = host_info["network"]["interfaces"]["by-mac"]
++ by_ipv4 = host_info["network"]["interfaces"]["by-ipv4"]
++ by_ipv6 = host_info["network"]["interfaces"]["by-ipv6"]
++
++ ifaces = netifaces.interfaces()
++ for dev_name in ifaces:
++ addr_fams = netifaces.ifaddresses(dev_name)
++ af_link = addr_fams.get(netifaces.AF_LINK)
++ af_inet4 = addr_fams.get(netifaces.AF_INET)
++ af_inet6 = addr_fams.get(netifaces.AF_INET6)
++
++ mac = None
++ if af_link and "addr" in af_link[0]:
++ mac = af_link[0]["addr"]
++
++ # Do not bother recording localhost
++ if mac == "00:00:00:00:00:00":
++ continue
++
++ if mac and (af_inet4 or af_inet6):
++ key = mac
++ val = {}
++ if af_inet4:
++ af_inet4_vals = []
++ for ip_info in af_inet4:
++ if not is_valid_ip_addr(ip_info["addr"]):
++ continue
++ af_inet4_vals.append(ip_info)
++ val["ipv4"] = af_inet4_vals
++ if af_inet6:
++ af_inet6_vals = []
++ for ip_info in af_inet6:
++ if not is_valid_ip_addr(ip_info["addr"]):
++ continue
++ af_inet6_vals.append(ip_info)
++ val["ipv6"] = af_inet6_vals
++ by_mac[key] = val
++
++ if af_inet4:
++ for ip_info in af_inet4:
++ key = ip_info["addr"]
++ if not is_valid_ip_addr(key):
++ continue
++ val = copy.deepcopy(ip_info)
++ del val["addr"]
++ if mac:
++ val["mac"] = mac
++ by_ipv4[key] = val
++
++ if af_inet6:
++ for ip_info in af_inet6:
++ key = ip_info["addr"]
++ if not is_valid_ip_addr(key):
++ continue
++ val = copy.deepcopy(ip_info)
++ del val["addr"]
++ if mac:
++ val["mac"] = mac
++ by_ipv6[key] = val
++
++ return host_info
++
++
++def wait_on_network(metadata):
++ # Determine whether we need to wait on the network coming online.
++ wait_on_ipv4 = False
++ wait_on_ipv6 = False
++ if WAIT_ON_NETWORK in metadata:
++ wait_on_network = metadata[WAIT_ON_NETWORK]
++ if WAIT_ON_NETWORK_IPV4 in wait_on_network:
++ wait_on_ipv4_val = wait_on_network[WAIT_ON_NETWORK_IPV4]
++ if isinstance(wait_on_ipv4_val, bool):
++ wait_on_ipv4 = wait_on_ipv4_val
++ else:
++ wait_on_ipv4 = util.translate_bool(wait_on_ipv4_val)
++ if WAIT_ON_NETWORK_IPV6 in wait_on_network:
++ wait_on_ipv6_val = wait_on_network[WAIT_ON_NETWORK_IPV6]
++ if isinstance(wait_on_ipv6_val, bool):
++ wait_on_ipv6 = wait_on_ipv6_val
++ else:
++ wait_on_ipv6 = util.translate_bool(wait_on_ipv6_val)
++
++ # Get information about the host.
++ host_info = None
++ while host_info is None:
++ # This loop + sleep results in two logs every second while waiting
++ # for either ipv4 or ipv6 up. Do we really need to log each iteration
++ # or can we log once and log on successful exit?
++ host_info = get_host_info()
++
++ network = host_info.get("network") or {}
++ interfaces = network.get("interfaces") or {}
++ by_ipv4 = interfaces.get("by-ipv4") or {}
++ by_ipv6 = interfaces.get("by-ipv6") or {}
++
++ if wait_on_ipv4:
++ ipv4_ready = len(by_ipv4) > 0 if by_ipv4 else False
++ if not ipv4_ready:
++ host_info = None
++
++ if wait_on_ipv6:
++ ipv6_ready = len(by_ipv6) > 0 if by_ipv6 else False
++ if not ipv6_ready:
++ host_info = None
++
++ if host_info is None:
++ LOG.debug(
++ "waiting on network: wait4=%s, ready4=%s, wait6=%s, ready6=%s",
++ wait_on_ipv4,
++ ipv4_ready,
++ wait_on_ipv6,
++ ipv6_ready,
++ )
++ time.sleep(1)
++
++ LOG.debug("waiting on network complete")
++ return host_info
++
++
++def main():
++ """
++ Executed when this file is used as a program.
++ """
++ try:
++ logging.setupBasicLogging()
++ except Exception:
++ pass
++ metadata = {
++ "wait-on-network": {"ipv4": True, "ipv6": "false"},
++ "network": {"config": {"dhcp": True}},
++ }
++ host_info = wait_on_network(metadata)
++ metadata = util.mergemanydict([metadata, host_info])
++ print(util.json_dumps(metadata))
++
++
++if __name__ == "__main__":
++ main()
++
++# vi: ts=4 expandtab
+diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst
+index f58b2b38..6606367c 100644
+--- a/doc/rtd/topics/availability.rst
++++ b/doc/rtd/topics/availability.rst
+@@ -64,5 +64,6 @@ Additionally, cloud-init is supported on these private clouds:
+ - LXD
+ - KVM
+ - Metal-as-a-Service (MAAS)
++- VMware
+
+ .. vi: textwidth=79
+diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
+index 228173d2..8afed470 100644
+--- a/doc/rtd/topics/datasources.rst
++++ b/doc/rtd/topics/datasources.rst
+@@ -49,7 +49,7 @@ The following is a list of documents for each supported datasource:
+ datasources/smartos.rst
+ datasources/upcloud.rst
+ datasources/zstack.rst
+-
++ datasources/vmware.rst
+
+ Creation
+ ========
+diff --git a/doc/rtd/topics/datasources/vmware.rst b/doc/rtd/topics/datasources/vmware.rst
+new file mode 100644
+index 00000000..996eb61f
+--- /dev/null
++++ b/doc/rtd/topics/datasources/vmware.rst
+@@ -0,0 +1,359 @@
++.. _datasource_vmware:
++
++VMware
++======
++
++This datasource is for use with systems running on a VMware platform such as
++vSphere and currently supports the following data transports:
++
++
++* `GuestInfo <https://github.com/vmware/govmomi/blob/master/govc/USAGE.md#vmchange>`_ keys
++
++Configuration
++-------------
++
++The configuration method is dependent upon the transport:
++
++GuestInfo Keys
++^^^^^^^^^^^^^^
++
++One method of providing meta, user, and vendor data is by setting the following
++key/value pairs on a VM's ``extraConfig`` `property <https://vdc-repo.vmware.com/vmwb-repository/dcr-public/723e7f8b-4f21-448b-a830-5f22fd931b01/5a8257bd-7f41-4423-9a73-03307535bd42/doc/vim.vm.ConfigInfo.html>`_ :
++
++.. list-table::
++ :header-rows: 1
++
++ * - Property
++ - Description
++ * - ``guestinfo.metadata``
++ - A YAML or JSON document containing the cloud-init metadata.
++ * - ``guestinfo.metadata.encoding``
++ - The encoding type for ``guestinfo.metadata``.
++ * - ``guestinfo.userdata``
++ - A YAML document containing the cloud-init user data.
++ * - ``guestinfo.userdata.encoding``
++ - The encoding type for ``guestinfo.userdata``.
++ * - ``guestinfo.vendordata``
++ - A YAML document containing the cloud-init vendor data.
++ * - ``guestinfo.vendordata.encoding``
++ - The encoding type for ``guestinfo.vendordata``.
++
++
++All ``guestinfo.*.encoding`` values may be set to ``base64`` or
++``gzip+base64``.
++
++Features
++--------
++
++This section reviews several features available in this datasource, regardless
++of how the meta, user, and vendor data was discovered.
++
++Instance data and lazy networks
++^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
++
++One of the hallmarks of cloud-init is `its use of instance-data and JINJA
++queries <../instancedata.html#using-instance-data>`_
++-- the ability to write queries in user and vendor data that reference runtime
++information present in ``/run/cloud-init/instance-data.json``. This works well
++when the metadata provides all of the information up front, such as the network
++configuration. For systems that rely on DHCP, however, this information may not
++be available when the metadata is persisted to disk.
++
++This datasource ensures that even if the instance is using DHCP to configure
++networking, the same details about the configured network are available in
++``/run/cloud-init/instance-data.json`` as if static networking was used. This
++information collected at runtime is easy to demonstrate by executing the
++datasource on the command line. From the root of this repository, run the
++following command:
++
++.. code-block:: bash
++
++ PYTHONPATH="$(pwd)" python3 cloudinit/sources/DataSourceVMware.py
++
++The above command will result in output similar to the below JSON:
++
++.. code-block:: json
++
++ {
++ "hostname": "akutz.localhost",
++ "local-hostname": "akutz.localhost",
++ "local-ipv4": "192.168.0.188",
++ "local_hostname": "akutz.localhost",
++ "network": {
++ "config": {
++ "dhcp": true
++ },
++ "interfaces": {
++ "by-ipv4": {
++ "172.0.0.2": {
++ "netmask": "255.255.255.255",
++ "peer": "172.0.0.2"
++ },
++ "192.168.0.188": {
++ "broadcast": "192.168.0.255",
++ "mac": "64:4b:f0:18:9a:21",
++ "netmask": "255.255.255.0"
++ }
++ },
++ "by-ipv6": {
++ "fd8e:d25e:c5b6:1:1f5:b2fd:8973:22f2": {
++ "flags": 208,
++ "mac": "64:4b:f0:18:9a:21",
++ "netmask": "ffff:ffff:ffff:ffff::/64"
++ }
++ },
++ "by-mac": {
++ "64:4b:f0:18:9a:21": {
++ "ipv4": [
++ {
++ "addr": "192.168.0.188",
++ "broadcast": "192.168.0.255",
++ "netmask": "255.255.255.0"
++ }
++ ],
++ "ipv6": [
++ {
++ "addr": "fd8e:d25e:c5b6:1:1f5:b2fd:8973:22f2",
++ "flags": 208,
++ "netmask": "ffff:ffff:ffff:ffff::/64"
++ }
++ ]
++ },
++ "ac:de:48:00:11:22": {
++ "ipv6": []
++ }
++ }
++ }
++ },
++ "wait-on-network": {
++ "ipv4": true,
++ "ipv6": "false"
++ }
++ }
++
++
++Redacting sensitive information
++^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
++
++Sometimes the cloud-init userdata might contain sensitive information, and it
++may be desirable to have the ``guestinfo.userdata`` key (or other guestinfo
++keys) redacted as soon as its data is read by the datasource. This is possible
++by adding the following to the metadata:
++
++.. code-block:: yaml
++
++ redact: # formerly named cleanup-guestinfo, which will also work
++ - userdata
++ - vendordata
++
++When the above snippet is added to the metadata, the datasource will iterate
++over the elements in the ``redact`` array and clear each of the keys. For
++example, when the guestinfo transport is used, the above snippet will cause
++the following commands to be executed:
++
++.. code-block:: shell
++
++ vmware-rpctool "info-set guestinfo.userdata ---"
++ vmware-rpctool "info-set guestinfo.userdata.encoding "
++ vmware-rpctool "info-set guestinfo.vendordata ---"
++ vmware-rpctool "info-set guestinfo.vendordata.encoding "
++
++Please note that keys are set to the valid YAML string ``---`` as it is not
++possible remove an existing key from the guestinfo key-space. A key's analogous
++encoding property will be set to a single white-space character, causing the
++datasource to treat the actual key value as plain-text, thereby loading it as
++an empty YAML doc (hence the aforementioned ``---``\ ).
++
++Reading the local IP addresses
++^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
++
++This datasource automatically discovers the local IPv4 and IPv6 addresses for
++a guest operating system based on the default routes. However, when inspecting
++a VM externally, it's not possible to know what the *default* IP address is for
++the guest OS. That's why this datasource sets the discovered, local IPv4 and
++IPv6 addresses back in the guestinfo namespace as the following keys:
++
++
++* ``guestinfo.local-ipv4``
++* ``guestinfo.local-ipv6``
++
++It is possible that a host may not have any default, local IP addresses. It's
++also possible the reported, local addresses are link-local addresses. But these
++two keys may be used to discover what this datasource determined were the local
++IPv4 and IPv6 addresses for a host.
++
++Waiting on the network
++^^^^^^^^^^^^^^^^^^^^^^
++
++Sometimes cloud-init may bring up the network, but it will not finish coming
++online before the datasource's ``setup`` function is called, resulting in an
++``/var/run/cloud-init/instance-data.json`` file that does not have the correct
++network information. It is possible to instruct the datasource to wait until an
++IPv4 or IPv6 address is available before writing the instance data with the
++following metadata properties:
++
++.. code-block:: yaml
++
++ wait-on-network:
++ ipv4: true
++ ipv6: true
++
++If either of the above values are true, then the datasource will sleep for a
++second, check the network status, and repeat until one or both addresses from
++the specified families are available.
++
++Walkthrough
++-----------
++
++The following series of steps is a demonstration on how to configure a VM with
++this datasource:
++
++
++#. Create the metadata file for the VM. Save the following YAML to a file named
++ ``metadata.yaml``\ :
++
++ .. code-block:: yaml
++
++ instance-id: cloud-vm
++ local-hostname: cloud-vm
++ network:
++ version: 2
++ ethernets:
++ nics:
++ match:
++ name: ens*
++ dhcp4: yes
++
++#. Create the userdata file ``userdata.yaml``\ :
++
++ .. code-block:: yaml
++
++ #cloud-config
++
++ users:
++ - default
++ - name: akutz
++ primary_group: akutz
++ sudo: ALL=(ALL) NOPASSWD:ALL
++ groups: sudo, wheel
++ ssh_import_id: None
++ lock_passwd: true
++ ssh_authorized_keys:
++ - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDE0c5FczvcGSh/tG4iw+Fhfi/O5/EvUM/96js65tly4++YTXK1d9jcznPS5ruDlbIZ30oveCBd3kT8LLVFwzh6hepYTf0YmCTpF4eDunyqmpCXDvVscQYRXyasEm5olGmVe05RrCJSeSShAeptv4ueIn40kZKOghinGWLDSZG4+FFfgrmcMCpx5YSCtX2gvnEYZJr0czt4rxOZuuP7PkJKgC/mt2PcPjooeX00vAj81jjU2f3XKrjjz2u2+KIt9eba+vOQ6HiC8c2IzRkUAJ5i1atLy8RIbejo23+0P4N2jjk17QySFOVHwPBDTYb0/0M/4ideeU74EN/CgVsvO6JrLsPBR4dojkV5qNbMNxIVv5cUwIy2ThlLgqpNCeFIDLCWNZEFKlEuNeSQ2mPtIO7ETxEL2Cz5y/7AIuildzYMc6wi2bofRC8HmQ7rMXRWdwLKWsR0L7SKjHblIwarxOGqLnUI+k2E71YoP7SZSlxaKi17pqkr0OMCF+kKqvcvHAQuwGqyumTEWOlH6TCx1dSPrW+pVCZSHSJtSTfDW2uzL6y8k10MT06+pVunSrWo5LHAXcS91htHV1M1UrH/tZKSpjYtjMb5+RonfhaFRNzvj7cCE1f3Kp8UVqAdcGBTtReoE8eRUT63qIxjw03a7VwAyB2w+9cu1R9/vAo8SBeRqw== sakutz@gmail.com
++
++#. Please note this step requires that the VM be powered off. All of the
++ commands below use the VMware CLI tool, `govc <https://github.com/vmware/govmomi/blob/master/govc>`_.
++
++ Go ahead and assign the path to the VM to the environment variable ``VM``\ :
++
++ .. code-block:: shell
++
++ export VM="/inventory/path/to/the/vm"
++
++#. Power off the VM:
++
++ .. raw:: html
++
++ <hr />
++
++ ⚠️ <strong>First Boot Mode</strong>
++
++ To ensure the next power-on operation results in a first-boot scenario for
++ cloud-init, it may be necessary to run the following command just before
++ powering off the VM:
++
++ .. code-block:: bash
++
++ cloud-init clean
++
++ Otherwise cloud-init may not run in first-boot mode. For more information
++ on how the boot mode is determined, please see the
++ `First Boot Documentation <../boot.html#first-boot-determination>`_.
++
++ .. raw:: html
++
++ <hr />
++
++ .. code-block:: shell
++
++ govc vm.power -off "${VM}"
++
++#.
++ Export the environment variables that contain the cloud-init metadata and
++ userdata:
++
++ .. code-block:: shell
++
++ export METADATA=$(gzip -c9 <metadata.yaml | { base64 -w0 2>/dev/null || base64; }) \
++ USERDATA=$(gzip -c9 <userdata.yaml | { base64 -w0 2>/dev/null || base64; })
++
++#.
++ Assign the metadata and userdata to the VM:
++
++ .. code-block:: shell
++
++ govc vm.change -vm "${VM}" \
++ -e guestinfo.metadata="${METADATA}" \
++ -e guestinfo.metadata.encoding="gzip+base64" \
++ -e guestinfo.userdata="${USERDATA}" \
++ -e guestinfo.userdata.encoding="gzip+base64"
++
++ Please note the above commands include specifying the encoding for the
++ properties. This is important as it informs the datasource how to decode
++ the data for cloud-init. Valid values for ``metadata.encoding`` and
++ ``userdata.encoding`` include:
++
++
++ * ``base64``
++ * ``gzip+base64``
++
++#.
++ Power on the VM:
++
++ .. code-block:: shell
++
++ govc vm.power -vm "${VM}" -on
++
++If all went according to plan, the CentOS box is:
++
++* Locked down, allowing SSH access only for the user in the userdata
++* Configured for a dynamic IP address via DHCP
++* Has a hostname of ``cloud-vm``
++
++Examples
++--------
++
++This section reviews common configurations:
++
++Setting the hostname
++^^^^^^^^^^^^^^^^^^^^
++
++The hostname is set by way of the metadata key ``local-hostname``.
++
++Setting the instance ID
++^^^^^^^^^^^^^^^^^^^^^^^
++
++The instance ID may be set by way of the metadata key ``instance-id``. However,
++if this value is absent then then the instance ID is read from the file
++``/sys/class/dmi/id/product_uuid``.
++
++Providing public SSH keys
++^^^^^^^^^^^^^^^^^^^^^^^^^
++
++The public SSH keys may be set by way of the metadata key ``public-keys-data``.
++Each newline-terminated string will be interpreted as a separate SSH public
++key, which will be placed in distro's default user's
++``~/.ssh/authorized_keys``. If the value is empty or absent, then nothing will
++be written to ``~/.ssh/authorized_keys``.
++
++Configuring the network
++^^^^^^^^^^^^^^^^^^^^^^^
++
++The network is configured by setting the metadata key ``network`` with a value
++consistent with Network Config Versions
++`1 <../network-config-format-v1.html>`_ or
++`2 <../network-config-format-v2.html>`_\ , depending on the Linux
++distro's version of cloud-init.
++
++The metadata key ``network.encoding`` may be used to indicate the format of
++the metadata key "network". Valid encodings are ``base64`` and ``gzip+base64``.
+diff --git a/requirements.txt b/requirements.txt
+index 5b8becd7..41d01d62 100644
+--- a/requirements.txt
++++ b/requirements.txt
+@@ -29,3 +29,15 @@ requests
+
+ # For patching pieces of cloud-config together
+ jsonpatch
++
++# For validating cloud-config sections per schema definitions
++jsonschema
++
++# Used by DataSourceVMware to inspect the host's network configuration during
++# the "setup()" function.
++#
++# This allows a host that uses DHCP to bring up the network during BootLocal
++# and still participate in instance-data by gathering the network in detail at
++# runtime and merge that information into the metadata and repersist that to
++# disk.
++netifaces>=0.10.9
+diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
+index 5912f7ee..475a2cf8 100644
+--- a/tests/unittests/test_datasource/test_common.py
++++ b/tests/unittests/test_datasource/test_common.py
+@@ -28,6 +28,7 @@ from cloudinit.sources import (
+ DataSourceScaleway as Scaleway,
+ DataSourceSmartOS as SmartOS,
+ DataSourceUpCloud as UpCloud,
++ DataSourceVMware as VMware,
+ )
+ from cloudinit.sources import DataSourceNone as DSNone
+
+@@ -50,6 +51,7 @@ DEFAULT_LOCAL = [
+ RbxCloud.DataSourceRbxCloud,
+ Scaleway.DataSourceScaleway,
+ UpCloud.DataSourceUpCloudLocal,
++ VMware.DataSourceVMware,
+ ]
+
+ DEFAULT_NETWORK = [
+@@ -66,6 +68,7 @@ DEFAULT_NETWORK = [
+ OpenStack.DataSourceOpenStack,
+ OVF.DataSourceOVFNet,
+ UpCloud.DataSourceUpCloud,
++ VMware.DataSourceVMware,
+ ]
+
+
+diff --git a/tests/unittests/test_datasource/test_vmware.py b/tests/unittests/test_datasource/test_vmware.py
+new file mode 100644
+index 00000000..597db7c8
+--- /dev/null
++++ b/tests/unittests/test_datasource/test_vmware.py
+@@ -0,0 +1,377 @@
++# Copyright (c) 2021 VMware, Inc. All Rights Reserved.
++#
++# Authors: Andrew Kutz <akutz@vmware.com>
++#
++# This file is part of cloud-init. See LICENSE file for license information.
++
++import base64
++import gzip
++from cloudinit import dmi, helpers, safeyaml
++from cloudinit import settings
++from cloudinit.sources import DataSourceVMware
++from cloudinit.tests.helpers import (
++ mock,
++ CiTestCase,
++ FilesystemMockingTestCase,
++ populate_dir,
++)
++
++import os
++
++PRODUCT_NAME_FILE_PATH = "/sys/class/dmi/id/product_name"
++PRODUCT_NAME = "VMware7,1"
++PRODUCT_UUID = "82343CED-E4C7-423B-8F6B-0D34D19067AB"
++REROOT_FILES = {
++ DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID,
++ PRODUCT_NAME_FILE_PATH: PRODUCT_NAME,
++}
++
++VMW_MULTIPLE_KEYS = [
++ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@vmw.com",
++ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@vmw.com",
++]
++VMW_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@vmw.com"
++
++VMW_METADATA_YAML = """instance-id: cloud-vm
++local-hostname: cloud-vm
++network:
++ version: 2
++ ethernets:
++ nics:
++ match:
++ name: ens*
++ dhcp4: yes
++"""
++
++VMW_USERDATA_YAML = """## template: jinja
++#cloud-config
++users:
++- default
++"""
++
++VMW_VENDORDATA_YAML = """## template: jinja
++#cloud-config
++runcmd:
++- echo "Hello, world."
++"""
++
++
++class TestDataSourceVMware(CiTestCase):
++ """
++ Test common functionality that is not transport specific.
++ """
++
++ def setUp(self):
++ super(TestDataSourceVMware, self).setUp()
++ self.tmp = self.tmp_dir()
++
++ def test_no_data_access_method(self):
++ ds = get_ds(self.tmp)
++ ds.vmware_rpctool = None
++ ret = ds.get_data()
++ self.assertFalse(ret)
++
++ def test_get_host_info(self):
++ host_info = DataSourceVMware.get_host_info()
++ self.assertTrue(host_info)
++ self.assertTrue(host_info["hostname"])
++ self.assertTrue(host_info["local-hostname"])
++ self.assertTrue(host_info["local_hostname"])
++ self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4])
++
++
++class TestDataSourceVMwareEnvVars(FilesystemMockingTestCase):
++ """
++ Test the envvar transport.
++ """
++
++ def setUp(self):
++ super(TestDataSourceVMwareEnvVars, self).setUp()
++ self.tmp = self.tmp_dir()
++ os.environ[DataSourceVMware.VMX_GUESTINFO] = "1"
++ self.create_system_files()
++
++ def tearDown(self):
++ del os.environ[DataSourceVMware.VMX_GUESTINFO]
++ return super(TestDataSourceVMwareEnvVars, self).tearDown()
++
++ def create_system_files(self):
++ rootd = self.tmp_dir()
++ populate_dir(
++ rootd,
++ {
++ DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID,
++ },
++ )
++ self.assertTrue(self.reRoot(rootd))
++
++ def assert_get_data_ok(self, m_fn, m_fn_call_count=6):
++ ds = get_ds(self.tmp)
++ ds.vmware_rpctool = None
++ ret = ds.get_data()
++ self.assertTrue(ret)
++ self.assertEqual(m_fn_call_count, m_fn.call_count)
++ self.assertEqual(
++ ds.data_access_method, DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR
++ )
++ return ds
++
++ def assert_metadata(self, metadata, m_fn, m_fn_call_count=6):
++ ds = self.assert_get_data_ok(m_fn, m_fn_call_count)
++ assert_metadata(self, ds, metadata)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_subplatform(self, m_fn):
++ m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""]
++ ds = self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++ self.assertEqual(
++ ds.subplatform,
++ "%s (%s)"
++ % (
++ DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR,
++ DataSourceVMware.get_guestinfo_envvar_key_name("metadata"),
++ ),
++ )
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_metadata_only(self, m_fn):
++ m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_userdata_only(self, m_fn):
++ m_fn.side_effect = ["", VMW_USERDATA_YAML, "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_vendordata_only(self, m_fn):
++ m_fn.side_effect = ["", "", VMW_VENDORDATA_YAML, ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_metadata_base64(self, m_fn):
++ data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8"))
++ m_fn.side_effect = [data, "base64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_metadata_b64(self, m_fn):
++ data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8"))
++ m_fn.side_effect = [data, "b64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_metadata_gzip_base64(self, m_fn):
++ data = VMW_METADATA_YAML.encode("utf-8")
++ data = gzip.compress(data)
++ data = base64.b64encode(data)
++ m_fn.side_effect = [data, "gzip+base64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_get_data_metadata_gz_b64(self, m_fn):
++ data = VMW_METADATA_YAML.encode("utf-8")
++ data = gzip.compress(data)
++ data = base64.b64encode(data)
++ m_fn.side_effect = [data, "gz+b64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_metadata_single_ssh_key(self, m_fn):
++ metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML)
++ metadata["public_keys"] = VMW_SINGLE_KEY
++ metadata_yaml = safeyaml.dumps(metadata)
++ m_fn.side_effect = [metadata_yaml, "", "", ""]
++ self.assert_metadata(metadata, m_fn, m_fn_call_count=4)
++
++ @mock.patch(
++ "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value"
++ )
++ def test_metadata_multiple_ssh_keys(self, m_fn):
++ metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML)
++ metadata["public_keys"] = VMW_MULTIPLE_KEYS
++ metadata_yaml = safeyaml.dumps(metadata)
++ m_fn.side_effect = [metadata_yaml, "", "", ""]
++ self.assert_metadata(metadata, m_fn, m_fn_call_count=4)
++
++
++class TestDataSourceVMwareGuestInfo(FilesystemMockingTestCase):
++ """
++ Test the guestinfo transport on a VMware platform.
++ """
++
++ def setUp(self):
++ super(TestDataSourceVMwareGuestInfo, self).setUp()
++ self.tmp = self.tmp_dir()
++ self.create_system_files()
++
++ def create_system_files(self):
++ rootd = self.tmp_dir()
++ populate_dir(
++ rootd,
++ {
++ DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID,
++ PRODUCT_NAME_FILE_PATH: PRODUCT_NAME,
++ },
++ )
++ self.assertTrue(self.reRoot(rootd))
++
++ def assert_get_data_ok(self, m_fn, m_fn_call_count=6):
++ ds = get_ds(self.tmp)
++ ds.vmware_rpctool = "vmware-rpctool"
++ ret = ds.get_data()
++ self.assertTrue(ret)
++ self.assertEqual(m_fn_call_count, m_fn.call_count)
++ self.assertEqual(
++ ds.data_access_method,
++ DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO,
++ )
++ return ds
++
++ def assert_metadata(self, metadata, m_fn, m_fn_call_count=6):
++ ds = self.assert_get_data_ok(m_fn, m_fn_call_count)
++ assert_metadata(self, ds, metadata)
++
++ def test_ds_valid_on_vmware_platform(self):
++ system_type = dmi.read_dmi_data("system-product-name")
++ self.assertEqual(system_type, PRODUCT_NAME)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_subplatform(self, m_fn):
++ m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""]
++ ds = self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++ self.assertEqual(
++ ds.subplatform,
++ "%s (%s)"
++ % (
++ DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO,
++ DataSourceVMware.get_guestinfo_key_name("metadata"),
++ ),
++ )
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_data_userdata_only(self, m_fn):
++ m_fn.side_effect = ["", VMW_USERDATA_YAML, "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_data_vendordata_only(self, m_fn):
++ m_fn.side_effect = ["", "", VMW_VENDORDATA_YAML, ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_metadata_single_ssh_key(self, m_fn):
++ metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML)
++ metadata["public_keys"] = VMW_SINGLE_KEY
++ metadata_yaml = safeyaml.dumps(metadata)
++ m_fn.side_effect = [metadata_yaml, "", "", ""]
++ self.assert_metadata(metadata, m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_metadata_multiple_ssh_keys(self, m_fn):
++ metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML)
++ metadata["public_keys"] = VMW_MULTIPLE_KEYS
++ metadata_yaml = safeyaml.dumps(metadata)
++ m_fn.side_effect = [metadata_yaml, "", "", ""]
++ self.assert_metadata(metadata, m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_data_metadata_base64(self, m_fn):
++ data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8"))
++ m_fn.side_effect = [data, "base64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_data_metadata_b64(self, m_fn):
++ data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8"))
++ m_fn.side_effect = [data, "b64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_data_metadata_gzip_base64(self, m_fn):
++ data = VMW_METADATA_YAML.encode("utf-8")
++ data = gzip.compress(data)
++ data = base64.b64encode(data)
++ m_fn.side_effect = [data, "gzip+base64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_get_data_metadata_gz_b64(self, m_fn):
++ data = VMW_METADATA_YAML.encode("utf-8")
++ data = gzip.compress(data)
++ data = base64.b64encode(data)
++ m_fn.side_effect = [data, "gz+b64", "", ""]
++ self.assert_get_data_ok(m_fn, m_fn_call_count=4)
++
++
++class TestDataSourceVMwareGuestInfo_InvalidPlatform(FilesystemMockingTestCase):
++ """
++ Test the guestinfo transport on a non-VMware platform.
++ """
++
++ def setUp(self):
++ super(TestDataSourceVMwareGuestInfo_InvalidPlatform, self).setUp()
++ self.tmp = self.tmp_dir()
++ self.create_system_files()
++
++ def create_system_files(self):
++ rootd = self.tmp_dir()
++ populate_dir(
++ rootd,
++ {
++ DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID,
++ },
++ )
++ self.assertTrue(self.reRoot(rootd))
++
++ @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value")
++ def test_ds_invalid_on_non_vmware_platform(self, m_fn):
++ system_type = dmi.read_dmi_data("system-product-name")
++ self.assertEqual(system_type, None)
++
++ m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""]
++ ds = get_ds(self.tmp)
++ ds.vmware_rpctool = "vmware-rpctool"
++ ret = ds.get_data()
++ self.assertFalse(ret)
++
++
++def assert_metadata(test_obj, ds, metadata):
++ test_obj.assertEqual(metadata.get("instance-id"), ds.get_instance_id())
++ test_obj.assertEqual(metadata.get("local-hostname"), ds.get_hostname())
++
++ expected_public_keys = metadata.get("public_keys")
++ if not isinstance(expected_public_keys, list):
++ expected_public_keys = [expected_public_keys]
++
++ test_obj.assertEqual(expected_public_keys, ds.get_public_ssh_keys())
++ test_obj.assertIsInstance(ds.get_public_ssh_keys(), list)
++
++
++def get_ds(temp_dir):
++ ds = DataSourceVMware.DataSourceVMware(
++ settings.CFG_BUILTIN, None, helpers.Paths({"run_dir": temp_dir})
++ )
++ ds.vmware_rpctool = "vmware-rpctool"
++ return ds
++
++
++# vi: ts=4 expandtab
+diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
+index 1d8aaf18..8617d7bd 100644
+--- a/tests/unittests/test_ds_identify.py
++++ b/tests/unittests/test_ds_identify.py
+@@ -649,6 +649,50 @@ class TestDsIdentify(DsIdentifyBase):
+ """EC2: bobrightbox.com in product_serial is not brightbox'"""
+ self._test_ds_not_found('Ec2-E24Cloud-negative')
+
++ def test_vmware_no_valid_transports(self):
++ """VMware: no valid transports"""
++ self._test_ds_not_found('VMware-NoValidTransports')
++
++ def test_vmware_envvar_no_data(self):
++ """VMware: envvar transport no data"""
++ self._test_ds_not_found('VMware-EnvVar-NoData')
++
++ def test_vmware_envvar_no_virt_id(self):
++ """VMware: envvar transport success if no virt id"""
++ self._test_ds_found('VMware-EnvVar-NoVirtID')
++
++ def test_vmware_envvar_activated_by_metadata(self):
++ """VMware: envvar transport activated by metadata"""
++ self._test_ds_found('VMware-EnvVar-Metadata')
++
++ def test_vmware_envvar_activated_by_userdata(self):
++ """VMware: envvar transport activated by userdata"""
++ self._test_ds_found('VMware-EnvVar-Userdata')
++
++ def test_vmware_envvar_activated_by_vendordata(self):
++ """VMware: envvar transport activated by vendordata"""
++ self._test_ds_found('VMware-EnvVar-Vendordata')
++
++ def test_vmware_guestinfo_no_data(self):
++ """VMware: guestinfo transport no data"""
++ self._test_ds_not_found('VMware-GuestInfo-NoData')
++
++ def test_vmware_guestinfo_no_virt_id(self):
++ """VMware: guestinfo transport fails if no virt id"""
++ self._test_ds_not_found('VMware-GuestInfo-NoVirtID')
++
++ def test_vmware_guestinfo_activated_by_metadata(self):
++ """VMware: guestinfo transport activated by metadata"""
++ self._test_ds_found('VMware-GuestInfo-Metadata')
++
++ def test_vmware_guestinfo_activated_by_userdata(self):
++ """VMware: guestinfo transport activated by userdata"""
++ self._test_ds_found('VMware-GuestInfo-Userdata')
++
++ def test_vmware_guestinfo_activated_by_vendordata(self):
++ """VMware: guestinfo transport activated by vendordata"""
++ self._test_ds_found('VMware-GuestInfo-Vendordata')
++
+
+ class TestBSDNoSys(DsIdentifyBase):
+ """Test *BSD code paths
+@@ -1136,7 +1180,240 @@ VALID_CFG = {
+ 'Ec2-E24Cloud-negative': {
+ 'ds': 'Ec2',
+ 'files': {P_SYS_VENDOR: 'e24cloudyday\n'},
+- }
++ },
++ 'VMware-NoValidTransports': {
++ 'ds': 'VMware',
++ 'mocks': [
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-EnvVar-NoData': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_metadata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-EnvVar-NoVirtID': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_metadata',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ ],
++ },
++ 'VMware-EnvVar-Metadata': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_metadata',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-EnvVar-Userdata': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_metadata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_userdata',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-EnvVar-Vendordata': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo',
++ 'ret': 0,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_metadata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_has_envvar_vmx_guestinfo_vendordata',
++ 'ret': 0,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-GuestInfo-NoData': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_rpctool',
++ 'ret': 0,
++ 'out': '/usr/bin/vmware-rpctool',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_metadata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-GuestInfo-NoVirtID': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_rpctool',
++ 'ret': 0,
++ 'out': '/usr/bin/vmware-rpctool',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_metadata',
++ 'ret': 0,
++ 'out': '---',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ ],
++ },
++ 'VMware-GuestInfo-Metadata': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_rpctool',
++ 'ret': 0,
++ 'out': '/usr/bin/vmware-rpctool',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_metadata',
++ 'ret': 0,
++ 'out': '---',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-GuestInfo-Userdata': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_rpctool',
++ 'ret': 0,
++ 'out': '/usr/bin/vmware-rpctool',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_metadata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_userdata',
++ 'ret': 0,
++ 'out': '---',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_vendordata',
++ 'ret': 1,
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
++ 'VMware-GuestInfo-Vendordata': {
++ 'ds': 'VMware',
++ 'mocks': [
++ {
++ 'name': 'vmware_has_rpctool',
++ 'ret': 0,
++ 'out': '/usr/bin/vmware-rpctool',
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_metadata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_userdata',
++ 'ret': 1,
++ },
++ {
++ 'name': 'vmware_rpctool_guestinfo_vendordata',
++ 'ret': 0,
++ 'out': '---',
++ },
++ MOCK_VIRT_IS_VMWARE,
++ ],
++ },
+ }
+
+ # vi: ts=4 expandtab
+diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
+index 689d7902..cbfa883c 100644
+--- a/tools/.github-cla-signers
++++ b/tools/.github-cla-signers
+@@ -1,5 +1,6 @@
+ ader1990
+ ajmyyra
++akutz
+ AlexBaranowski
+ Aman306
+ andrewbogott
+diff --git a/tools/ds-identify b/tools/ds-identify
+index 2f2486f7..c01eae3d 100755
+--- a/tools/ds-identify
++++ b/tools/ds-identify
+@@ -125,7 +125,7 @@ DI_DSNAME=""
+ # be searched if there is no setting found in config.
+ DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
+ CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
+-OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud"
++OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale RbxCloud UpCloud VMware"
+ DI_DSLIST=""
+ DI_MODE=""
+ DI_ON_FOUND=""
+@@ -1350,6 +1350,80 @@ dscheck_IBMCloud() {
+ return ${DS_NOT_FOUND}
+ }
+
++vmware_has_envvar_vmx_guestinfo() {
++ [ -n "${VMX_GUESTINFO:-}" ]
++}
++
++vmware_has_envvar_vmx_guestinfo_metadata() {
++ [ -n "${VMX_GUESTINFO_METADATA:-}" ]
++}
++
++vmware_has_envvar_vmx_guestinfo_userdata() {
++ [ -n "${VMX_GUESTINFO_USERDATA:-}" ]
++}
++
++vmware_has_envvar_vmx_guestinfo_vendordata() {
++ [ -n "${VMX_GUESTINFO_VENDORDATA:-}" ]
++}
++
++vmware_has_rpctool() {
++ command -v vmware-rpctool >/dev/null 2>&1
++}
++
++vmware_rpctool_guestinfo_metadata() {
++ vmware-rpctool "info-get guestinfo.metadata"
++}
++
++vmware_rpctool_guestinfo_userdata() {
++ vmware-rpctool "info-get guestinfo.userdata"
++}
++
++vmware_rpctool_guestinfo_vendordata() {
++ vmware-rpctool "info-get guestinfo.vendordata"
++}
++
++dscheck_VMware() {
++ # Checks to see if there is valid data for the VMware datasource.
++ # The data transports are checked in the following order:
++ #
++ # * envvars
++ # * guestinfo
++ #
++ # Please note when updating this function with support for new data
++ # transports, the order should match the order in the _get_data
++ # function from the file DataSourceVMware.py.
++
++ # Check to see if running in a container and the VMware
++ # datasource is configured via environment variables.
++ if vmware_has_envvar_vmx_guestinfo; then
++ if vmware_has_envvar_vmx_guestinfo_metadata || \
++ vmware_has_envvar_vmx_guestinfo_userdata || \
++ vmware_has_envvar_vmx_guestinfo_vendordata; then
++ return "${DS_FOUND}"
++ fi
++ fi
++
++ # Do not proceed unless the detected platform is VMware.
++ if [ ! "${DI_VIRT}" = "vmware" ]; then
++ return "${DS_NOT_FOUND}"
++ fi
++
++ # Do not proceed if the vmware-rpctool command is not present.
++ if ! vmware_has_rpctool; then
++ return "${DS_NOT_FOUND}"
++ fi
++
++ # Activate the VMware datasource only if any of the fields used
++ # by the datasource are present in the guestinfo table.
++ if { vmware_rpctool_guestinfo_metadata || \
++ vmware_rpctool_guestinfo_userdata || \
++ vmware_rpctool_guestinfo_vendordata; } >/dev/null 2>&1; then
++ return "${DS_FOUND}"
++ fi
++
++ return "${DS_NOT_FOUND}"
++}
++
+ collect_info() {
+ read_uname_info
+ read_virt
+--
+2.27.0
+
diff --git a/SOURCES/ci-Revert-unnecesary-lcase-in-ds-identify-978.patch b/SOURCES/ci-Revert-unnecesary-lcase-in-ds-identify-978.patch
new file mode 100644
index 0000000..8efdfa4
--- /dev/null
+++ b/SOURCES/ci-Revert-unnecesary-lcase-in-ds-identify-978.patch
@@ -0,0 +1,47 @@
+From 0aba80bf749458960945acf106833b098c3c5c97 Mon Sep 17 00:00:00 2001
+From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+Date: Fri, 14 Jan 2022 16:50:44 +0100
+Subject: [PATCH 4/5] Revert unnecesary lcase in ds-identify (#978)
+
+RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+RH-MergeRequest: 17: Datasource for VMware
+RH-Commit: [4/5] 334aae223b966173238a905150cf7bc07829c255 (eesposit/cloud-init-centos-)
+RH-Bugzilla: 2040090
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
+
+commit f516a7d37c1654addc02485e681b4358d7e7c0db
+Author: Andrew Kutz <101085+akutz@users.noreply.github.com>
+Date: Fri Aug 13 14:30:55 2021 -0500
+
+ Revert unnecesary lcase in ds-identify (#978)
+
+ This patch reverts an unnecessary lcase optimization in the
+ ds-identify script. SystemD documents the values produced by
+ the systemd-detect-virt command are lower case, and the mapping
+ table used by the FreeBSD check is also lower-case.
+
+ The optimization added two new forked processes, needlessly
+ causing overhead.
+
+Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+---
+ tools/ds-identify | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/tools/ds-identify b/tools/ds-identify
+index 0e12298f..7b782462 100755
+--- a/tools/ds-identify
++++ b/tools/ds-identify
+@@ -449,7 +449,7 @@ detect_virt() {
+ read_virt() {
+ cached "$DI_VIRT" && return 0
+ detect_virt
+- DI_VIRT="$(echo "${_RET}" | tr '[:upper:]' '[:lower:]')"
++ DI_VIRT="${_RET}"
+ }
+
+ is_container() {
+--
+2.27.0
+
diff --git a/SOURCES/ci-Update-dscheck_VMware-s-rpctool-check-970.patch b/SOURCES/ci-Update-dscheck_VMware-s-rpctool-check-970.patch
new file mode 100644
index 0000000..81deba6
--- /dev/null
+++ b/SOURCES/ci-Update-dscheck_VMware-s-rpctool-check-970.patch
@@ -0,0 +1,97 @@
+From f284c2925b7076b81afb9207161f01718ba70951 Mon Sep 17 00:00:00 2001
+From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+Date: Fri, 14 Jan 2022 16:50:18 +0100
+Subject: [PATCH 3/5] Update dscheck_VMware's rpctool check (#970)
+
+RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+RH-MergeRequest: 17: Datasource for VMware
+RH-Commit: [3/5] 0739bc18b46b8877fb3825d13f7cda57acda2dde (eesposit/cloud-init-centos-)
+RH-Bugzilla: 2040090
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
+
+commit 7781dec3306e9467f216cfcb36b7e10a8b38547a
+Author: Shreenidhi Shedi <53473811+sshedi@users.noreply.github.com>
+Date: Fri Aug 13 00:40:39 2021 +0530
+
+ Update dscheck_VMware's rpctool check (#970)
+
+ This patch updates the dscheck_VMware function's use of "vmware-rpctool".
+
+ When checking to see if a "guestinfo" property is set.
+ Because a successful exit code can occur even if there is an empty
+ string returned, it is possible that the VMware datasource will be
+ loaded as a false-positive. This patch ensures that in addition to
+ validating the exit code, the emitted output is also examined to ensure
+ a non-empty value is returned by rpctool before returning "${DS_FOUND}"
+ from "dscheck_VMware()".
+
+Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+---
+ tools/ds-identify | 15 +++++++++------
+ 1 file changed, 9 insertions(+), 6 deletions(-)
+
+diff --git a/tools/ds-identify b/tools/ds-identify
+index c01eae3d..0e12298f 100755
+--- a/tools/ds-identify
++++ b/tools/ds-identify
+@@ -141,6 +141,7 @@ error() {
+ debug 0 "$@"
+ stderr "$@"
+ }
++
+ warn() {
+ set -- "WARN:" "$@"
+ debug 0 "$@"
+@@ -344,7 +345,6 @@ geom_label_status_as() {
+ return $ret
+ }
+
+-
+ read_fs_info_freebsd() {
+ local oifs="$IFS" line="" delim=","
+ local ret=0 labels="" dev="" label="" ftype="" isodevs=""
+@@ -404,7 +404,6 @@ cached() {
+ [ -n "$1" ] && _RET="$1" && return || return 1
+ }
+
+-
+ detect_virt() {
+ local virt="${UNAVAILABLE}" r="" out=""
+ if [ -d /run/systemd ]; then
+@@ -450,7 +449,7 @@ detect_virt() {
+ read_virt() {
+ cached "$DI_VIRT" && return 0
+ detect_virt
+- DI_VIRT=${_RET}
++ DI_VIRT="$(echo "${_RET}" | tr '[:upper:]' '[:lower:]')"
+ }
+
+ is_container() {
+@@ -1370,16 +1369,20 @@ vmware_has_rpctool() {
+ command -v vmware-rpctool >/dev/null 2>&1
+ }
+
++vmware_rpctool_guestinfo() {
++ vmware-rpctool "info-get guestinfo.${1}" 2>/dev/null | grep "[[:alnum:]]"
++}
++
+ vmware_rpctool_guestinfo_metadata() {
+- vmware-rpctool "info-get guestinfo.metadata"
++ vmware_rpctool_guestinfo "metadata"
+ }
+
+ vmware_rpctool_guestinfo_userdata() {
+- vmware-rpctool "info-get guestinfo.userdata"
++ vmware_rpctool_guestinfo "userdata"
+ }
+
+ vmware_rpctool_guestinfo_vendordata() {
+- vmware-rpctool "info-get guestinfo.vendordata"
++ vmware_rpctool_guestinfo "vendordata"
+ }
+
+ dscheck_VMware() {
+--
+2.27.0
+
diff --git a/SOURCES/ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch b/SOURCES/ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch
new file mode 100644
index 0000000..17b2187
--- /dev/null
+++ b/SOURCES/ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch
@@ -0,0 +1,470 @@
+From 9ccb738cf078555b68122b1fc745a45fe952c439 Mon Sep 17 00:00:00 2001
+From: Anh Vo <anhvo@microsoft.com>
+Date: Tue, 13 Apr 2021 17:39:39 -0400
+Subject: [PATCH 3/7] azure: Removing ability to invoke walinuxagent (#799)
+
+RH-Author: Eduardo Otubo <otubo@redhat.com>
+RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
+RH-Commit: [3/7] 7431b912e3df7ea384820f45e0230b47ab54643c (otubo/cloud-init-src)
+RH-Bugzilla: 2042351
+RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
+RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
+
+Invoking walinuxagent from within cloud-init is no longer
+supported/necessary
+---
+ cloudinit/sources/DataSourceAzure.py | 137 ++++--------------
+ doc/rtd/topics/datasources/azure.rst | 62 ++------
+ tests/unittests/test_datasource/test_azure.py | 97 -------------
+ 3 files changed, 35 insertions(+), 261 deletions(-)
+
+diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
+index de1452ce..020b7006 100755
+--- a/cloudinit/sources/DataSourceAzure.py
++++ b/cloudinit/sources/DataSourceAzure.py
+@@ -381,53 +381,6 @@ class DataSourceAzure(sources.DataSource):
+ util.logexc(LOG, "handling set_hostname failed")
+ return False
+
+- @azure_ds_telemetry_reporter
+- def get_metadata_from_agent(self):
+- temp_hostname = self.metadata.get('local-hostname')
+- agent_cmd = self.ds_cfg['agent_command']
+- LOG.debug("Getting metadata via agent. hostname=%s cmd=%s",
+- temp_hostname, agent_cmd)
+-
+- self.bounce_network_with_azure_hostname()
+-
+- try:
+- invoke_agent(agent_cmd)
+- except subp.ProcessExecutionError:
+- # claim the datasource even if the command failed
+- util.logexc(LOG, "agent command '%s' failed.",
+- self.ds_cfg['agent_command'])
+-
+- ddir = self.ds_cfg['data_dir']
+-
+- fp_files = []
+- key_value = None
+- for pk in self.cfg.get('_pubkeys', []):
+- if pk.get('value', None):
+- key_value = pk['value']
+- LOG.debug("SSH authentication: using value from fabric")
+- else:
+- bname = str(pk['fingerprint'] + ".crt")
+- fp_files += [os.path.join(ddir, bname)]
+- LOG.debug("SSH authentication: "
+- "using fingerprint from fabric")
+-
+- with events.ReportEventStack(
+- name="waiting-for-ssh-public-key",
+- description="wait for agents to retrieve SSH keys",
+- parent=azure_ds_reporter):
+- # wait very long for public SSH keys to arrive
+- # https://bugs.launchpad.net/cloud-init/+bug/1717611
+- missing = util.log_time(logfunc=LOG.debug,
+- msg="waiting for SSH public key files",
+- func=util.wait_for_files,
+- args=(fp_files, 900))
+- if len(missing):
+- LOG.warning("Did not find files, but going on: %s", missing)
+-
+- metadata = {}
+- metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
+- return metadata
+-
+ def _get_subplatform(self):
+ """Return the subplatform metadata source details."""
+ if self.seed.startswith('/dev'):
+@@ -1354,35 +1307,32 @@ class DataSourceAzure(sources.DataSource):
+ On failure, returns False.
+ """
+
+- if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN:
+- self.bounce_network_with_azure_hostname()
++ self.bounce_network_with_azure_hostname()
+
+- pubkey_info = None
+- try:
+- raise KeyError(
+- "Not using public SSH keys from IMDS"
+- )
+- # pylint:disable=unreachable
+- public_keys = self.metadata['imds']['compute']['publicKeys']
+- LOG.debug(
+- 'Successfully retrieved %s key(s) from IMDS',
+- len(public_keys)
+- if public_keys is not None
+- else 0
+- )
+- except KeyError:
+- LOG.debug(
+- 'Unable to retrieve SSH keys from IMDS during '
+- 'negotiation, falling back to OVF'
+- )
+- pubkey_info = self.cfg.get('_pubkeys', None)
+-
+- metadata_func = partial(get_metadata_from_fabric,
+- fallback_lease_file=self.
+- dhclient_lease_file,
+- pubkey_info=pubkey_info)
+- else:
+- metadata_func = self.get_metadata_from_agent
++ pubkey_info = None
++ try:
++ raise KeyError(
++ "Not using public SSH keys from IMDS"
++ )
++ # pylint:disable=unreachable
++ public_keys = self.metadata['imds']['compute']['publicKeys']
++ LOG.debug(
++ 'Successfully retrieved %s key(s) from IMDS',
++ len(public_keys)
++ if public_keys is not None
++ else 0
++ )
++ except KeyError:
++ LOG.debug(
++ 'Unable to retrieve SSH keys from IMDS during '
++ 'negotiation, falling back to OVF'
++ )
++ pubkey_info = self.cfg.get('_pubkeys', None)
++
++ metadata_func = partial(get_metadata_from_fabric,
++ fallback_lease_file=self.
++ dhclient_lease_file,
++ pubkey_info=pubkey_info)
+
+ LOG.debug("negotiating with fabric via agent command %s",
+ self.ds_cfg['agent_command'])
+@@ -1617,33 +1567,6 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
+ return True
+
+
+-@azure_ds_telemetry_reporter
+-def crtfile_to_pubkey(fname, data=None):
+- pipeline = ('openssl x509 -noout -pubkey < "$0" |'
+- 'ssh-keygen -i -m PKCS8 -f /dev/stdin')
+- (out, _err) = subp.subp(['sh', '-c', pipeline, fname],
+- capture=True, data=data)
+- return out.rstrip()
+-
+-
+-@azure_ds_telemetry_reporter
+-def pubkeys_from_crt_files(flist):
+- pubkeys = []
+- errors = []
+- for fname in flist:
+- try:
+- pubkeys.append(crtfile_to_pubkey(fname))
+- except subp.ProcessExecutionError:
+- errors.append(fname)
+-
+- if errors:
+- report_diagnostic_event(
+- "failed to convert the crt files to pubkey: %s" % errors,
+- logger_func=LOG.warning)
+-
+- return pubkeys
+-
+-
+ @azure_ds_telemetry_reporter
+ def write_files(datadir, files, dirmode=None):
+
+@@ -1672,16 +1595,6 @@ def write_files(datadir, files, dirmode=None):
+ util.write_file(filename=fname, content=content, mode=0o600)
+
+
+-@azure_ds_telemetry_reporter
+-def invoke_agent(cmd):
+- # this is a function itself to simplify patching it for test
+- if cmd:
+- LOG.debug("invoking agent: %s", cmd)
+- subp.subp(cmd, shell=(not isinstance(cmd, list)))
+- else:
+- LOG.debug("not invoking agent")
+-
+-
+ def find_child(node, filter_func):
+ ret = []
+ if not node.hasChildNodes():
+diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst
+index e04c3a33..ad9f2236 100644
+--- a/doc/rtd/topics/datasources/azure.rst
++++ b/doc/rtd/topics/datasources/azure.rst
+@@ -5,28 +5,6 @@ Azure
+
+ This datasource finds metadata and user-data from the Azure cloud platform.
+
+-walinuxagent
+-------------
+-walinuxagent has several functions within images. For cloud-init
+-specifically, the relevant functionality it performs is to register the
+-instance with the Azure cloud platform at boot so networking will be
+-permitted. For more information about the other functionality of
+-walinuxagent, see `Azure's documentation
+-<https://github.com/Azure/WALinuxAgent#introduction>`_ for more details.
+-(Note, however, that only one of walinuxagent's provisioning and cloud-init
+-should be used to perform instance customisation.)
+-
+-If you are configuring walinuxagent yourself, you will want to ensure that you
+-have `Provisioning.UseCloudInit
+-<https://github.com/Azure/WALinuxAgent#provisioningusecloudinit>`_ set to
+-``y``.
+-
+-
+-Builtin Agent
+--------------
+-An alternative to using walinuxagent to register to the Azure cloud platform
+-is to use the ``__builtin__`` agent command. This section contains more
+-background on what that code path does, and how to enable it.
+
+ The Azure cloud platform provides initial data to an instance via an attached
+ CD formatted in UDF. That CD contains a 'ovf-env.xml' file that provides some
+@@ -41,16 +19,6 @@ by calling a script in /etc/dhcp/dhclient-exit-hooks or a file in
+ 'dhclient_hook' of cloud-init itself. This sub-command will write the client
+ information in json format to /run/cloud-init/dhclient.hook/<interface>.json.
+
+-In order for cloud-init to leverage this method to find the endpoint, the
+-cloud.cfg file must contain:
+-
+-.. sourcecode:: yaml
+-
+- datasource:
+- Azure:
+- set_hostname: False
+- agent_command: __builtin__
+-
+ If those files are not available, the fallback is to check the leases file
+ for the endpoint server (again option 245).
+
+@@ -83,9 +51,6 @@ configuration (in ``/etc/cloud/cloud.cfg`` or ``/etc/cloud/cloud.cfg.d/``).
+
+ The settings that may be configured are:
+
+- * **agent_command**: Either __builtin__ (default) or a command to run to getcw
+- metadata. If __builtin__, get metadata from walinuxagent. Otherwise run the
+- provided command to obtain metadata.
+ * **apply_network_config**: Boolean set to True to use network configuration
+ described by Azure's IMDS endpoint instead of fallback network config of
+ dhcp on eth0. Default is True. For Ubuntu 16.04 or earlier, default is
+@@ -121,7 +86,6 @@ An example configuration with the default values is provided below:
+
+ datasource:
+ Azure:
+- agent_command: __builtin__
+ apply_network_config: true
+ data_dir: /var/lib/waagent
+ dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases
+@@ -144,9 +108,7 @@ child of the ``LinuxProvisioningConfigurationSet`` (a sibling to ``UserName``)
+ If both ``UserData`` and ``CustomData`` are provided behavior is undefined on
+ which will be selected.
+
+-In the example below, user-data provided is 'this is my userdata', and the
+-datasource config provided is ``{"agent_command": ["start", "walinuxagent"]}``.
+-That agent command will take affect as if it were specified in system config.
++In the example below, user-data provided is 'this is my userdata'
+
+ Example:
+
+@@ -184,20 +146,16 @@ The hostname is provided to the instance in the ovf-env.xml file as
+ Whatever value the instance provides in its dhcp request will resolve in the
+ domain returned in the 'search' request.
+
+-The interesting issue is that a generic image will already have a hostname
+-configured. The ubuntu cloud images have 'ubuntu' as the hostname of the
+-system, and the initial dhcp request on eth0 is not guaranteed to occur after
+-the datasource code has been run. So, on first boot, that initial value will
+-be sent in the dhcp request and *that* value will resolve.
+-
+-In order to make the ``HostName`` provided in the ovf-env.xml resolve, a
+-dhcp request must be made with the new value. Walinuxagent (in its current
+-version) handles this by polling the state of hostname and bouncing ('``ifdown
+-eth0; ifup eth0``' the network interface if it sees that a change has been
+-made.
++A generic image will already have a hostname configured. The ubuntu
++cloud images have 'ubuntu' as the hostname of the system, and the
++initial dhcp request on eth0 is not guaranteed to occur after the
++datasource code has been run. So, on first boot, that initial value
++will be sent in the dhcp request and *that* value will resolve.
+
+-cloud-init handles this by setting the hostname in the DataSource's 'get_data'
+-method via '``hostname $HostName``', and then bouncing the interface. This
++In order to make the ``HostName`` provided in the ovf-env.xml resolve,
++a dhcp request must be made with the new value. cloud-init handles
++this by setting the hostname in the DataSource's 'get_data' method via
++'``hostname $HostName``', and then bouncing the interface. This
+ behavior can be configured or disabled in the datasource config. See
+ 'Configuration' above.
+
+diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
+index dedebeb1..320fa857 100644
+--- a/tests/unittests/test_datasource/test_azure.py
++++ b/tests/unittests/test_datasource/test_azure.py
+@@ -638,17 +638,10 @@ scbus-1 on xpt0 bus 0
+ def dsdevs():
+ return data.get('dsdevs', [])
+
+- def _invoke_agent(cmd):
+- data['agent_invoked'] = cmd
+-
+ def _wait_for_files(flist, _maxwait=None, _naplen=None):
+ data['waited'] = flist
+ return []
+
+- def _pubkeys_from_crt_files(flist):
+- data['pubkey_files'] = flist
+- return ["pubkey_from: %s" % f for f in flist]
+-
+ if data.get('ovfcontent') is not None:
+ populate_dir(os.path.join(self.paths.seed_dir, "azure"),
+ {'ovf-env.xml': data['ovfcontent']})
+@@ -675,8 +668,6 @@ scbus-1 on xpt0 bus 0
+
+ self.apply_patches([
+ (dsaz, 'list_possible_azure_ds_devs', dsdevs),
+- (dsaz, 'invoke_agent', _invoke_agent),
+- (dsaz, 'pubkeys_from_crt_files', _pubkeys_from_crt_files),
+ (dsaz, 'perform_hostname_bounce', mock.MagicMock()),
+ (dsaz, 'get_hostname', mock.MagicMock()),
+ (dsaz, 'set_hostname', mock.MagicMock()),
+@@ -765,7 +756,6 @@ scbus-1 on xpt0 bus 0
+ ret = dsrc.get_data()
+ self.m_is_platform_viable.assert_called_with(dsrc.seed_dir)
+ self.assertFalse(ret)
+- self.assertNotIn('agent_invoked', data)
+ # Assert that for non viable platforms,
+ # there is no communication with the Azure datasource.
+ self.assertEqual(
+@@ -789,7 +779,6 @@ scbus-1 on xpt0 bus 0
+ ret = dsrc.get_data()
+ self.m_is_platform_viable.assert_called_with(dsrc.seed_dir)
+ self.assertFalse(ret)
+- self.assertNotIn('agent_invoked', data)
+ self.assertEqual(
+ 1,
+ m_report_failure.call_count)
+@@ -806,7 +795,6 @@ scbus-1 on xpt0 bus 0
+ 1,
+ m_crawl_metadata.call_count)
+ self.assertFalse(ret)
+- self.assertNotIn('agent_invoked', data)
+
+ def test_crawl_metadata_exception_should_report_failure_with_msg(self):
+ data = {}
+@@ -1086,21 +1074,6 @@ scbus-1 on xpt0 bus 0
+ self.assertTrue(os.path.isdir(self.waagent_d))
+ self.assertEqual(stat.S_IMODE(os.stat(self.waagent_d).st_mode), 0o700)
+
+- def test_user_cfg_set_agent_command_plain(self):
+- # set dscfg in via plaintext
+- # we must have friendly-to-xml formatted plaintext in yaml_cfg
+- # not all plaintext is expected to work.
+- yaml_cfg = "{agent_command: my_command}\n"
+- cfg = yaml.safe_load(yaml_cfg)
+- odata = {'HostName': "myhost", 'UserName': "myuser",
+- 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}}
+- data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
+-
+- dsrc = self._get_ds(data)
+- ret = self._get_and_setup(dsrc)
+- self.assertTrue(ret)
+- self.assertEqual(data['agent_invoked'], cfg['agent_command'])
+-
+ @mock.patch('cloudinit.sources.DataSourceAzure.device_driver',
+ return_value=None)
+ def test_network_config_set_from_imds(self, m_driver):
+@@ -1205,29 +1178,6 @@ scbus-1 on xpt0 bus 0
+ dsrc.get_data()
+ self.assertEqual('eastus2', dsrc.region)
+
+- def test_user_cfg_set_agent_command(self):
+- # set dscfg in via base64 encoded yaml
+- cfg = {'agent_command': "my_command"}
+- odata = {'HostName': "myhost", 'UserName': "myuser",
+- 'dscfg': {'text': b64e(yaml.dump(cfg)),
+- 'encoding': 'base64'}}
+- data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
+-
+- dsrc = self._get_ds(data)
+- ret = self._get_and_setup(dsrc)
+- self.assertTrue(ret)
+- self.assertEqual(data['agent_invoked'], cfg['agent_command'])
+-
+- def test_sys_cfg_set_agent_command(self):
+- sys_cfg = {'datasource': {'Azure': {'agent_command': '_COMMAND'}}}
+- data = {'ovfcontent': construct_valid_ovf_env(data={}),
+- 'sys_cfg': sys_cfg}
+-
+- dsrc = self._get_ds(data)
+- ret = self._get_and_setup(dsrc)
+- self.assertTrue(ret)
+- self.assertEqual(data['agent_invoked'], '_COMMAND')
+-
+ def test_sys_cfg_set_never_destroy_ntfs(self):
+ sys_cfg = {'datasource': {'Azure': {
+ 'never_destroy_ntfs': 'user-supplied-value'}}}
+@@ -1311,51 +1261,6 @@ scbus-1 on xpt0 bus 0
+ self.assertTrue(ret)
+ self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8'))
+
+- def test_cfg_has_pubkeys_fingerprint(self):
+- odata = {'HostName': "myhost", 'UserName': "myuser"}
+- mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}]
+- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist]
+- data = {'ovfcontent': construct_valid_ovf_env(data=odata,
+- pubkeys=pubkeys)}
+-
+- dsrc = self._get_ds(data, agent_command=['not', '__builtin__'])
+- ret = self._get_and_setup(dsrc)
+- self.assertTrue(ret)
+- for mypk in mypklist:
+- self.assertIn(mypk, dsrc.cfg['_pubkeys'])
+- self.assertIn('pubkey_from', dsrc.metadata['public-keys'][-1])
+-
+- def test_cfg_has_pubkeys_value(self):
+- # make sure that provided key is used over fingerprint
+- odata = {'HostName': "myhost", 'UserName': "myuser"}
+- mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': 'value1'}]
+- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist]
+- data = {'ovfcontent': construct_valid_ovf_env(data=odata,
+- pubkeys=pubkeys)}
+-
+- dsrc = self._get_ds(data, agent_command=['not', '__builtin__'])
+- ret = self._get_and_setup(dsrc)
+- self.assertTrue(ret)
+-
+- for mypk in mypklist:
+- self.assertIn(mypk, dsrc.cfg['_pubkeys'])
+- self.assertIn(mypk['value'], dsrc.metadata['public-keys'])
+-
+- def test_cfg_has_no_fingerprint_has_value(self):
+- # test value is used when fingerprint not provided
+- odata = {'HostName': "myhost", 'UserName': "myuser"}
+- mypklist = [{'fingerprint': None, 'path': 'path1', 'value': 'value1'}]
+- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist]
+- data = {'ovfcontent': construct_valid_ovf_env(data=odata,
+- pubkeys=pubkeys)}
+-
+- dsrc = self._get_ds(data, agent_command=['not', '__builtin__'])
+- ret = self._get_and_setup(dsrc)
+- self.assertTrue(ret)
+-
+- for mypk in mypklist:
+- self.assertIn(mypk['value'], dsrc.metadata['public-keys'])
+-
+ def test_default_ephemeral_configs_ephemeral_exists(self):
+ # make sure the ephemeral configs are correct if disk present
+ odata = {}
+@@ -1919,8 +1824,6 @@ class TestAzureBounce(CiTestCase):
+ with_logs = True
+
+ def mock_out_azure_moving_parts(self):
+- self.patches.enter_context(
+- mock.patch.object(dsaz, 'invoke_agent'))
+ self.patches.enter_context(
+ mock.patch.object(dsaz.util, 'wait_for_files'))
+ self.patches.enter_context(
+--
+2.27.0
+
diff --git a/SPECS/cloud-init.spec b/SPECS/cloud-init.spec
index c2d87d9..c16a07c 100644
--- a/SPECS/cloud-init.spec
+++ b/SPECS/cloud-init.spec
@@ -1,6 +1,6 @@
Name: cloud-init
Version: 21.1
-Release: 15%{?dist}
+Release: 17%{?dist}
Summary: Cloud instance init scripts
License: ASL 2.0 or GPLv3
URL: http://launchpad.net/cloud-init
@@ -34,6 +34,28 @@ Patch13: ci-remove-unnecessary-EOF-string-in-disable-sshd-keygen.patch
Patch14: ci-fix-error-on-upgrade-caused-by-new-vendordata2-attri.patch
# For bz#2028031 - [RHEL-9] Above 19.2 of cloud-init fails to configure routes when configuring static and default routes to the same destination IP
Patch15: ci-cloudinit-net-handle-two-different-routes-for-the-sa.patch
+# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
+Patch16: ci-Datasource-for-VMware-953.patch
+# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
+Patch17: ci-Change-netifaces-dependency-to-0.10.4-965.patch
+# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
+Patch18: ci-Update-dscheck_VMware-s-rpctool-check-970.patch
+# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
+Patch19: ci-Revert-unnecesary-lcase-in-ds-identify-978.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch20: ci-Add-flexibility-to-IMDS-api-version-793.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch21: ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch22: ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch23: ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch24: ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch25: ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch
+# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
+Patch26: ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch
# Source-git patches
@@ -86,6 +108,7 @@ Requires: dhcp-client
# https://bugzilla.redhat.com/show_bug.cgi?id=2032524
Requires: gdisk
Requires: openssl
+Requires: python3-netifaces
%{?systemd_requires}
@@ -234,6 +257,26 @@ fi
%config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf
%changelog
+* Mon Feb 07 2022 Miroslav Rezanina <mrezanin@redhat.com> - 21.1-17
+- ci-Add-flexibility-to-IMDS-api-version-793.patch [bz#2042351]
+- ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch [bz#2042351]
+- ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch [bz#2042351]
+- ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch [bz#2042351]
+- ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch [bz#2042351]
+- ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch [bz#2042351]
+- ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch [bz#2042351]
+- Resolves: bz#2042351
+ ([RHEL-9] Support for provisioning Azure VM with userdata)
+
+* Fri Jan 21 2022 Miroslav Rezanina <mrezanin@redhat.com> - 21.1-16
+- ci-Datasource-for-VMware-953.patch [bz#2040090]
+- ci-Change-netifaces-dependency-to-0.10.4-965.patch [bz#2040090]
+- ci-Update-dscheck_VMware-s-rpctool-check-970.patch [bz#2040090]
+- ci-Revert-unnecesary-lcase-in-ds-identify-978.patch [bz#2040090]
+- ci-Add-netifaces-package-as-a-Requires-in-cloud-init.sp.patch [bz#2040090]
+- Resolves: bz#2040090
+ ([cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo')
+
* Thu Jan 13 2022 Miroslav Rezanina <mrezanin@redhat.com> - 21.1-15
- ci-Add-gdisk-and-openssl-as-deps-to-fix-UEFI-Azure-init.patch [bz#2032524]
- Resolves: bz#2032524