faf1e5
From 68b3718124b63fdf0c077452b559f0fccb01200d Mon Sep 17 00:00:00 2001
faf1e5
From: Eduardo Otubo <otubo@redhat.com>
faf1e5
Date: Tue, 5 May 2020 08:08:32 +0200
faf1e5
Subject: [PATCH 5/5] ec2: Add support for AWS IMDS v2 (session-oriented) (#55)
faf1e5
faf1e5
RH-Author: Eduardo Otubo <otubo@redhat.com>
faf1e5
Message-id: <20200504085238.25884-6-otubo@redhat.com>
faf1e5
Patchwork-id: 96245
faf1e5
O-Subject: [RHEL-7.8.z cloud-init PATCH 5/5] ec2: Add support for AWS IMDS v2 (session-oriented) (#55)
faf1e5
Bugzilla: 1827207
faf1e5
RH-Acked-by: Cathy Avery <cavery@redhat.com>
faf1e5
RH-Acked-by: Mohammed Gamal <mgamal@redhat.com>
faf1e5
RH-Acked-by: Vitaly Kuznetsov <vkuznets@redhat.com>
faf1e5
faf1e5
commit 4bc399e0cd0b7e9177f948aecd49f6b8323ff30b
faf1e5
Author: Ryan Harper <ryan.harper@canonical.com>
faf1e5
Date:   Fri Nov 22 21:05:44 2019 -0600
faf1e5
faf1e5
    ec2: Add support for AWS IMDS v2 (session-oriented) (#55)
faf1e5
faf1e5
    * ec2: Add support for AWS IMDS v2 (session-oriented)
faf1e5
faf1e5
    AWS now supports a new version of fetching Instance Metadata[1].
faf1e5
faf1e5
    Update cloud-init's ec2 utility functions and update ec2 derived
faf1e5
    datasources accordingly.  For DataSourceEc2 (versus ec2-look-alikes)
faf1e5
    cloud-init will issue the PUT request to obtain an API token for
faf1e5
    the maximum lifetime and then all subsequent interactions with the
faf1e5
    IMDS will include the token in the header.
faf1e5
faf1e5
    If the API token endpoint is unreachable on Ec2 platform, log a
faf1e5
    warning and fallback to using IMDS v1 and which does not use
faf1e5
    session tokens when communicating with the Instance metadata
faf1e5
    service.
faf1e5
faf1e5
    We handle read errors, typically seen if the IMDS is beyond one
faf1e5
    etwork hop (IMDSv2 responses have a ttl=1), by setting the api token
faf1e5
    to a disabled value and then using IMDSv1 paths.
faf1e5
faf1e5
    To support token-based headers, ec2_utils functions were updated
faf1e5
    to support custom headers_cb and exception_cb callback functions
faf1e5
    so Ec2 could store, or refresh API tokens in the event of token
faf1e5
    becoming stale.
faf1e5
faf1e5
    [1] https://docs.aws.amazon.com/AWSEC2/latest/ \
faf1e5
    UserGuide/ec2-instance-metadata.html \
faf1e5
    #instance-metadata-v2-how-it-works
faf1e5
faf1e5
Signed-off-by: Eduardo Otubo <otubo@redhat.com>
faf1e5
Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>
faf1e5
---
faf1e5
 cloudinit/ec2_utils.py                             |  37 +++--
faf1e5
 cloudinit/sources/DataSourceCloudStack.py          |   2 +-
faf1e5
 cloudinit/sources/DataSourceEc2.py                 | 166 ++++++++++++++++++---
faf1e5
 cloudinit/sources/DataSourceExoscale.py            |   2 +-
faf1e5
 cloudinit/sources/DataSourceMAAS.py                |   2 +-
faf1e5
 cloudinit/sources/DataSourceOpenStack.py           |   2 +-
faf1e5
 cloudinit/url_helper.py                            |  15 +-
faf1e5
 tests/unittests/test_datasource/test_cloudstack.py |  21 ++-
faf1e5
 tests/unittests/test_datasource/test_ec2.py        |   6 +-
faf1e5
 9 files changed, 201 insertions(+), 52 deletions(-)
faf1e5
faf1e5
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
faf1e5
index 3b7b17f..57708c1 100644
faf1e5
--- a/cloudinit/ec2_utils.py
faf1e5
+++ b/cloudinit/ec2_utils.py
faf1e5
@@ -134,25 +134,28 @@ class MetadataMaterializer(object):
faf1e5
         return joined
faf1e5
 
faf1e5
 
faf1e5
-def _skip_retry_on_codes(status_codes, _request_args, cause):
faf1e5
+def skip_retry_on_codes(status_codes, _request_args, cause):
faf1e5
     """Returns False if cause.code is in status_codes."""
faf1e5
     return cause.code not in status_codes
faf1e5
 
faf1e5
 
faf1e5
 def get_instance_userdata(api_version='latest',
faf1e5
                           metadata_address='http://169.254.169.254',
faf1e5
-                          ssl_details=None, timeout=5, retries=5):
faf1e5
+                          ssl_details=None, timeout=5, retries=5,
faf1e5
+                          headers_cb=None, exception_cb=None):
faf1e5
     ud_url = url_helper.combine_url(metadata_address, api_version)
faf1e5
     ud_url = url_helper.combine_url(ud_url, 'user-data')
faf1e5
     user_data = ''
faf1e5
     try:
faf1e5
-        # It is ok for userdata to not exist (thats why we are stopping if
faf1e5
-        # NOT_FOUND occurs) and just in that case returning an empty string.
faf1e5
-        exception_cb = functools.partial(_skip_retry_on_codes,
faf1e5
-                                         SKIP_USERDATA_CODES)
faf1e5
+        if not exception_cb:
faf1e5
+            # It is ok for userdata to not exist (thats why we are stopping if
faf1e5
+            # NOT_FOUND occurs) and just in that case returning an empty
faf1e5
+            # string.
faf1e5
+            exception_cb = functools.partial(skip_retry_on_codes,
faf1e5
+                                             SKIP_USERDATA_CODES)
faf1e5
         response = url_helper.read_file_or_url(
faf1e5
             ud_url, ssl_details=ssl_details, timeout=timeout,
faf1e5
-            retries=retries, exception_cb=exception_cb)
faf1e5
+            retries=retries, exception_cb=exception_cb, headers_cb=headers_cb)
faf1e5
         user_data = response.contents
faf1e5
     except url_helper.UrlError as e:
faf1e5
         if e.code not in SKIP_USERDATA_CODES:
faf1e5
@@ -165,11 +168,13 @@ def get_instance_userdata(api_version='latest',
faf1e5
 def _get_instance_metadata(tree, api_version='latest',
faf1e5
                            metadata_address='http://169.254.169.254',
faf1e5
                            ssl_details=None, timeout=5, retries=5,
faf1e5
-                           leaf_decoder=None):
faf1e5
+                           leaf_decoder=None, headers_cb=None,
faf1e5
+                           exception_cb=None):
faf1e5
     md_url = url_helper.combine_url(metadata_address, api_version, tree)
faf1e5
     caller = functools.partial(
faf1e5
         url_helper.read_file_or_url, ssl_details=ssl_details,
faf1e5
-        timeout=timeout, retries=retries)
faf1e5
+        timeout=timeout, retries=retries, headers_cb=headers_cb,
faf1e5
+        exception_cb=exception_cb)
faf1e5
 
faf1e5
     def mcaller(url):
faf1e5
         return caller(url).contents
faf1e5
@@ -191,22 +196,28 @@ def _get_instance_metadata(tree, api_version='latest',
faf1e5
 def get_instance_metadata(api_version='latest',
faf1e5
                           metadata_address='http://169.254.169.254',
faf1e5
                           ssl_details=None, timeout=5, retries=5,
faf1e5
-                          leaf_decoder=None):
faf1e5
+                          leaf_decoder=None, headers_cb=None,
faf1e5
+                          exception_cb=None):
faf1e5
     # Note, 'meta-data' explicitly has trailing /.
faf1e5
     # this is required for CloudStack (LP: #1356855)
faf1e5
     return _get_instance_metadata(tree='meta-data/', api_version=api_version,
faf1e5
                                   metadata_address=metadata_address,
faf1e5
                                   ssl_details=ssl_details, timeout=timeout,
faf1e5
-                                  retries=retries, leaf_decoder=leaf_decoder)
faf1e5
+                                  retries=retries, leaf_decoder=leaf_decoder,
faf1e5
+                                  headers_cb=headers_cb,
faf1e5
+                                  exception_cb=exception_cb)
faf1e5
 
faf1e5
 
faf1e5
 def get_instance_identity(api_version='latest',
faf1e5
                           metadata_address='http://169.254.169.254',
faf1e5
                           ssl_details=None, timeout=5, retries=5,
faf1e5
-                          leaf_decoder=None):
faf1e5
+                          leaf_decoder=None, headers_cb=None,
faf1e5
+                          exception_cb=None):
faf1e5
     return _get_instance_metadata(tree='dynamic/instance-identity',
faf1e5
                                   api_version=api_version,
faf1e5
                                   metadata_address=metadata_address,
faf1e5
                                   ssl_details=ssl_details, timeout=timeout,
faf1e5
-                                  retries=retries, leaf_decoder=leaf_decoder)
faf1e5
+                                  retries=retries, leaf_decoder=leaf_decoder,
faf1e5
+                                  headers_cb=headers_cb,
faf1e5
+                                  exception_cb=exception_cb)
faf1e5
 # vi: ts=4 expandtab
faf1e5
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
faf1e5
index d4b758f..6bd2efe 100644
faf1e5
--- a/cloudinit/sources/DataSourceCloudStack.py
faf1e5
+++ b/cloudinit/sources/DataSourceCloudStack.py
faf1e5
@@ -93,7 +93,7 @@ class DataSourceCloudStack(sources.DataSource):
faf1e5
         urls = [uhelp.combine_url(self.metadata_address,
faf1e5
                                   'latest/meta-data/instance-id')]
faf1e5
         start_time = time.time()
faf1e5
-        url = uhelp.wait_for_url(
faf1e5
+        url, _response = uhelp.wait_for_url(
faf1e5
             urls=urls, max_wait=url_params.max_wait_seconds,
faf1e5
             timeout=url_params.timeout_seconds, status_cb=LOG.warn)
faf1e5
 
faf1e5
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
faf1e5
index 9ccf2cd..fbe8f3f 100644
faf1e5
--- a/cloudinit/sources/DataSourceEc2.py
faf1e5
+++ b/cloudinit/sources/DataSourceEc2.py
faf1e5
@@ -27,6 +27,10 @@ SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND])
faf1e5
 STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
faf1e5
 STRICT_ID_DEFAULT = "warn"
faf1e5
 
faf1e5
+API_TOKEN_ROUTE = 'latest/api/token'
faf1e5
+API_TOKEN_DISABLED = '_ec2_disable_api_token'
faf1e5
+AWS_TOKEN_TTL_SECONDS = '21600'
faf1e5
+
faf1e5
 
faf1e5
 class CloudNames(object):
faf1e5
     ALIYUN = "aliyun"
faf1e5
@@ -59,6 +63,7 @@ class DataSourceEc2(sources.DataSource):
faf1e5
     url_max_wait = 120
faf1e5
     url_timeout = 50
faf1e5
 
faf1e5
+    _api_token = None  # API token for accessing the metadata service
faf1e5
     _network_config = sources.UNSET  # Used to cache calculated network cfg v1
faf1e5
 
faf1e5
     # Whether we want to get network configuration from the metadata service.
faf1e5
@@ -132,11 +137,12 @@ class DataSourceEc2(sources.DataSource):
faf1e5
         min_metadata_version.
faf1e5
         """
faf1e5
         # Assumes metadata service is already up
faf1e5
+        url_tmpl = '{0}/{1}/meta-data/instance-id'
faf1e5
+        headers = self._get_headers()
faf1e5
         for api_ver in self.extended_metadata_versions:
faf1e5
-            url = '{0}/{1}/meta-data/instance-id'.format(
faf1e5
-                self.metadata_address, api_ver)
faf1e5
+            url = url_tmpl.format(self.metadata_address, api_ver)
faf1e5
             try:
faf1e5
-                resp = uhelp.readurl(url=url)
faf1e5
+                resp = uhelp.readurl(url=url, headers=headers)
faf1e5
             except uhelp.UrlError as e:
faf1e5
                 LOG.debug('url %s raised exception %s', url, e)
faf1e5
             else:
faf1e5
@@ -156,12 +162,39 @@ class DataSourceEc2(sources.DataSource):
faf1e5
                 # setup self.identity. So we need to do that now.
faf1e5
                 api_version = self.get_metadata_api_version()
faf1e5
                 self.identity = ec2.get_instance_identity(
faf1e5
-                    api_version, self.metadata_address).get('document', {})
faf1e5
+                    api_version, self.metadata_address,
faf1e5
+                    headers_cb=self._get_headers,
faf1e5
+                    exception_cb=self._refresh_stale_aws_token_cb).get(
faf1e5
+                        'document', {})
faf1e5
             return self.identity.get(
faf1e5
                 'instanceId', self.metadata['instance-id'])
faf1e5
         else:
faf1e5
             return self.metadata['instance-id']
faf1e5
 
faf1e5
+    def _maybe_fetch_api_token(self, mdurls, timeout=None, max_wait=None):
faf1e5
+        if self.cloud_name != CloudNames.AWS:
faf1e5
+            return
faf1e5
+
faf1e5
+        urls = []
faf1e5
+        url2base = {}
faf1e5
+        url_path = API_TOKEN_ROUTE
faf1e5
+        request_method = 'PUT'
faf1e5
+        for url in mdurls:
faf1e5
+            cur = '{0}/{1}'.format(url, url_path)
faf1e5
+            urls.append(cur)
faf1e5
+            url2base[cur] = url
faf1e5
+
faf1e5
+        # use the self._status_cb to check for Read errors, which means
faf1e5
+        # we can't reach the API token URL, so we should disable IMDSv2
faf1e5
+        LOG.debug('Fetching Ec2 IMDSv2 API Token')
faf1e5
+        url, response = uhelp.wait_for_url(
faf1e5
+            urls=urls, max_wait=1, timeout=1, status_cb=self._status_cb,
faf1e5
+            headers_cb=self._get_headers, request_method=request_method)
faf1e5
+
faf1e5
+        if url and response:
faf1e5
+            self._api_token = response
faf1e5
+            return url2base[url]
faf1e5
+
faf1e5
     def wait_for_metadata_service(self):
faf1e5
         mcfg = self.ds_cfg
faf1e5
 
faf1e5
@@ -183,27 +216,39 @@ class DataSourceEc2(sources.DataSource):
faf1e5
             LOG.warning("Empty metadata url list! using default list")
faf1e5
             mdurls = self.metadata_urls
faf1e5
 
faf1e5
-        urls = []
faf1e5
-        url2base = {}
faf1e5
-        for url in mdurls:
faf1e5
-            cur = '{0}/{1}/meta-data/instance-id'.format(
faf1e5
-                url, self.min_metadata_version)
faf1e5
-            urls.append(cur)
faf1e5
-            url2base[cur] = url
faf1e5
-
faf1e5
-        start_time = time.time()
faf1e5
-        url = uhelp.wait_for_url(
faf1e5
-            urls=urls, max_wait=url_params.max_wait_seconds,
faf1e5
-            timeout=url_params.timeout_seconds, status_cb=LOG.warn)
faf1e5
-
faf1e5
-        if url:
faf1e5
-            self.metadata_address = url2base[url]
faf1e5
+        # try the api token path first
faf1e5
+        metadata_address = self._maybe_fetch_api_token(mdurls)
faf1e5
+        if not metadata_address:
faf1e5
+            if self._api_token == API_TOKEN_DISABLED:
faf1e5
+                LOG.warning('Retrying with IMDSv1')
faf1e5
+            # if we can't get a token, use instance-id path
faf1e5
+            urls = []
faf1e5
+            url2base = {}
faf1e5
+            url_path = '{ver}/meta-data/instance-id'.format(
faf1e5
+                ver=self.min_metadata_version)
faf1e5
+            request_method = 'GET'
faf1e5
+            for url in mdurls:
faf1e5
+                cur = '{0}/{1}'.format(url, url_path)
faf1e5
+                urls.append(cur)
faf1e5
+                url2base[cur] = url
faf1e5
+
faf1e5
+            start_time = time.time()
faf1e5
+            url, _ = uhelp.wait_for_url(
faf1e5
+                urls=urls, max_wait=url_params.max_wait_seconds,
faf1e5
+                timeout=url_params.timeout_seconds, status_cb=LOG.warning,
faf1e5
+                headers_cb=self._get_headers, request_method=request_method)
faf1e5
+
faf1e5
+            if url:
faf1e5
+                metadata_address = url2base[url]
faf1e5
+
faf1e5
+        if metadata_address:
faf1e5
+            self.metadata_address = metadata_address
faf1e5
             LOG.debug("Using metadata source: '%s'", self.metadata_address)
faf1e5
         else:
faf1e5
             LOG.critical("Giving up on md from %s after %s seconds",
faf1e5
                          urls, int(time.time() - start_time))
faf1e5
 
faf1e5
-        return bool(url)
faf1e5
+        return bool(metadata_address)
faf1e5
 
faf1e5
     def device_name_to_device(self, name):
faf1e5
         # Consult metadata service, that has
faf1e5
@@ -349,14 +394,22 @@ class DataSourceEc2(sources.DataSource):
faf1e5
             return {}
faf1e5
         api_version = self.get_metadata_api_version()
faf1e5
         crawled_metadata = {}
faf1e5
+        if self.cloud_name == CloudNames.AWS:
faf1e5
+            exc_cb = self._refresh_stale_aws_token_cb
faf1e5
+            exc_cb_ud = self._skip_or_refresh_stale_aws_token_cb
faf1e5
+        else:
faf1e5
+            exc_cb = exc_cb_ud = None
faf1e5
         try:
faf1e5
             crawled_metadata['user-data'] = ec2.get_instance_userdata(
faf1e5
-                api_version, self.metadata_address)
faf1e5
+                api_version, self.metadata_address,
faf1e5
+                headers_cb=self._get_headers, exception_cb=exc_cb_ud)
faf1e5
             crawled_metadata['meta-data'] = ec2.get_instance_metadata(
faf1e5
-                api_version, self.metadata_address)
faf1e5
+                api_version, self.metadata_address,
faf1e5
+                headers_cb=self._get_headers, exception_cb=exc_cb)
faf1e5
             if self.cloud_name == CloudNames.AWS:
faf1e5
                 identity = ec2.get_instance_identity(
faf1e5
-                    api_version, self.metadata_address)
faf1e5
+                    api_version, self.metadata_address,
faf1e5
+                    headers_cb=self._get_headers, exception_cb=exc_cb)
faf1e5
                 crawled_metadata['dynamic'] = {'instance-identity': identity}
faf1e5
         except Exception:
faf1e5
             util.logexc(
faf1e5
@@ -366,6 +419,73 @@ class DataSourceEc2(sources.DataSource):
faf1e5
         crawled_metadata['_metadata_api_version'] = api_version
faf1e5
         return crawled_metadata
faf1e5
 
faf1e5
+    def _refresh_api_token(self, seconds=AWS_TOKEN_TTL_SECONDS):
faf1e5
+        """Request new metadata API token.
faf1e5
+        @param seconds: The lifetime of the token in seconds
faf1e5
+
faf1e5
+        @return: The API token or None if unavailable.
faf1e5
+        """
faf1e5
+        if self.cloud_name != CloudNames.AWS:
faf1e5
+            return None
faf1e5
+        LOG.debug("Refreshing Ec2 metadata API token")
faf1e5
+        request_header = {'X-aws-ec2-metadata-token-ttl-seconds': seconds}
faf1e5
+        token_url = '{}/{}'.format(self.metadata_address, API_TOKEN_ROUTE)
faf1e5
+        try:
faf1e5
+            response = uhelp.readurl(
faf1e5
+                token_url, headers=request_header, request_method="PUT")
faf1e5
+        except uhelp.UrlError as e:
faf1e5
+            LOG.warning(
faf1e5
+                'Unable to get API token: %s raised exception %s',
faf1e5
+                token_url, e)
faf1e5
+            return None
faf1e5
+        return response.contents
faf1e5
+
faf1e5
+    def _skip_or_refresh_stale_aws_token_cb(self, msg, exception):
faf1e5
+        """Callback will not retry on SKIP_USERDATA_CODES or if no token
faf1e5
+           is available."""
faf1e5
+        retry = ec2.skip_retry_on_codes(
faf1e5
+            ec2.SKIP_USERDATA_CODES, msg, exception)
faf1e5
+        if not retry:
faf1e5
+            return False  # False raises exception
faf1e5
+        return self._refresh_stale_aws_token_cb(msg, exception)
faf1e5
+
faf1e5
+    def _refresh_stale_aws_token_cb(self, msg, exception):
faf1e5
+        """Exception handler for Ec2 to refresh token if token is stale."""
faf1e5
+        if isinstance(exception, uhelp.UrlError) and exception.code == 401:
faf1e5
+            # With _api_token as None, _get_headers will _refresh_api_token.
faf1e5
+            LOG.debug("Clearing cached Ec2 API token due to expiry")
faf1e5
+            self._api_token = None
faf1e5
+        return True  # always retry
faf1e5
+
faf1e5
+    def _status_cb(self, msg, exc=None):
faf1e5
+        LOG.warning(msg)
faf1e5
+        if 'Read timed out' in msg:
faf1e5
+            LOG.warning('Cannot use Ec2 IMDSv2 API tokens, using IMDSv1')
faf1e5
+            self._api_token = API_TOKEN_DISABLED
faf1e5
+
faf1e5
+    def _get_headers(self, url=''):
faf1e5
+        """Return a dict of headers for accessing a url.
faf1e5
+
faf1e5
+        If _api_token is unset on AWS, attempt to refresh the token via a PUT
faf1e5
+        and then return the updated token header.
faf1e5
+        """
faf1e5
+        if self.cloud_name != CloudNames.AWS or (self._api_token ==
faf1e5
+                                                 API_TOKEN_DISABLED):
faf1e5
+            return {}
faf1e5
+        # Request a 6 hour token if URL is API_TOKEN_ROUTE
faf1e5
+        request_token_header = {
faf1e5
+            'X-aws-ec2-metadata-token-ttl-seconds': AWS_TOKEN_TTL_SECONDS}
faf1e5
+        if API_TOKEN_ROUTE in url:
faf1e5
+            return request_token_header
faf1e5
+        if not self._api_token:
faf1e5
+            # If we don't yet have an API token, get one via a PUT against
faf1e5
+            # API_TOKEN_ROUTE. This _api_token may get unset by a 403 due
faf1e5
+            # to an invalid or expired token
faf1e5
+            self._api_token = self._refresh_api_token()
faf1e5
+            if not self._api_token:
faf1e5
+                return {}
faf1e5
+        return {'X-aws-ec2-metadata-token': self._api_token}
faf1e5
+
faf1e5
 
faf1e5
 class DataSourceEc2Local(DataSourceEc2):
faf1e5
     """Datasource run at init-local which sets up network to query metadata.
faf1e5
diff --git a/cloudinit/sources/DataSourceExoscale.py b/cloudinit/sources/DataSourceExoscale.py
faf1e5
index 4616daa..d59aefd 100644
faf1e5
--- a/cloudinit/sources/DataSourceExoscale.py
faf1e5
+++ b/cloudinit/sources/DataSourceExoscale.py
faf1e5
@@ -61,7 +61,7 @@ class DataSourceExoscale(sources.DataSource):
faf1e5
         metadata_url = "{}/{}/meta-data/instance-id".format(
faf1e5
             self.metadata_url, self.api_version)
faf1e5
 
faf1e5
-        url = url_helper.wait_for_url(
faf1e5
+        url, _response = url_helper.wait_for_url(
faf1e5
             urls=[metadata_url],
faf1e5
             max_wait=self.url_max_wait,
faf1e5
             timeout=self.url_timeout,
faf1e5
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
faf1e5
index 61aa6d7..517913a 100644
faf1e5
--- a/cloudinit/sources/DataSourceMAAS.py
faf1e5
+++ b/cloudinit/sources/DataSourceMAAS.py
faf1e5
@@ -136,7 +136,7 @@ class DataSourceMAAS(sources.DataSource):
faf1e5
             url = url[:-1]
faf1e5
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
faf1e5
         urls = [check_url]
faf1e5
-        url = self.oauth_helper.wait_for_url(
faf1e5
+        url, _response = self.oauth_helper.wait_for_url(
faf1e5
             urls=urls, max_wait=max_wait, timeout=timeout)
faf1e5
 
faf1e5
         if url:
faf1e5
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
faf1e5
index 4a01524..7a5e71b 100644
faf1e5
--- a/cloudinit/sources/DataSourceOpenStack.py
faf1e5
+++ b/cloudinit/sources/DataSourceOpenStack.py
faf1e5
@@ -76,7 +76,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
faf1e5
 
faf1e5
         url_params = self.get_url_params()
faf1e5
         start_time = time.time()
faf1e5
-        avail_url = url_helper.wait_for_url(
faf1e5
+        avail_url, _response = url_helper.wait_for_url(
faf1e5
             urls=md_urls, max_wait=url_params.max_wait_seconds,
faf1e5
             timeout=url_params.timeout_seconds)
faf1e5
         if avail_url:
faf1e5
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
faf1e5
index 1b0721b..a951b8b 100644
faf1e5
--- a/cloudinit/url_helper.py
faf1e5
+++ b/cloudinit/url_helper.py
faf1e5
@@ -101,7 +101,7 @@ def read_file_or_url(url, timeout=5, retries=10,
faf1e5
             raise UrlError(cause=e, code=code, headers=None, url=url)
faf1e5
         return FileResponse(file_path, contents=contents)
faf1e5
     else:
faf1e5
-        return readurl(url, timeout=timeout, retries=retries, headers=headers,
faf1e5
+        return readurl(url, timeout=timeout, retries=retries,
faf1e5
                        headers_cb=headers_cb, data=data,
faf1e5
                        sec_between=sec_between, ssl_details=ssl_details,
faf1e5
                        exception_cb=exception_cb)
faf1e5
@@ -310,7 +310,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
faf1e5
 
faf1e5
 def wait_for_url(urls, max_wait=None, timeout=None,
faf1e5
                  status_cb=None, headers_cb=None, sleep_time=1,
faf1e5
-                 exception_cb=None, sleep_time_cb=None):
faf1e5
+                 exception_cb=None, sleep_time_cb=None, request_method=None):
faf1e5
     """
faf1e5
     urls:      a list of urls to try
faf1e5
     max_wait:  roughly the maximum time to wait before giving up
faf1e5
@@ -325,6 +325,8 @@ def wait_for_url(urls, max_wait=None, timeout=None,
faf1e5
                   'exception', the exception that occurred.
faf1e5
     sleep_time_cb: call method with 2 arguments (response, loop_n) that
faf1e5
                    generates the next sleep time.
faf1e5
+    request_method: indicate the type of HTTP request, GET, PUT, or POST
faf1e5
+    returns: tuple of (url, response contents), on failure, (False, None)
faf1e5
 
faf1e5
     the idea of this routine is to wait for the EC2 metdata service to
faf1e5
     come up.  On both Eucalyptus and EC2 we have seen the case where
faf1e5
@@ -381,8 +383,9 @@ def wait_for_url(urls, max_wait=None, timeout=None,
faf1e5
                 else:
faf1e5
                     headers = {}
faf1e5
 
faf1e5
-                response = readurl(url, headers=headers, timeout=timeout,
faf1e5
-                                   check_status=False)
faf1e5
+                response = readurl(
faf1e5
+                    url, headers=headers, timeout=timeout,
faf1e5
+                    check_status=False, request_method=request_method)
faf1e5
                 if not response.contents:
faf1e5
                     reason = "empty response [%s]" % (response.code)
faf1e5
                     url_exc = UrlError(ValueError(reason), code=response.code,
faf1e5
@@ -392,7 +395,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
faf1e5
                     url_exc = UrlError(ValueError(reason), code=response.code,
faf1e5
                                        headers=response.headers, url=url)
faf1e5
                 else:
faf1e5
-                    return url
faf1e5
+                    return url, response.contents
faf1e5
             except UrlError as e:
faf1e5
                 reason = "request error [%s]" % e
faf1e5
                 url_exc = e
faf1e5
@@ -421,7 +424,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
faf1e5
                   sleep_time)
faf1e5
         time.sleep(sleep_time)
faf1e5
 
faf1e5
-    return False
faf1e5
+    return False, None
faf1e5
 
faf1e5
 
faf1e5
 class OauthUrlHelper(object):
faf1e5
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
faf1e5
index d6d2d6b..83c2f75 100644
faf1e5
--- a/tests/unittests/test_datasource/test_cloudstack.py
faf1e5
+++ b/tests/unittests/test_datasource/test_cloudstack.py
faf1e5
@@ -10,6 +10,9 @@ from cloudinit.tests.helpers import CiTestCase, ExitStack, mock
faf1e5
 import os
faf1e5
 import time
faf1e5
 
faf1e5
+MOD_PATH = 'cloudinit.sources.DataSourceCloudStack'
faf1e5
+DS_PATH = MOD_PATH + '.DataSourceCloudStack'
faf1e5
+
faf1e5
 
faf1e5
 class TestCloudStackPasswordFetching(CiTestCase):
faf1e5
 
faf1e5
@@ -17,7 +20,7 @@ class TestCloudStackPasswordFetching(CiTestCase):
faf1e5
         super(TestCloudStackPasswordFetching, self).setUp()
faf1e5
         self.patches = ExitStack()
faf1e5
         self.addCleanup(self.patches.close)
faf1e5
-        mod_name = 'cloudinit.sources.DataSourceCloudStack'
faf1e5
+        mod_name = MOD_PATH
faf1e5
         self.patches.enter_context(mock.patch('{0}.ec2'.format(mod_name)))
faf1e5
         self.patches.enter_context(mock.patch('{0}.uhelp'.format(mod_name)))
faf1e5
         default_gw = "192.201.20.0"
faf1e5
@@ -56,7 +59,9 @@ class TestCloudStackPasswordFetching(CiTestCase):
faf1e5
         ds.get_data()
faf1e5
         self.assertEqual({}, ds.get_config_obj())
faf1e5
 
faf1e5
-    def test_password_sets_password(self):
faf1e5
+    @mock.patch(DS_PATH + '.wait_for_metadata_service')
faf1e5
+    def test_password_sets_password(self, m_wait):
faf1e5
+        m_wait.return_value = True
faf1e5
         password = 'SekritSquirrel'
faf1e5
         self._set_password_server_response(password)
faf1e5
         ds = DataSourceCloudStack(
faf1e5
@@ -64,7 +69,9 @@ class TestCloudStackPasswordFetching(CiTestCase):
faf1e5
         ds.get_data()
faf1e5
         self.assertEqual(password, ds.get_config_obj()['password'])
faf1e5
 
faf1e5
-    def test_bad_request_doesnt_stop_ds_from_working(self):
faf1e5
+    @mock.patch(DS_PATH + '.wait_for_metadata_service')
faf1e5
+    def test_bad_request_doesnt_stop_ds_from_working(self, m_wait):
faf1e5
+        m_wait.return_value = True
faf1e5
         self._set_password_server_response('bad_request')
faf1e5
         ds = DataSourceCloudStack(
faf1e5
             {}, None, helpers.Paths({'run_dir': self.tmp}))
faf1e5
@@ -79,7 +86,9 @@ class TestCloudStackPasswordFetching(CiTestCase):
faf1e5
                     request_types.append(arg.split()[1])
faf1e5
         self.assertEqual(expected_request_types, request_types)
faf1e5
 
faf1e5
-    def test_valid_response_means_password_marked_as_saved(self):
faf1e5
+    @mock.patch(DS_PATH + '.wait_for_metadata_service')
faf1e5
+    def test_valid_response_means_password_marked_as_saved(self, m_wait):
faf1e5
+        m_wait.return_value = True
faf1e5
         password = 'SekritSquirrel'
faf1e5
         subp = self._set_password_server_response(password)
faf1e5
         ds = DataSourceCloudStack(
faf1e5
@@ -92,7 +101,9 @@ class TestCloudStackPasswordFetching(CiTestCase):
faf1e5
         subp = self._set_password_server_response(response_string)
faf1e5
         ds = DataSourceCloudStack(
faf1e5
             {}, None, helpers.Paths({'run_dir': self.tmp}))
faf1e5
-        ds.get_data()
faf1e5
+        with mock.patch(DS_PATH + '.wait_for_metadata_service') as m_wait:
faf1e5
+            m_wait.return_value = True
faf1e5
+            ds.get_data()
faf1e5
         self.assertRequestTypesSent(subp, ['send_my_password'])
faf1e5
 
faf1e5
     def test_password_not_saved_if_empty(self):
faf1e5
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
faf1e5
index 1a5956d..5c5c787 100644
faf1e5
--- a/tests/unittests/test_datasource/test_ec2.py
faf1e5
+++ b/tests/unittests/test_datasource/test_ec2.py
faf1e5
@@ -191,7 +191,9 @@ def register_mock_metaserver(base_url, data):
faf1e5
             register(base_url, 'not found', status=404)
faf1e5
 
faf1e5
     def myreg(*argc, **kwargs):
faf1e5
-        return httpretty.register_uri(httpretty.GET, *argc, **kwargs)
faf1e5
+        url = argc[0]
faf1e5
+        method = httpretty.PUT if ec2.API_TOKEN_ROUTE in url else httpretty.GET
faf1e5
+        return httpretty.register_uri(method, *argc, **kwargs)
faf1e5
 
faf1e5
     register_helper(myreg, base_url, data)
faf1e5
 
faf1e5
@@ -237,6 +239,8 @@ class TestEc2(test_helpers.HttprettyTestCase):
faf1e5
         if md:
faf1e5
             all_versions = (
faf1e5
                 [ds.min_metadata_version] + ds.extended_metadata_versions)
faf1e5
+            token_url = self.data_url('latest', data_item='api/token')
faf1e5
+            register_mock_metaserver(token_url, 'API-TOKEN')
faf1e5
             for version in all_versions:
faf1e5
                 metadata_url = self.data_url(version) + '/'
faf1e5
                 if version == md_version:
faf1e5
-- 
faf1e5
1.8.3.1
faf1e5