From 3ee42e6e6ca51b3fd0b6461f707d62c89d54e227 Mon Sep 17 00:00:00 2001
From: Johnson Shi <Johnson.Shi@microsoft.com>
Date: Thu, 25 Mar 2021 07:20:10 -0700
Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between
 retries (#842)

RH-Author: Eduardo Otubo <otubo@redhat.com>
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
RH-Commit: [2/7] 65672cdfe2265f32e6d3c440ba5a8accafdb6ca6 (otubo/cloud-init-src)
RH-Bugzilla: 2042351
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

Ensure that the Azure helper's http handler sleeps a fixed duration
between retry failure attempts. The http handler will sleep a fixed
duration between failed attempts regardless of whether the attempt
failed due to (1) request timing out or (2) instant failure (no
timeout).

Due to certain platform issues, the http request to the Azure endpoint
may instantly fail without reaching the http timeout duration. Without
sleeping a fixed duration in between retry attempts, the http handler
will loop through the max retry attempts quickly. This causes the
communication between cloud-init and the Azure platform to be less
resilient due to the short total duration if there is no sleep in
between retries.
---
 cloudinit/sources/helpers/azure.py                   |  2 ++
 tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index d3055d08..03e7156b 100755
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str:
 
     max_readurl_attempts = 240
     default_readurl_timeout = 5
+    sleep_duration_between_retries = 5
     periodic_logging_attempts = 12
 
     if 'timeout' not in kwargs:
@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str:
                     'attempt %d with exception: %s' %
                     (url, attempt, e),
                     logger_func=LOG.debug)
+            time.sleep(sleep_duration_between_retries)
 
     raise exc
 
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b8899807..63482c6c 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
 
     max_readurl_attempts = 240
     default_readurl_timeout = 5
+    sleep_duration_between_retries = 5
     periodic_logging_attempts = 12
 
     def setUp(self):
@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
         self.m_readurl = patches.enter_context(
             mock.patch.object(
                 azure_helper.url_helper, 'readurl', mock.MagicMock()))
-        patches.enter_context(
-            mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
+        self.m_sleep = patches.enter_context(
+            mock.patch.object(azure_helper.time, 'sleep', autospec=True))
 
     def test_http_with_retries(self):
         self.m_readurl.return_value = 'TestResp'
@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
             self.m_readurl.call_count,
             self.periodic_logging_attempts + 1)
 
+        # Ensure that cloud-init did sleep between each failed request
+        self.assertEqual(
+            self.m_sleep.call_count,
+            self.periodic_logging_attempts)
+        self.m_sleep.assert_called_with(self.sleep_duration_between_retries)
+
     def test_http_with_retries_long_delay_logs_periodic_failure_msg(self):
         self.m_readurl.side_effect = \
             [SentinelException] * self.periodic_logging_attempts + \
-- 
2.27.0