|
 |
4eb3b8 |
From f11bbe7f04a48eebcb446e283820d7592f76cf86 Mon Sep 17 00:00:00 2001
|
|
|
4eb3b8 |
From: Johnson Shi <Johnson.Shi@microsoft.com>
|
|
|
4eb3b8 |
Date: Thu, 25 Mar 2021 07:20:10 -0700
|
|
|
4eb3b8 |
Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between
|
|
|
4eb3b8 |
retries (#842)
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
|
|
4eb3b8 |
RH-MergeRequest: 45: Add support for userdata on Azure from IMDS
|
|
|
4eb3b8 |
RH-Commit: [2/7] e8f8bb658b629a8444bd2ba19f109952acf33311
|
|
|
4eb3b8 |
RH-Bugzilla: 2023940
|
|
|
4eb3b8 |
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
|
|
4eb3b8 |
RH-Acked-by: Mohamed Gamal Morsy <mmorsy@redhat.com>
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
Ensure that the Azure helper's http handler sleeps a fixed duration
|
|
|
4eb3b8 |
between retry failure attempts. The http handler will sleep a fixed
|
|
|
4eb3b8 |
duration between failed attempts regardless of whether the attempt
|
|
|
4eb3b8 |
failed due to (1) request timing out or (2) instant failure (no
|
|
|
4eb3b8 |
timeout).
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
Due to certain platform issues, the http request to the Azure endpoint
|
|
|
4eb3b8 |
may instantly fail without reaching the http timeout duration. Without
|
|
|
4eb3b8 |
sleeping a fixed duration in between retry attempts, the http handler
|
|
|
4eb3b8 |
will loop through the max retry attempts quickly. This causes the
|
|
|
4eb3b8 |
communication between cloud-init and the Azure platform to be less
|
|
|
4eb3b8 |
resilient due to the short total duration if there is no sleep in
|
|
|
4eb3b8 |
between retries.
|
|
|
4eb3b8 |
---
|
|
|
4eb3b8 |
cloudinit/sources/helpers/azure.py | 2 ++
|
|
|
4eb3b8 |
tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++--
|
|
|
4eb3b8 |
2 files changed, 11 insertions(+), 2 deletions(-)
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
|
|
|
4eb3b8 |
index d3055d08..03e7156b 100755
|
|
|
4eb3b8 |
--- a/cloudinit/sources/helpers/azure.py
|
|
|
4eb3b8 |
+++ b/cloudinit/sources/helpers/azure.py
|
|
|
4eb3b8 |
@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str:
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
max_readurl_attempts = 240
|
|
|
4eb3b8 |
default_readurl_timeout = 5
|
|
|
4eb3b8 |
+ sleep_duration_between_retries = 5
|
|
|
4eb3b8 |
periodic_logging_attempts = 12
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
if 'timeout' not in kwargs:
|
|
|
4eb3b8 |
@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str:
|
|
|
4eb3b8 |
'attempt %d with exception: %s' %
|
|
|
4eb3b8 |
(url, attempt, e),
|
|
|
4eb3b8 |
logger_func=LOG.debug)
|
|
|
4eb3b8 |
+ time.sleep(sleep_duration_between_retries)
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
raise exc
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
|
|
|
4eb3b8 |
index b8899807..63482c6c 100644
|
|
|
4eb3b8 |
--- a/tests/unittests/test_datasource/test_azure_helper.py
|
|
|
4eb3b8 |
+++ b/tests/unittests/test_datasource/test_azure_helper.py
|
|
|
4eb3b8 |
@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
max_readurl_attempts = 240
|
|
|
4eb3b8 |
default_readurl_timeout = 5
|
|
|
4eb3b8 |
+ sleep_duration_between_retries = 5
|
|
|
4eb3b8 |
periodic_logging_attempts = 12
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
def setUp(self):
|
|
|
4eb3b8 |
@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
|
|
4eb3b8 |
self.m_readurl = patches.enter_context(
|
|
|
4eb3b8 |
mock.patch.object(
|
|
|
4eb3b8 |
azure_helper.url_helper, 'readurl', mock.MagicMock()))
|
|
|
4eb3b8 |
- patches.enter_context(
|
|
|
4eb3b8 |
- mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
|
|
|
4eb3b8 |
+ self.m_sleep = patches.enter_context(
|
|
|
4eb3b8 |
+ mock.patch.object(azure_helper.time, 'sleep', autospec=True))
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
def test_http_with_retries(self):
|
|
|
4eb3b8 |
self.m_readurl.return_value = 'TestResp'
|
|
|
4eb3b8 |
@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
|
|
4eb3b8 |
self.m_readurl.call_count,
|
|
|
4eb3b8 |
self.periodic_logging_attempts + 1)
|
|
|
4eb3b8 |
|
|
|
4eb3b8 |
+ # Ensure that cloud-init did sleep between each failed request
|
|
|
4eb3b8 |
+ self.assertEqual(
|
|
|
4eb3b8 |
+ self.m_sleep.call_count,
|
|
|
4eb3b8 |
+ self.periodic_logging_attempts)
|
|
|
4eb3b8 |
+ self.m_sleep.assert_called_with(self.sleep_duration_between_retries)
|
|
|
4eb3b8 |
+
|
|
|
4eb3b8 |
def test_http_with_retries_long_delay_logs_periodic_failure_msg(self):
|
|
|
4eb3b8 |
self.m_readurl.side_effect = \
|
|
|
4eb3b8 |
[SentinelException] * self.periodic_logging_attempts + \
|
|
|
4eb3b8 |
--
|
|
|
4eb3b8 |
2.27.0
|
|
|
4eb3b8 |
|