Blame SOURCES/bz1816203-fence_aws-1-fix-race-condition.patch

bf99d3
From 1742baf17954c58a84b9c668a617bac78303ce95 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Tue, 17 Mar 2020 13:18:38 +0000
bf99d3
Subject: [PATCH 1/9] fence_aws: Fix fence race condition by checking local
bf99d3
 instance status
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 37 ++++++++++++++++++++++++++++++++++++-
bf99d3
 1 file changed, 36 insertions(+), 1 deletion(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index 4a4d9de2..f37f68d6 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -3,6 +3,7 @@
bf99d3
 import sys, re
bf99d3
 import logging
bf99d3
 import atexit
bf99d3
+import requests
bf99d3
 sys.path.append("@FENCEAGENTSLIBDIR@")
bf99d3
 from fencing import *
bf99d3
 from fencing import fail, fail_usage, run_delay, EC_STATUS
bf99d3
@@ -10,6 +11,17 @@
bf99d3
 import boto3
bf99d3
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
bf99d3
 
bf99d3
+def get_instance_id():
bf99d3
+	try:
bf99d3
+		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
bf99d3
+		return r.content
bf99d3
+	except HTTPError as http_err:
bf99d3
+		logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
bf99d3
+	except Exception as err:
bf99d3
+		logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
bf99d3
+	return None
bf99d3
+	
bf99d3
+
bf99d3
 def get_nodes_list(conn, options):
bf99d3
 	result = {}
bf99d3
 	try:
bf99d3
@@ -45,10 +57,33 @@ def get_power_status(conn, options):
bf99d3
 		logging.error("Failed to get power status: %s", e)
bf99d3
 		fail(EC_STATUS)
bf99d3
 
bf99d3
+def get_self_power_status(conn, options):
bf99d3
+	try:
bf99d3
+		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
bf99d3
+		state = list(instance)[0].state["Name"]
bf99d3
+		if state == "running":
bf99d3
+			logging.debug("Captured my (%s) state and it %s - returning OK - Proceeding with fencing",instance_id,state.upper())
bf99d3
+			return "ok"
bf99d3
+		else:
bf99d3
+			logging.debug("Captured my (%s) state it is %s - returning Alert - Unable to fence other nodes",instance_id,state.upper())
bf99d3
+			return "alert"
bf99d3
+	
bf99d3
+	except ClientError:
bf99d3
+		fail_usage("Failed: Incorrect Access Key or Secret Key.")
bf99d3
+	except EndpointConnectionError:
bf99d3
+		fail_usage("Failed: Incorrect Region.")
bf99d3
+	except IndexError:
bf99d3
+		return "fail"
bf99d3
+
bf99d3
 def set_power_status(conn, options):
bf99d3
+	my_instance = get_instance_id()
bf99d3
 	try:
bf99d3
 		if (options["--action"]=="off"):
bf99d3
-			conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
bf99d3
+			if (get_self_power_status(conn,myinstance) == "ok"):
bf99d3
+				logging.info("Called StopInstance API call for %s", options["--plug"])
bf99d3
+				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
bf99d3
+			else:
bf99d3
+				logging.info("Skipping fencing as instance is not in running status")
bf99d3
 		elif (options["--action"]=="on"):
bf99d3
 			conn.instances.filter(InstanceIds=[options["--plug"]]).start()
bf99d3
 	except Exception as e:
bf99d3
bf99d3
From 45e429b3132ebc9e78121c3fbb15f0bf46845a59 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Tue, 17 Mar 2020 13:28:34 +0000
bf99d3
Subject: [PATCH 2/9] fence_aws: Use local logger and improve logging
bf99d3
 experience
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 34 ++++++++++++++++++++++++++--------
bf99d3
 1 file changed, 26 insertions(+), 8 deletions(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index f37f68d6..b0b6685a 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -6,7 +6,7 @@
bf99d3
 import requests
bf99d3
 sys.path.append("@FENCEAGENTSLIBDIR@")
bf99d3
 from fencing import *
bf99d3
-from fencing import fail, fail_usage, run_delay, EC_STATUS
bf99d3
+from fencing import fail, fail_usage, run_delay, EC_STATUS, SyslogLibHandler
bf99d3
 
bf99d3
 import boto3
bf99d3
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
bf99d3
@@ -16,13 +16,14 @@ def get_instance_id():
bf99d3
 		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
bf99d3
 		return r.content
bf99d3
 	except HTTPError as http_err:
bf99d3
-		logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
bf99d3
+		logger.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
bf99d3
 	except Exception as err:
bf99d3
-		logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
bf99d3
+		logger.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
bf99d3
 	return None
bf99d3
 	
bf99d3
 
bf99d3
 def get_nodes_list(conn, options):
bf99d3
+	logger.info("Starting monitor operation")
bf99d3
 	result = {}
bf99d3
 	try:
bf99d3
 		for instance in conn.instances.all():
bf99d3
@@ -32,14 +33,16 @@ def get_nodes_list(conn, options):
bf99d3
 	except EndpointConnectionError:
bf99d3
 		fail_usage("Failed: Incorrect Region.")
bf99d3
 	except Exception as e:
bf99d3
-		logging.error("Failed to get node list: %s", e)
bf99d3
-
bf99d3
+		logger.error("Failed to get node list: %s", e)
bf99d3
+	logger.debug("Monitor operation OK: %s",result)
bf99d3
 	return result
bf99d3
 
bf99d3
 def get_power_status(conn, options):
bf99d3
+	logger.debug("Starting status operation")
bf99d3
 	try:
bf99d3
 		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [options["--plug"]]}])
bf99d3
 		state = list(instance)[0].state["Name"]
bf99d3
+		logger.info("Status operation for EC2 instance %s returned state: %s",options["--plug"],state.upper())
bf99d3
 		if state == "running":
bf99d3
 			return "on"
bf99d3
 		elif state == "stopped":
bf99d3
@@ -80,14 +83,14 @@ def set_power_status(conn, options):
bf99d3
 	try:
bf99d3
 		if (options["--action"]=="off"):
bf99d3
 			if (get_self_power_status(conn,myinstance) == "ok"):
bf99d3
-				logging.info("Called StopInstance API call for %s", options["--plug"])
bf99d3
 				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
bf99d3
+				logger.info("Called StopInstance API call for %s", options["--plug"])
bf99d3
 			else:
bf99d3
-				logging.info("Skipping fencing as instance is not in running status")
bf99d3
+				logger.info("Skipping fencing as instance is not in running status")
bf99d3
 		elif (options["--action"]=="on"):
bf99d3
 			conn.instances.filter(InstanceIds=[options["--plug"]]).start()
bf99d3
 	except Exception as e:
bf99d3
-		logging.error("Failed to power %s %s: %s", \
bf99d3
+		logger.error("Failed to power %s %s: %s", \
bf99d3
 				options["--action"], options["--plug"], e)
bf99d3
 
bf99d3
 def define_new_opts():
bf99d3
@@ -142,6 +145,13 @@ def main():
bf99d3
 
bf99d3
 	run_delay(options)
bf99d3
 
bf99d3
+	if options.get("--verbose") is not None:
bf99d3
+		lh = logging.FileHandler('/var/log/fence_aws_debug.log')
bf99d3
+		logger.addHandler(lh)
bf99d3
+		lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
bf99d3
+		lh.setFormatter(lhf)
bf99d3
+		logger.setLevel(logging.DEBUG)
bf99d3
+
bf99d3
 	region = options.get("--region")
bf99d3
 	access_key = options.get("--access-key")
bf99d3
 	secret_key = options.get("--secret-key")
bf99d3
@@ -157,4 +167,12 @@ def main():
bf99d3
 	sys.exit(result)
bf99d3
 
bf99d3
 if __name__ == "__main__":
bf99d3
+
bf99d3
+	logger = logging.getLogger("fence_aws")
bf99d3
+	logger.propagate = False
bf99d3
+	logger.setLevel(logging.INFO)
bf99d3
+	logger.addHandler(SyslogLibHandler())
bf99d3
+	logger.getLogger('botocore.vendored').propagate = False
bf99d3
+	
bf99d3
+
bf99d3
 	main()
bf99d3
bf99d3
From 00569921597b8007c67296ab8332747baf1e6fae Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Tue, 17 Mar 2020 13:33:02 +0000
bf99d3
Subject: [PATCH 3/9] fence_aws: Decouple boto3 and botocore debug logging from
bf99d3
 local logging
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 26 +++++++++++++++++++++++++-
bf99d3
 1 file changed, 25 insertions(+), 1 deletion(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index b0b6685a..11714315 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -118,18 +118,27 @@ def define_new_opts():
bf99d3
 		"required" : "0",
bf99d3
 		"order" : 4
bf99d3
 	}
bf99d3
+	all_opt["boto3_debug"] = {
bf99d3
+		"getopt" : "b:",
bf99d3
+		"longopt" : "boto3_debug",
bf99d3
+		"help" : "-b, --boto3_debug=on|off      Boto3 and Botocore library debug logging",
bf99d3
+		"shortdesc": "Boto Lib debug",
bf99d3
+		"required": "0",
bf99d3
+		"order": 5
bf99d3
+	}
bf99d3
 
bf99d3
 # Main agent method
bf99d3
 def main():
bf99d3
 	conn = None
bf99d3
 
bf99d3
-	device_opt = ["port", "no_password", "region", "access_key", "secret_key"]
bf99d3
+	device_opt = ["port", "no_password", "region", "access_key", "secret_key", "boto3_debug"]
bf99d3
 
bf99d3
 	atexit.register(atexit_handler)
bf99d3
 
bf99d3
 	define_new_opts()
bf99d3
 
bf99d3
 	all_opt["power_timeout"]["default"] = "60"
bf99d3
+	all_opt["boto3_debug"]["default"] = "off"
bf99d3
 
bf99d3
 	options = check_input(device_opt, process_input(device_opt))
bf99d3
 
bf99d3
@@ -151,6 +160,21 @@ def main():
bf99d3
 		lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
bf99d3
 		lh.setFormatter(lhf)
bf99d3
 		logger.setLevel(logging.DEBUG)
bf99d3
+	
bf99d3
+	if options["--boto3_debug"] != "on":
bf99d3
+		boto3.set_stream_logger('boto3',logging.INFO)
bf99d3
+		boto3.set_stream_logger('botocore',logging.INFO)
bf99d3
+		logging.getLogger('botocore').propagate = False
bf99d3
+		logging.getLogger('boto3').propagate = False
bf99d3
+	else:
bf99d3
+		log_format = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
bf99d3
+		logging.getLogger('botocore').propagate = False
bf99d3
+		logging.getLogger('boto3').propagate = False
bf99d3
+		fdh = logging.FileHandler('/var/log/fence_aws_boto3.log')
bf99d3
+		fdh.setFormatter(log_format)
bf99d3
+		logging.getLogger('boto3').addHandler(fdh)
bf99d3
+		logging.getLogger('botocore').addHandler(fdh)
bf99d3
+		logging.debug("Boto debug level is %s and sending debug info to /var/log/fence_aws_boto3.log", options["--boto3_debug"])
bf99d3
 
bf99d3
 	region = options.get("--region")
bf99d3
 	access_key = options.get("--access-key")
bf99d3
bf99d3
From ed309bd51dfd5e0fed30156e7a312d5b5a8f4bd4 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Thu, 19 Mar 2020 16:02:47 +0000
bf99d3
Subject: [PATCH 4/9] fence_aws: Fix typos and variable names
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 6 +++---
bf99d3
 1 file changed, 3 insertions(+), 3 deletions(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index 11714315..207631e8 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -60,7 +60,7 @@ def get_power_status(conn, options):
bf99d3
 		logging.error("Failed to get power status: %s", e)
bf99d3
 		fail(EC_STATUS)
bf99d3
 
bf99d3
-def get_self_power_status(conn, options):
bf99d3
+def get_self_power_status(conn, instance_id):
bf99d3
 	try:
bf99d3
 		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
bf99d3
 		state = list(instance)[0].state["Name"]
bf99d3
@@ -82,7 +82,7 @@ def set_power_status(conn, options):
bf99d3
 	my_instance = get_instance_id()
bf99d3
 	try:
bf99d3
 		if (options["--action"]=="off"):
bf99d3
-			if (get_self_power_status(conn,myinstance) == "ok"):
bf99d3
+			if (get_self_power_status(conn,my_instance) == "ok"):
bf99d3
 				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
bf99d3
 				logger.info("Called StopInstance API call for %s", options["--plug"])
bf99d3
 			else:
bf99d3
@@ -196,7 +196,7 @@ def main():
bf99d3
 	logger.propagate = False
bf99d3
 	logger.setLevel(logging.INFO)
bf99d3
 	logger.addHandler(SyslogLibHandler())
bf99d3
-	logger.getLogger('botocore.vendored').propagate = False
bf99d3
+	logging.getLogger('botocore.vendored').propagate = False
bf99d3
 	
bf99d3
 
bf99d3
 	main()
bf99d3
bf99d3
From 624c652a95a676286af408898186186b7d7fcf55 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Thu, 19 Mar 2020 16:58:45 +0000
bf99d3
Subject: [PATCH 5/9] fence_aws: Missing brackets on boto3_debug metadata
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 2 +-
bf99d3
 1 file changed, 1 insertion(+), 1 deletion(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index 207631e8..8916f4a0 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -121,7 +121,7 @@ def define_new_opts():
bf99d3
 	all_opt["boto3_debug"] = {
bf99d3
 		"getopt" : "b:",
bf99d3
 		"longopt" : "boto3_debug",
bf99d3
-		"help" : "-b, --boto3_debug=on|off      Boto3 and Botocore library debug logging",
bf99d3
+		"help" : "-b, --boto3_debug=[on|off]      Boto3 and Botocore library debug logging",
bf99d3
 		"shortdesc": "Boto Lib debug",
bf99d3
 		"required": "0",
bf99d3
 		"order": 5
bf99d3
bf99d3
From 7c641a6885c4ab67b7739a43892d92d95a6f566c Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Thu, 19 Mar 2020 17:04:31 +0000
bf99d3
Subject: [PATCH 6/9] fence_aws: Fix travis build #1
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 2 +-
bf99d3
 1 file changed, 1 insertion(+), 1 deletion(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index 8916f4a0..f41a47e4 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -121,7 +121,7 @@ def define_new_opts():
bf99d3
 	all_opt["boto3_debug"] = {
bf99d3
 		"getopt" : "b:",
bf99d3
 		"longopt" : "boto3_debug",
bf99d3
-		"help" : "-b, --boto3_debug=[on|off]      Boto3 and Botocore library debug logging",
bf99d3
+		"help" : "-b, --boto3_debug=[option]      Boto3 and Botocore library debug logging",
bf99d3
 		"shortdesc": "Boto Lib debug",
bf99d3
 		"required": "0",
bf99d3
 		"order": 5
bf99d3
bf99d3
From 257af7ccc9789646adc7abf1e7dbac744b756071 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Fri, 20 Mar 2020 10:59:56 +0000
bf99d3
Subject: [PATCH 7/9] fence_aws: Updated metadata XML file
bf99d3
bf99d3
---
bf99d3
 tests/data/metadata/fence_aws.xml | 5 +++++
bf99d3
 1 file changed, 5 insertions(+)
bf99d3
bf99d3
diff --git a/tests/data/metadata/fence_aws.xml b/tests/data/metadata/fence_aws.xml
bf99d3
index 5e5d5d99..acfebb61 100644
bf99d3
--- a/tests/data/metadata/fence_aws.xml
bf99d3
+++ b/tests/data/metadata/fence_aws.xml
bf99d3
@@ -36,6 +36,11 @@ For instructions see: https://boto3.readthedocs.io/en/latest/guide/quickstart.ht
bf99d3
 		<content type="string"  />
bf99d3
 		<shortdesc lang="en">Secret Key.</shortdesc>
bf99d3
 	</parameter>
bf99d3
+	<parameter name="boto3_debug" unique="0" required="0">
bf99d3
+		<getopt mixed="-b, --boto3_debug=[option]" />
bf99d3
+		<content type="string" default="off"  />
bf99d3
+		<shortdesc lang="en">Boto Lib debug</shortdesc>
bf99d3
+	</parameter>
bf99d3
 	<parameter name="quiet" unique="0" required="0">
bf99d3
 		<getopt mixed="-q, --quiet" />
bf99d3
 		<content type="boolean"  />
bf99d3
bf99d3
From 8f78bc19356b5e07d0021aaf7da3fc4e712e00f0 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Fri, 20 Mar 2020 12:13:16 +0000
bf99d3
Subject: [PATCH 8/9] fence_aws: Moving logger config next to import statements
bf99d3
 for visibility
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 13 ++++++-------
bf99d3
 1 file changed, 6 insertions(+), 7 deletions(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index f41a47e4..72fb8843 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -11,6 +11,12 @@
bf99d3
 import boto3
bf99d3
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
bf99d3
 
bf99d3
+logger = logging.getLogger("fence_aws")
bf99d3
+logger.propagate = False
bf99d3
+logger.setLevel(logging.INFO)
bf99d3
+logger.addHandler(SyslogLibHandler())
bf99d3
+logging.getLogger('botocore.vendored').propagate = False
bf99d3
+	
bf99d3
 def get_instance_id():
bf99d3
 	try:
bf99d3
 		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
bf99d3
@@ -192,11 +198,4 @@ def main():
bf99d3
 
bf99d3
 if __name__ == "__main__":
bf99d3
 
bf99d3
-	logger = logging.getLogger("fence_aws")
bf99d3
-	logger.propagate = False
bf99d3
-	logger.setLevel(logging.INFO)
bf99d3
-	logger.addHandler(SyslogLibHandler())
bf99d3
-	logging.getLogger('botocore.vendored').propagate = False
bf99d3
-	
bf99d3
-
bf99d3
 	main()
bf99d3
bf99d3
From 570a05c425fe55008c8892ebaad8a73d36143909 Mon Sep 17 00:00:00 2001
bf99d3
From: Guilherme Felix <fguilher@amazon.com>
bf99d3
Date: Fri, 20 Mar 2020 14:17:55 +0000
bf99d3
Subject: [PATCH 9/9] fence_aws: Remove empty line
bf99d3
bf99d3
---
bf99d3
 agents/aws/fence_aws.py | 3 +--
bf99d3
 1 file changed, 1 insertion(+), 2 deletions(-)
bf99d3
bf99d3
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
bf99d3
index 72fb8843..ed55f390 100644
bf99d3
--- a/agents/aws/fence_aws.py
bf99d3
+++ b/agents/aws/fence_aws.py
bf99d3
@@ -197,5 +197,4 @@ def main():
bf99d3
 	sys.exit(result)
bf99d3
 
bf99d3
 if __name__ == "__main__":
bf99d3
-
bf99d3
-	main()
bf99d3
+	main()
bf99d3
\ No newline at end of file