From 1742baf17954c58a84b9c668a617bac78303ce95 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Tue, 17 Mar 2020 13:18:38 +0000
Subject: [PATCH 1/9] fence_aws: Fix fence race condition by checking local
instance status
---
agents/aws/fence_aws.py | 37 ++++++++++++++++++++++++++++++++++++-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 4a4d9de2..f37f68d6 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -3,6 +3,7 @@
import sys, re
import logging
import atexit
+import requests
sys.path.append("@FENCEAGENTSLIBDIR@")
from fencing import *
from fencing import fail, fail_usage, run_delay, EC_STATUS
@@ -10,6 +11,17 @@
import boto3
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
+def get_instance_id():
+ try:
+ r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
+ return r.content
+ except HTTPError as http_err:
+ logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
+ except Exception as err:
+ logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
+ return None
+
+
def get_nodes_list(conn, options):
result = {}
try:
@@ -45,10 +57,33 @@ def get_power_status(conn, options):
logging.error("Failed to get power status: %s", e)
fail(EC_STATUS)
+def get_self_power_status(conn, options):
+ try:
+ instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
+ state = list(instance)[0].state["Name"]
+ if state == "running":
+ logging.debug("Captured my (%s) state and it %s - returning OK - Proceeding with fencing",instance_id,state.upper())
+ return "ok"
+ else:
+ logging.debug("Captured my (%s) state it is %s - returning Alert - Unable to fence other nodes",instance_id,state.upper())
+ return "alert"
+
+ except ClientError:
+ fail_usage("Failed: Incorrect Access Key or Secret Key.")
+ except EndpointConnectionError:
+ fail_usage("Failed: Incorrect Region.")
+ except IndexError:
+ return "fail"
+
def set_power_status(conn, options):
+ my_instance = get_instance_id()
try:
if (options["--action"]=="off"):
- conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+ if (get_self_power_status(conn,myinstance) == "ok"):
+ logging.info("Called StopInstance API call for %s", options["--plug"])
+ conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+ else:
+ logging.info("Skipping fencing as instance is not in running status")
elif (options["--action"]=="on"):
conn.instances.filter(InstanceIds=[options["--plug"]]).start()
except Exception as e:
From 45e429b3132ebc9e78121c3fbb15f0bf46845a59 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Tue, 17 Mar 2020 13:28:34 +0000
Subject: [PATCH 2/9] fence_aws: Use local logger and improve logging
experience
---
agents/aws/fence_aws.py | 34 ++++++++++++++++++++++++++--------
1 file changed, 26 insertions(+), 8 deletions(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index f37f68d6..b0b6685a 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -6,7 +6,7 @@
import requests
sys.path.append("@FENCEAGENTSLIBDIR@")
from fencing import *
-from fencing import fail, fail_usage, run_delay, EC_STATUS
+from fencing import fail, fail_usage, run_delay, EC_STATUS, SyslogLibHandler
import boto3
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
@@ -16,13 +16,14 @@ def get_instance_id():
r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
return r.content
except HTTPError as http_err:
- logging.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
+ logger.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
except Exception as err:
- logging.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
+ logger.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
return None
def get_nodes_list(conn, options):
+ logger.info("Starting monitor operation")
result = {}
try:
for instance in conn.instances.all():
@@ -32,14 +33,16 @@ def get_nodes_list(conn, options):
except EndpointConnectionError:
fail_usage("Failed: Incorrect Region.")
except Exception as e:
- logging.error("Failed to get node list: %s", e)
-
+ logger.error("Failed to get node list: %s", e)
+ logger.debug("Monitor operation OK: %s",result)
return result
def get_power_status(conn, options):
+ logger.debug("Starting status operation")
try:
instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [options["--plug"]]}])
state = list(instance)[0].state["Name"]
+ logger.info("Status operation for EC2 instance %s returned state: %s",options["--plug"],state.upper())
if state == "running":
return "on"
elif state == "stopped":
@@ -80,14 +83,14 @@ def set_power_status(conn, options):
try:
if (options["--action"]=="off"):
if (get_self_power_status(conn,myinstance) == "ok"):
- logging.info("Called StopInstance API call for %s", options["--plug"])
conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+ logger.info("Called StopInstance API call for %s", options["--plug"])
else:
- logging.info("Skipping fencing as instance is not in running status")
+ logger.info("Skipping fencing as instance is not in running status")
elif (options["--action"]=="on"):
conn.instances.filter(InstanceIds=[options["--plug"]]).start()
except Exception as e:
- logging.error("Failed to power %s %s: %s", \
+ logger.error("Failed to power %s %s: %s", \
options["--action"], options["--plug"], e)
def define_new_opts():
@@ -142,6 +145,13 @@ def main():
run_delay(options)
+ if options.get("--verbose") is not None:
+ lh = logging.FileHandler('/var/log/fence_aws_debug.log')
+ logger.addHandler(lh)
+ lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ lh.setFormatter(lhf)
+ logger.setLevel(logging.DEBUG)
+
region = options.get("--region")
access_key = options.get("--access-key")
secret_key = options.get("--secret-key")
@@ -157,4 +167,12 @@ def main():
sys.exit(result)
if __name__ == "__main__":
+
+ logger = logging.getLogger("fence_aws")
+ logger.propagate = False
+ logger.setLevel(logging.INFO)
+ logger.addHandler(SyslogLibHandler())
+ logger.getLogger('botocore.vendored').propagate = False
+
+
main()
From 00569921597b8007c67296ab8332747baf1e6fae Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Tue, 17 Mar 2020 13:33:02 +0000
Subject: [PATCH 3/9] fence_aws: Decouple boto3 and botocore debug logging from
local logging
---
agents/aws/fence_aws.py | 26 +++++++++++++++++++++++++-
1 file changed, 25 insertions(+), 1 deletion(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index b0b6685a..11714315 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -118,18 +118,27 @@ def define_new_opts():
"required" : "0",
"order" : 4
}
+ all_opt["boto3_debug"] = {
+ "getopt" : "b:",
+ "longopt" : "boto3_debug",
+ "help" : "-b, --boto3_debug=on|off Boto3 and Botocore library debug logging",
+ "shortdesc": "Boto Lib debug",
+ "required": "0",
+ "order": 5
+ }
# Main agent method
def main():
conn = None
- device_opt = ["port", "no_password", "region", "access_key", "secret_key"]
+ device_opt = ["port", "no_password", "region", "access_key", "secret_key", "boto3_debug"]
atexit.register(atexit_handler)
define_new_opts()
all_opt["power_timeout"]["default"] = "60"
+ all_opt["boto3_debug"]["default"] = "off"
options = check_input(device_opt, process_input(device_opt))
@@ -151,6 +160,21 @@ def main():
lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
lh.setFormatter(lhf)
logger.setLevel(logging.DEBUG)
+
+ if options["--boto3_debug"] != "on":
+ boto3.set_stream_logger('boto3',logging.INFO)
+ boto3.set_stream_logger('botocore',logging.INFO)
+ logging.getLogger('botocore').propagate = False
+ logging.getLogger('boto3').propagate = False
+ else:
+ log_format = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
+ logging.getLogger('botocore').propagate = False
+ logging.getLogger('boto3').propagate = False
+ fdh = logging.FileHandler('/var/log/fence_aws_boto3.log')
+ fdh.setFormatter(log_format)
+ logging.getLogger('boto3').addHandler(fdh)
+ logging.getLogger('botocore').addHandler(fdh)
+ logging.debug("Boto debug level is %s and sending debug info to /var/log/fence_aws_boto3.log", options["--boto3_debug"])
region = options.get("--region")
access_key = options.get("--access-key")
From ed309bd51dfd5e0fed30156e7a312d5b5a8f4bd4 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Thu, 19 Mar 2020 16:02:47 +0000
Subject: [PATCH 4/9] fence_aws: Fix typos and variable names
---
agents/aws/fence_aws.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 11714315..207631e8 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -60,7 +60,7 @@ def get_power_status(conn, options):
logging.error("Failed to get power status: %s", e)
fail(EC_STATUS)
-def get_self_power_status(conn, options):
+def get_self_power_status(conn, instance_id):
try:
instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
state = list(instance)[0].state["Name"]
@@ -82,7 +82,7 @@ def set_power_status(conn, options):
my_instance = get_instance_id()
try:
if (options["--action"]=="off"):
- if (get_self_power_status(conn,myinstance) == "ok"):
+ if (get_self_power_status(conn,my_instance) == "ok"):
conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
logger.info("Called StopInstance API call for %s", options["--plug"])
else:
@@ -196,7 +196,7 @@ def main():
logger.propagate = False
logger.setLevel(logging.INFO)
logger.addHandler(SyslogLibHandler())
- logger.getLogger('botocore.vendored').propagate = False
+ logging.getLogger('botocore.vendored').propagate = False
main()
From 624c652a95a676286af408898186186b7d7fcf55 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Thu, 19 Mar 2020 16:58:45 +0000
Subject: [PATCH 5/9] fence_aws: Missing brackets on boto3_debug metadata
---
agents/aws/fence_aws.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 207631e8..8916f4a0 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -121,7 +121,7 @@ def define_new_opts():
all_opt["boto3_debug"] = {
"getopt" : "b:",
"longopt" : "boto3_debug",
- "help" : "-b, --boto3_debug=on|off Boto3 and Botocore library debug logging",
+ "help" : "-b, --boto3_debug=[on|off] Boto3 and Botocore library debug logging",
"shortdesc": "Boto Lib debug",
"required": "0",
"order": 5
From 7c641a6885c4ab67b7739a43892d92d95a6f566c Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Thu, 19 Mar 2020 17:04:31 +0000
Subject: [PATCH 6/9] fence_aws: Fix travis build #1
---
agents/aws/fence_aws.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 8916f4a0..f41a47e4 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -121,7 +121,7 @@ def define_new_opts():
all_opt["boto3_debug"] = {
"getopt" : "b:",
"longopt" : "boto3_debug",
- "help" : "-b, --boto3_debug=[on|off] Boto3 and Botocore library debug logging",
+ "help" : "-b, --boto3_debug=[option] Boto3 and Botocore library debug logging",
"shortdesc": "Boto Lib debug",
"required": "0",
"order": 5
From 257af7ccc9789646adc7abf1e7dbac744b756071 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Fri, 20 Mar 2020 10:59:56 +0000
Subject: [PATCH 7/9] fence_aws: Updated metadata XML file
---
tests/data/metadata/fence_aws.xml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/tests/data/metadata/fence_aws.xml b/tests/data/metadata/fence_aws.xml
index 5e5d5d99..acfebb61 100644
--- a/tests/data/metadata/fence_aws.xml
+++ b/tests/data/metadata/fence_aws.xml
@@ -36,6 +36,11 @@ For instructions see: https://boto3.readthedocs.io/en/latest/guide/quickstart.ht
<content type="string" />
<shortdesc lang="en">Secret Key.</shortdesc>
</parameter>
+ <parameter name="boto3_debug" unique="0" required="0">
+ <getopt mixed="-b, --boto3_debug=[option]" />
+ <content type="string" default="off" />
+ <shortdesc lang="en">Boto Lib debug</shortdesc>
+ </parameter>
<parameter name="quiet" unique="0" required="0">
<getopt mixed="-q, --quiet" />
<content type="boolean" />
From 8f78bc19356b5e07d0021aaf7da3fc4e712e00f0 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Fri, 20 Mar 2020 12:13:16 +0000
Subject: [PATCH 8/9] fence_aws: Moving logger config next to import statements
for visibility
---
agents/aws/fence_aws.py | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index f41a47e4..72fb8843 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -11,6 +11,12 @@
import boto3
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
+logger = logging.getLogger("fence_aws")
+logger.propagate = False
+logger.setLevel(logging.INFO)
+logger.addHandler(SyslogLibHandler())
+logging.getLogger('botocore.vendored').propagate = False
+
def get_instance_id():
try:
r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
@@ -192,11 +198,4 @@ def main():
if __name__ == "__main__":
- logger = logging.getLogger("fence_aws")
- logger.propagate = False
- logger.setLevel(logging.INFO)
- logger.addHandler(SyslogLibHandler())
- logging.getLogger('botocore.vendored').propagate = False
-
-
main()
From 570a05c425fe55008c8892ebaad8a73d36143909 Mon Sep 17 00:00:00 2001
From: Guilherme Felix <fguilher@amazon.com>
Date: Fri, 20 Mar 2020 14:17:55 +0000
Subject: [PATCH 9/9] fence_aws: Remove empty line
---
agents/aws/fence_aws.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/agents/aws/fence_aws.py b/agents/aws/fence_aws.py
index 72fb8843..ed55f390 100644
--- a/agents/aws/fence_aws.py
+++ b/agents/aws/fence_aws.py
@@ -197,5 +197,4 @@ def main():
sys.exit(result)
if __name__ == "__main__":
-
- main()
+ main()
\ No newline at end of file