--- a/agents/aws/fence_aws.py 2020-03-26 10:31:03.653171381 +0100
+++ b/agents/aws/fence_aws.py 2020-03-24 16:21:16.942155519 +0100
@@ -3,14 +3,33 @@
import sys, re
import logging
import atexit
+import requests
sys.path.append("@FENCEAGENTSLIBDIR@")
from fencing import *
-from fencing import fail, fail_usage, EC_TIMED_OUT, run_delay
+from fencing import fail, fail_usage, run_delay, EC_STATUS, SyslogLibHandler
import boto3
from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
+logger = logging.getLogger("fence_aws")
+logger.propagate = False
+logger.setLevel(logging.INFO)
+logger.addHandler(SyslogLibHandler())
+logging.getLogger('botocore.vendored').propagate = False
+
+def get_instance_id():
+ try:
+ r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
+ return r.content.decode("UTF-8")
+ except HTTPError as http_err:
+ logger.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
+ except Exception as err:
+ logger.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
+ return None
+
+
def get_nodes_list(conn, options):
+ logger.info("Starting monitor operation")
result = {}
try:
for instance in conn.instances.all():
@@ -19,13 +38,17 @@
fail_usage("Failed: Incorrect Access Key or Secret Key.")
except EndpointConnectionError:
fail_usage("Failed: Incorrect Region.")
-
+ except Exception as e:
+ logger.error("Failed to get node list: %s", e)
+ logger.debug("Monitor operation OK: %s",result)
return result
def get_power_status(conn, options):
+ logger.debug("Starting status operation")
try:
instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [options["--plug"]]}])
state = list(instance)[0].state["Name"]
+ logger.info("Status operation for EC2 instance %s returned state: %s",options["--plug"],state.upper())
if state == "running":
return "on"
elif state == "stopped":
@@ -38,20 +61,49 @@
except EndpointConnectionError:
fail_usage("Failed: Incorrect Region.")
except IndexError:
+ fail(EC_STATUS)
+ except Exception as e:
+ logging.error("Failed to get power status: %s", e)
+ fail(EC_STATUS)
+
+def get_self_power_status(conn, instance_id):
+ try:
+ instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
+ state = list(instance)[0].state["Name"]
+ if state == "running":
+ logging.debug("Captured my (%s) state and it %s - returning OK - Proceeding with fencing",instance_id,state.upper())
+ return "ok"
+ else:
+ logging.debug("Captured my (%s) state it is %s - returning Alert - Unable to fence other nodes",instance_id,state.upper())
+ return "alert"
+
+ except ClientError:
+ fail_usage("Failed: Incorrect Access Key or Secret Key.")
+ except EndpointConnectionError:
+ fail_usage("Failed: Incorrect Region.")
+ except IndexError:
return "fail"
def set_power_status(conn, options):
- if (options["--action"]=="off"):
- conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
- elif (options["--action"]=="on"):
- conn.instances.filter(InstanceIds=[options["--plug"]]).start()
-
+ my_instance = get_instance_id()
+ try:
+ if (options["--action"]=="off"):
+ if (get_self_power_status(conn,my_instance) == "ok"):
+ conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
+ logger.info("Called StopInstance API call for %s", options["--plug"])
+ else:
+ logger.info("Skipping fencing as instance is not in running status")
+ elif (options["--action"]=="on"):
+ conn.instances.filter(InstanceIds=[options["--plug"]]).start()
+ except Exception as e:
+ logger.error("Failed to power %s %s: %s", \
+ options["--action"], options["--plug"], e)
def define_new_opts():
all_opt["region"] = {
"getopt" : "r:",
"longopt" : "region",
- "help" : "-r, --region=[name] Region, e.g. us-east-1",
+ "help" : "-r, --region=[region] Region, e.g. us-east-1",
"shortdesc" : "Region.",
"required" : "0",
"order" : 2
@@ -59,7 +111,7 @@
all_opt["access_key"] = {
"getopt" : "a:",
"longopt" : "access-key",
- "help" : "-a, --access-key=[name] Access Key",
+ "help" : "-a, --access-key=[key] Access Key",
"shortdesc" : "Access Key.",
"required" : "0",
"order" : 3
@@ -67,23 +119,32 @@
all_opt["secret_key"] = {
"getopt" : "s:",
"longopt" : "secret-key",
- "help" : "-s, --secret-key=[name] Secret Key",
+ "help" : "-s, --secret-key=[key] Secret Key",
"shortdesc" : "Secret Key.",
"required" : "0",
"order" : 4
}
+ all_opt["boto3_debug"] = {
+ "getopt" : "b:",
+ "longopt" : "boto3_debug",
+ "help" : "-b, --boto3_debug=[option] Boto3 and Botocore library debug logging",
+ "shortdesc": "Boto Lib debug",
+ "required": "0",
+ "order": 5
+ }
# Main agent method
def main():
conn = None
- device_opt = ["port", "no_password", "region", "access_key", "secret_key"]
+ device_opt = ["port", "no_password", "region", "access_key", "secret_key", "boto3_debug"]
atexit.register(atexit_handler)
define_new_opts()
all_opt["power_timeout"]["default"] = "60"
+ all_opt["boto3_debug"]["default"] = "off"
options = check_input(device_opt, process_input(device_opt))
@@ -99,6 +160,28 @@
run_delay(options)
+ if options.get("--verbose") is not None:
+ lh = logging.FileHandler('/var/log/fence_aws_debug.log')
+ logger.addHandler(lh)
+ lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ lh.setFormatter(lhf)
+ logger.setLevel(logging.DEBUG)
+
+ if options["--boto3_debug"] != "on":
+ boto3.set_stream_logger('boto3',logging.INFO)
+ boto3.set_stream_logger('botocore',logging.INFO)
+ logging.getLogger('botocore').propagate = False
+ logging.getLogger('boto3').propagate = False
+ else:
+ log_format = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
+ logging.getLogger('botocore').propagate = False
+ logging.getLogger('boto3').propagate = False
+ fdh = logging.FileHandler('/var/log/fence_aws_boto3.log')
+ fdh.setFormatter(log_format)
+ logging.getLogger('boto3').addHandler(fdh)
+ logging.getLogger('botocore').addHandler(fdh)
+ logging.debug("Boto debug level is %s and sending debug info to /var/log/fence_aws_boto3.log", options["--boto3_debug"])
+
region = options.get("--region")
access_key = options.get("--access-key")
secret_key = options.get("--secret-key")
@@ -106,12 +189,12 @@
conn = boto3.resource('ec2', region_name=region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
- except:
- fail_usage("Failed: Unable to connect to AWS. Check your configuration.")
+ except Exception as e:
+ fail_usage("Failed: Unable to connect to AWS: " + str(e))
# Operate the fencing device
result = fence_action(conn, options, set_power_status, get_power_status, get_nodes_list)
sys.exit(result)
if __name__ == "__main__":
- main()
+ main()
\ No newline at end of file