a27284
--- a/agents/gce/fence_gce.py	2022-04-28 12:02:23.524448642 +0200
a27284
+++ b/agents/gce/fence_gce.py	2022-04-28 12:02:27.698469373 +0200
a27284
@@ -1,10 +1,23 @@
a27284
 #!@PYTHON@ -tt
a27284
 
a27284
+#
a27284
+# Requires the googleapiclient and oauth2client
a27284
+# RHEL 7.x: google-api-python-client==1.6.7 python-gflags==2.0 pyasn1==0.4.8 rsa==3.4.2 pysocks==1.7.1 httplib2==0.19.0
a27284
+# RHEL 8.x: pysocks==1.7.1 httplib2==0.19.0
a27284
+# SLES 12.x: python-google-api-python-client python-oauth2client python-oauth2client-gce pysocks==1.7.1 httplib2==0.19.0
a27284
+# SLES 15.x: python3-google-api-python-client python3-oauth2client pysocks==1.7.1 httplib2==0.19.0
a27284
+#
a27284
+
a27284
 import atexit
a27284
+import httplib2
a27284
 import logging
a27284
+import json
a27284
+import re
a27284
 import os
a27284
+import socket
a27284
 import sys
a27284
 import time
a27284
+
a27284
 if sys.version_info >= (3, 0):
a27284
   # Python 3 imports.
a27284
   import urllib.parse as urlparse
a27284
@@ -15,12 +28,98 @@
a27284
   import urllib2 as urlrequest
a27284
 sys.path.append("@FENCEAGENTSLIBDIR@")
a27284
 
a27284
-import googleapiclient.discovery
a27284
-from fencing import fail_usage, run_delay, all_opt, atexit_handler, check_input, process_input, show_docs, fence_action
a27284
-
a27284
+from fencing import fail_usage, run_delay, all_opt, atexit_handler, check_input, process_input, show_docs, fence_action, run_command
a27284
+try:
a27284
+  import googleapiclient.discovery
a27284
+  import socks
a27284
+  try:
a27284
+    from google.oauth2.credentials import Credentials as GoogleCredentials
a27284
+  except:
a27284
+    from oauth2client.client import GoogleCredentials
a27284
+except:
a27284
+  pass
a27284
 
a27284
 METADATA_SERVER = 'http://metadata.google.internal/computeMetadata/v1/'
a27284
 METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
a27284
+INSTANCE_LINK = 'https://www.googleapis.com/compute/v1/projects/{}/zones/{}/instances/{}'
a27284
+
a27284
+def run_on_fail(options):
a27284
+	if "--runonfail" in options:
a27284
+		run_command(options, options["--runonfail"])
a27284
+
a27284
+def fail_fence_agent(options, message):
a27284
+	run_on_fail(options)
a27284
+	fail_usage(message)
a27284
+
a27284
+def raise_fence_agent(options, message):
a27284
+	run_on_fail(options)
a27284
+	raise Exception(message)
a27284
+
a27284
+#
a27284
+# Will use baremetalsolution setting or the environment variable
a27284
+# FENCE_GCE_URI_REPLACEMENTS to replace the uri for calls to *.googleapis.com.
a27284
+#
a27284
+def replace_api_uri(options, http_request):
a27284
+	uri_replacements = []
a27284
+	# put any env var replacements first, then baremetalsolution if in options
a27284
+	if "FENCE_GCE_URI_REPLACEMENTS" in os.environ:
a27284
+		logging.debug("FENCE_GCE_URI_REPLACEMENTS environment variable exists")
a27284
+		env_uri_replacements = os.environ["FENCE_GCE_URI_REPLACEMENTS"]
a27284
+		try:
a27284
+			uri_replacements_json = json.loads(env_uri_replacements)
a27284
+			if isinstance(uri_replacements_json, list):
a27284
+				uri_replacements = uri_replacements_json
a27284
+			else:
a27284
+				logging.warning("FENCE_GCE_URI_REPLACEMENTS exists, but is not a JSON List")
a27284
+		except ValueError as e:
a27284
+			logging.warning("FENCE_GCE_URI_REPLACEMENTS exists but is not valid JSON")
a27284
+	if "--baremetalsolution" in options:
a27284
+		uri_replacements.append(
a27284
+			{
a27284
+				"matchlength": 4,
a27284
+				"match": "https://compute.googleapis.com/compute/v1/projects/(.*)/zones/(.*)/instances/(.*)/reset(.*)",
a27284
+				"replace": "https://baremetalsolution.googleapis.com/v1/projects/\\1/locations/\\2/instances/\\3:resetInstance\\4"
a27284
+			})
a27284
+	for uri_replacement in uri_replacements:
a27284
+		# each uri_replacement should have matchlength, match, and replace
a27284
+		if "matchlength" not in uri_replacement or "match" not in uri_replacement or "replace" not in uri_replacement:
a27284
+			logging.warning("FENCE_GCE_URI_REPLACEMENTS missing matchlength, match, or replace in %s" % uri_replacement)
a27284
+			continue
a27284
+		match = re.match(uri_replacement["match"], http_request.uri)
a27284
+		if match is None or len(match.groups()) != uri_replacement["matchlength"]:
a27284
+			continue
a27284
+		replaced_uri = re.sub(uri_replacement["match"], uri_replacement["replace"], http_request.uri)
a27284
+		match = re.match("https:\/\/.*.googleapis.com", replaced_uri)
a27284
+		if match is None or match.start() != 0:
a27284
+			logging.warning("FENCE_GCE_URI_REPLACEMENTS replace is not "
a27284
+				"targeting googleapis.com, ignoring it: %s" % replaced_uri)
a27284
+			continue
a27284
+		logging.debug("Replacing googleapis uri %s with %s" % (http_request.uri, replaced_uri))
a27284
+		http_request.uri = replaced_uri
a27284
+		break
a27284
+	return http_request
a27284
+
a27284
+def retry_api_execute(options, http_request):
a27284
+	replaced_http_request = replace_api_uri(options, http_request)
a27284
+	retries = 3
a27284
+	if options.get("--retries"):
a27284
+		retries = int(options.get("--retries"))
a27284
+	retry_sleep = 5
a27284
+	if options.get("--retrysleep"):
a27284
+		retry_sleep = int(options.get("--retrysleep"))
a27284
+	retry = 0
a27284
+	current_err = None
a27284
+	while retry <= retries:
a27284
+		if retry > 0:
a27284
+			time.sleep(retry_sleep)
a27284
+		try:
a27284
+			return replaced_http_request.execute()
a27284
+		except Exception as err:
a27284
+			current_err = err
a27284
+			logging.warning("Could not execute api call to: %s, retry: %s, "
a27284
+				"err: %s" % (replaced_http_request.uri, retry, str(err)))
a27284
+		retry += 1
a27284
+	raise current_err
a27284
 
a27284
 
a27284
 def translate_status(instance_status):
a27284
@@ -34,86 +133,174 @@
a27284
 
a27284
 def get_nodes_list(conn, options):
a27284
 	result = {}
a27284
+	if "--zone" not in options:
a27284
+		fail_fence_agent(options, "Failed: get_nodes_list: Please specify the --zone in the command")
a27284
 	try:
a27284
-		instanceList = conn.instances().list(project=options["--project"], zone=options["--zone"]).execute()
a27284
-		for instance in instanceList["items"]:
a27284
-			result[instance["id"]] = (instance["name"], translate_status(instance["status"]))
a27284
+		for zone in options["--zone"].split(","):
a27284
+			instanceList = retry_api_execute(options, conn.instances().list(
a27284
+				project=options["--project"],
a27284
+				zone=zone))
a27284
+			for instance in instanceList["items"]:
a27284
+				result[instance["id"]] = (instance["name"], translate_status(instance["status"]))
a27284
 	except Exception as err:
a27284
-		fail_usage("Failed: get_nodes_list: {}".format(str(err)))
a27284
+		fail_fence_agent(options, "Failed: get_nodes_list: {}".format(str(err)))
a27284
 
a27284
 	return result
a27284
 
a27284
 
a27284
 def get_power_status(conn, options):
a27284
+	logging.debug("get_power_status")
a27284
+	# if this is bare metal we need to just send back the opposite of the
a27284
+	# requested action: if on send off, if off send on
a27284
+	if "--baremetalsolution" in options:
a27284
+		if options.get("--action") == "on":
a27284
+			return "off"
a27284
+		else:
a27284
+			return "on"
a27284
+	# If zone is not listed for an entry we attempt to get it automatically
a27284
+	instance = options["--plug"]
a27284
+	zone = get_zone(conn, options, instance) if "--plugzonemap" not in options else options["--plugzonemap"][instance]
a27284
+	instance_status = get_instance_power_status(conn, options, instance, zone)
a27284
+	# If any of the instances do not match the intended status we return the
a27284
+	# the opposite status so that the fence agent can change it.
a27284
+	if instance_status != options.get("--action"):
a27284
+		return instance_status
a27284
+
a27284
+	return options.get("--action")
a27284
+
a27284
+
a27284
+def get_instance_power_status(conn, options, instance, zone):
a27284
 	try:
a27284
-		instance = conn.instances().get(
a27284
-				project=options["--project"],
a27284
-				zone=options["--zone"],
a27284
-				instance=options["--plug"]).execute()
a27284
+		instance = retry_api_execute(
a27284
+				options,
a27284
+				conn.instances().get(project=options["--project"], zone=zone, instance=instance))
a27284
 		return translate_status(instance["status"])
a27284
 	except Exception as err:
a27284
-		fail_usage("Failed: get_power_status: {}".format(str(err)))
a27284
+		fail_fence_agent(options, "Failed: get_instance_power_status: {}".format(str(err)))
a27284
 
a27284
 
a27284
-def wait_for_operation(conn, project, zone, operation):
a27284
+def check_for_existing_operation(conn, options, instance, zone, operation_type):
a27284
+	logging.debug("check_for_existing_operation")
a27284
+	if "--baremetalsolution" in options:
a27284
+		# There is no API for checking in progress operations
a27284
+		return False
a27284
+
a27284
+	project = options["--project"]
a27284
+	target_link = INSTANCE_LINK.format(project, zone, instance)
a27284
+	query_filter = '(targetLink = "{}") AND (operationType = "{}") AND (status = "RUNNING")'.format(target_link, operation_type)
a27284
+	result = retry_api_execute(
a27284
+			options,
a27284
+			conn.zoneOperations().list(project=project, zone=zone, filter=query_filter, maxResults=1))
a27284
+
a27284
+	if "items" in result and result["items"]:
a27284
+		logging.info("Existing %s operation found", operation_type)
a27284
+		return result["items"][0]
a27284
+
a27284
+
a27284
+def wait_for_operation(conn, options, zone, operation):
a27284
+	if 'name' not in operation:
a27284
+		logging.warning('Cannot wait for operation to complete, the'
a27284
+		' requested operation will continue asynchronously')
a27284
+		return False
a27284
+
a27284
+	wait_time = 0
a27284
+	project = options["--project"]
a27284
 	while True:
a27284
-		result = conn.zoneOperations().get(
a27284
+		result = retry_api_execute(options, conn.zoneOperations().get(
a27284
 			project=project,
a27284
 			zone=zone,
a27284
-			operation=operation['name']).execute()
a27284
+			operation=operation['name']))
a27284
 		if result['status'] == 'DONE':
a27284
 			if 'error' in result:
a27284
-				raise Exception(result['error'])
a27284
-			return
a27284
+				raise_fence_agent(options, result['error'])
a27284
+			return True
a27284
+
a27284
+		if "--errortimeout" in options and wait_time > int(options["--errortimeout"]):
a27284
+			raise_fence_agent(options, "Operation did not complete before the timeout.")
a27284
+
a27284
+		if "--warntimeout" in options and wait_time > int(options["--warntimeout"]):
a27284
+			logging.warning("Operation did not complete before the timeout.")
a27284
+			if "--runonwarn" in options:
a27284
+				run_command(options, options["--runonwarn"])
a27284
+			return False
a27284
+
a27284
+		wait_time = wait_time + 1
a27284
 		time.sleep(1)
a27284
 
a27284
 
a27284
 def set_power_status(conn, options):
a27284
+	logging.debug("set_power_status")
a27284
+	instance = options["--plug"]
a27284
+	# If zone is not listed for an entry we attempt to get it automatically
a27284
+	zone = get_zone(conn, options, instance) if "--plugzonemap" not in options else options["--plugzonemap"][instance]
a27284
+	set_instance_power_status(conn, options, instance, zone, options["--action"])
a27284
+
a27284
+
a27284
+def set_instance_power_status(conn, options, instance, zone, action):
a27284
+	logging.info("Setting power status of %s in zone %s", instance, zone)
a27284
+	project = options["--project"]
a27284
+
a27284
 	try:
a27284
-		if options["--action"] == "off":
a27284
-			logging.info("Issuing poweroff of %s in zone %s" % (options["--plug"], options["--zone"]))
a27284
-			operation = conn.instances().stop(
a27284
-					project=options["--project"],
a27284
-					zone=options["--zone"],
a27284
-					instance=options["--plug"]).execute()
a27284
-			wait_for_operation(conn, options["--project"], options["--zone"], operation)
a27284
-			logging.info("Poweroff of %s in zone %s complete" % (options["--plug"], options["--zone"]))
a27284
-		elif options["--action"] == "on":
a27284
-			logging.info("Issuing poweron of %s in zone %s" % (options["--plug"], options["--zone"]))
a27284
-			operation = conn.instances().start(
a27284
-					project=options["--project"],
a27284
-					zone=options["--zone"],
a27284
-					instance=options["--plug"]).execute()
a27284
-			wait_for_operation(conn, options["--project"], options["--zone"], operation)
a27284
-			logging.info("Poweron of %s in zone %s complete" % (options["--plug"], options["--zone"]))
a27284
+		if action == "off":
a27284
+			logging.info("Issuing poweroff of %s in zone %s", instance, zone)
a27284
+			operation = check_for_existing_operation(conn, options, instance, zone, "stop")
a27284
+			if operation and "--earlyexit" in options:
a27284
+				return
a27284
+			if not operation:
a27284
+				operation = retry_api_execute(
a27284
+						options,
a27284
+						conn.instances().stop(project=project, zone=zone, instance=instance))
a27284
+			logging.info("Poweroff command completed, waiting for the operation to complete")
a27284
+			if wait_for_operation(conn, options, zone, operation):
a27284
+				logging.info("Poweroff of %s in zone %s complete", instance, zone)
a27284
+		elif action == "on":
a27284
+			logging.info("Issuing poweron of %s in zone %s", instance, zone)
a27284
+			operation = check_for_existing_operation(conn, options, instance, zone, "start")
a27284
+			if operation and "--earlyexit" in options:
a27284
+				return
a27284
+			if not operation:
a27284
+				operation = retry_api_execute(
a27284
+						options,
a27284
+						conn.instances().start(project=project, zone=zone, instance=instance))
a27284
+			if wait_for_operation(conn, options, zone, operation):
a27284
+				logging.info("Poweron of %s in zone %s complete", instance, zone)
a27284
 	except Exception as err:
a27284
-		fail_usage("Failed: set_power_status: {}".format(str(err)))
a27284
-
a27284
+		fail_fence_agent(options, "Failed: set_instance_power_status: {}".format(str(err)))
a27284
 
a27284
 def power_cycle(conn, options):
a27284
+	logging.debug("power_cycle")
a27284
+	instance = options["--plug"]
a27284
+	# If zone is not listed for an entry we attempt to get it automatically
a27284
+	zone = get_zone(conn, options, instance) if "--plugzonemap" not in options else options["--plugzonemap"][instance]
a27284
+	return power_cycle_instance(conn, options, instance, zone)
a27284
+
a27284
+
a27284
+def power_cycle_instance(conn, options, instance, zone):
a27284
+	logging.info("Issuing reset of %s in zone %s", instance, zone)
a27284
+	project = options["--project"]
a27284
+
a27284
 	try:
a27284
-		logging.info('Issuing reset of %s in zone %s' % (options["--plug"], options["--zone"]))
a27284
-		operation = conn.instances().reset(
a27284
-				project=options["--project"],
a27284
-				zone=options["--zone"],
a27284
-				instance=options["--plug"]).execute()
a27284
-		wait_for_operation(conn, options["--project"], options["--zone"], operation)
a27284
-		logging.info('Reset of %s in zone %s complete' % (options["--plug"], options["--zone"]))
a27284
+		operation = check_for_existing_operation(conn, options, instance, zone, "reset")
a27284
+		if operation and "--earlyexit" in options:
a27284
+			return True
a27284
+		if not operation:
a27284
+			operation = retry_api_execute(
a27284
+					options,
a27284
+					conn.instances().reset(project=project, zone=zone, instance=instance))
a27284
+		logging.info("Reset command sent, waiting for the operation to complete")
a27284
+		if wait_for_operation(conn, options, zone, operation):
a27284
+			logging.info("Reset of %s in zone %s complete", instance, zone)
a27284
 		return True
a27284
 	except Exception as err:
a27284
-		logging.error("Failed: power_cycle: {}".format(str(err)))
a27284
-		return False
a27284
+		logging.exception("Failed: power_cycle")
a27284
+		raise err
a27284
 
a27284
 
a27284
-def get_instance(conn, project, zone, instance):
a27284
-	request = conn.instances().get(
a27284
-			project=project, zone=zone, instance=instance)
a27284
-	return request.execute()
a27284
-
a27284
-
a27284
-def get_zone(conn, project, instance):
a27284
+def get_zone(conn, options, instance):
a27284
+	logging.debug("get_zone");
a27284
+	project = options['--project']
a27284
 	fl = 'name="%s"' % instance
a27284
-	request = conn.instances().aggregatedList(project=project, filter=fl)
a27284
+	request = replace_api_uri(options, conn.instances().aggregatedList(project=project, filter=fl))
a27284
 	while request is not None:
a27284
 		response = request.execute()
a27284
 		zones = response.get('items', {})
a27284
@@ -121,9 +308,9 @@
a27284
 			for inst in zone.get('instances', []):
a27284
 				if inst['name'] == instance:
a27284
 					return inst['zone'].split("/")[-1]
a27284
-		request = conn.instances().aggregatedList_next(
a27284
-				previous_request=request, previous_response=response)
a27284
-	raise Exception("Unable to find instance %s" % (instance))
a27284
+		request = replace_api_uri(options, conn.instances().aggregatedList_next(
a27284
+				previous_request=request, previous_response=response))
a27284
+	raise_fence_agent(options, "Unable to find instance %s" % (instance))
a27284
 
a27284
 
a27284
 def get_metadata(metadata_key, params=None, timeout=None):
a27284
@@ -140,13 +327,14 @@
a27284
 	Raises:
a27284
 		urlerror.HTTPError: raises when the GET request fails.
a27284
 	"""
a27284
+	logging.debug("get_metadata");
a27284
 	timeout = timeout or 60
a27284
 	metadata_url = os.path.join(METADATA_SERVER, metadata_key)
a27284
 	params = urlparse.urlencode(params or {})
a27284
 	url = '%s?%s' % (metadata_url, params)
a27284
 	request = urlrequest.Request(url, headers=METADATA_HEADERS)
a27284
 	request_opener = urlrequest.build_opener(urlrequest.ProxyHandler({}))
a27284
-	return request_opener.open(request, timeout=timeout * 1.1).read()
a27284
+	return request_opener.open(request, timeout=timeout * 1.1).read().decode("utf-8")
a27284
 
a27284
 
a27284
 def define_new_opts():
a27284
@@ -169,18 +357,134 @@
a27284
 	all_opt["stackdriver-logging"] = {
a27284
 		"getopt" : "",
a27284
 		"longopt" : "stackdriver-logging",
a27284
-		"help" : "--stackdriver-logging		Enable Logging to Stackdriver. Using stackdriver logging requires additional libraries (google-cloud-logging).",
a27284
-		"shortdesc" : "Stackdriver-logging support. Requires additional libraries (google-cloud-logging).",
a27284
-		"longdesc" : "If enabled IP failover logs will be posted to stackdriver logging. Using stackdriver logging requires additional libraries (google-cloud-logging).",
a27284
+		"help" : "--stackdriver-logging          Enable Logging to Stackdriver",
a27284
+		"shortdesc" : "Stackdriver-logging support.",
a27284
+		"longdesc" : "If enabled IP failover logs will be posted to stackdriver logging.",
a27284
 		"required" : "0",
a27284
 		"order" : 4
a27284
 	}
a27284
+	all_opt["baremetalsolution"] = {
a27284
+		"getopt" : "",
a27284
+		"longopt" : "baremetalsolution",
a27284
+		"help" : "--baremetalsolution            Enable on bare metal",
a27284
+		"shortdesc" : "If enabled this is a bare metal offering from google.",
a27284
+		"required" : "0",
a27284
+		"order" : 5
a27284
+	}
a27284
+	all_opt["apitimeout"] = {
a27284
+		"getopt" : ":",
a27284
+		"type" : "second",
a27284
+		"longopt" : "apitimeout",
a27284
+		"help" : "--apitimeout=[seconds]         Timeout to use for API calls",
a27284
+		"shortdesc" : "Timeout in seconds to use for API calls, default is 60.",
a27284
+		"required" : "0",
a27284
+		"default" : 60,
a27284
+		"order" : 6
a27284
+	}
a27284
+	all_opt["retries"] = {
a27284
+		"getopt" : ":",
a27284
+		"type" : "integer",
a27284
+		"longopt" : "retries",
a27284
+		"help" : "--retries=[retries]            Number of retries on failure for API calls",
a27284
+		"shortdesc" : "Number of retries on failure for API calls, default is 3.",
a27284
+		"required" : "0",
a27284
+		"default" : 3,
a27284
+		"order" : 7
a27284
+	}
a27284
+	all_opt["retrysleep"] = {
a27284
+		"getopt" : ":",
a27284
+		"type" : "second",
a27284
+		"longopt" : "retrysleep",
a27284
+		"help" : "--retrysleep=[seconds]         Time to sleep between API retries",
a27284
+		"shortdesc" : "Time to sleep in seconds between API retries, default is 5.",
a27284
+		"required" : "0",
a27284
+		"default" : 5,
a27284
+		"order" : 8
a27284
+	}
a27284
+	all_opt["serviceaccount"] = {
a27284
+		"getopt" : ":",
a27284
+		"longopt" : "serviceaccount",
a27284
+		"help" : "--serviceaccount=[filename]    Service account json file location e.g. serviceaccount=/somedir/service_account.json",
a27284
+		"shortdesc" : "Service Account to use for authentication to the google cloud APIs.",
a27284
+		"required" : "0",
a27284
+		"order" : 9
a27284
+	}
a27284
+	all_opt["plugzonemap"] = {
a27284
+		"getopt" : ":",
a27284
+		"longopt" : "plugzonemap",
a27284
+		"help" : "--plugzonemap=[plugzonemap]    Comma separated zone map when fencing multiple plugs",
a27284
+		"shortdesc" : "Comma separated zone map when fencing multiple plugs.",
a27284
+		"required" : "0",
a27284
+		"order" : 10
a27284
+	}
a27284
+	all_opt["proxyhost"] = {
a27284
+		"getopt" : ":",
a27284
+		"longopt" : "proxyhost",
a27284
+		"help" : "--proxyhost=[proxy_host]       The proxy host to use, if one is needed to access the internet (Example: 10.122.0.33)",
a27284
+		"shortdesc" : "If a proxy is used for internet access, the proxy host should be specified.",
a27284
+		"required" : "0",
a27284
+		"order" : 11
a27284
+	}
a27284
+	all_opt["proxyport"] = {
a27284
+		"getopt" : ":",
a27284
+		"type" : "integer",
a27284
+		"longopt" : "proxyport",
a27284
+		"help" : "--proxyport=[proxy_port]       The proxy port to use, if one is needed to access the internet (Example: 3127)",
a27284
+		"shortdesc" : "If a proxy is used for internet access, the proxy port should be specified.",
a27284
+		"required" : "0",
a27284
+		"order" : 12
a27284
+	}
a27284
+	all_opt["earlyexit"] = {
a27284
+		"getopt" : "",
a27284
+		"longopt" : "earlyexit",
a27284
+		"help" : "--earlyexit                    Return early if reset is already in progress",
a27284
+		"shortdesc" : "If an existing reset operation is detected, the fence agent will return before the operation completes with a 0 return code.",
a27284
+		"required" : "0",
a27284
+		"order" : 13
a27284
+	}
a27284
+	all_opt["warntimeout"] = {
a27284
+		"getopt" : ":",
a27284
+		"type" : "second",
a27284
+		"longopt" : "warntimeout",
a27284
+		"help" : "--warntimeout=[warn_timeout]   Timeout seconds before logging a warning and returning a 0 status code",
a27284
+		"shortdesc" : "If the operation is not completed within the timeout, the cluster operations are allowed to continue.",
a27284
+		"required" : "0",
a27284
+		"order" : 14
a27284
+	}
a27284
+	all_opt["errortimeout"] = {
a27284
+		"getopt" : ":",
a27284
+		"type" : "second",
a27284
+		"longopt" : "errortimeout",
a27284
+		"help" : "--errortimeout=[error_timeout] Timeout seconds before failing and returning a non-zero status code",
a27284
+		"shortdesc" : "If the operation is not completed within the timeout, cluster is notified of the operation failure.",
a27284
+		"required" : "0",
a27284
+		"order" : 15
a27284
+	}
a27284
+	all_opt["runonwarn"] = {
a27284
+		"getopt" : ":",
a27284
+		"longopt" : "runonwarn",
a27284
+		"help" : "--runonwarn=[run_on_warn]      If a timeout occurs and warning is generated, run the supplied command",
a27284
+		"shortdesc" : "If a timeout would occur while running the agent, then the supplied command is run.",
a27284
+		"required" : "0",
a27284
+		"order" : 16
a27284
+	}
a27284
+	all_opt["runonfail"] = {
a27284
+		"getopt" : ":",
a27284
+		"longopt" : "runonfail",
a27284
+		"help" : "--runonfail=[run_on_fail]      If a failure occurs, run the supplied command",
a27284
+		"shortdesc" : "If a failure would occur while running the agent, then the supplied command is run.",
a27284
+		"required" : "0",
a27284
+		"order" : 17
a27284
+	}
a27284
 
a27284
 
a27284
 def main():
a27284
 	conn = None
a27284
 
a27284
-	device_opt = ["port", "no_password", "zone", "project", "stackdriver-logging", "method"]
a27284
+	device_opt = ["port", "no_password", "zone", "project", "stackdriver-logging",
a27284
+		"method", "baremetalsolution", "apitimeout", "retries", "retrysleep",
a27284
+		"serviceaccount", "plugzonemap", "proxyhost", "proxyport", "earlyexit",
a27284
+		"warntimeout", "errortimeout", "runonwarn", "runonfail"]
a27284
 
a27284
 	atexit.register(atexit_handler)
a27284
 
a27284
@@ -224,28 +528,76 @@
a27284
 			logging.error('Couldn\'t import google.cloud.logging, '
a27284
 				'disabling Stackdriver-logging support')
a27284
 
a27284
+  # if apitimeout is defined we set the socket timeout, if not we keep the
a27284
+  # socket default which is 60s
a27284
+	if options.get("--apitimeout"):
a27284
+		socket.setdefaulttimeout(options["--apitimeout"])
a27284
+
a27284
 	# Prepare cli
a27284
 	try:
a27284
-		credentials = None
a27284
-		if tuple(googleapiclient.__version__) < tuple("1.6.0"):
a27284
-			import oauth2client.client
a27284
-			credentials = oauth2client.client.GoogleCredentials.get_application_default()
a27284
-		conn = googleapiclient.discovery.build('compute', 'v1', credentials=credentials)
a27284
+		serviceaccount = options.get("--serviceaccount")
a27284
+		if serviceaccount:
a27284
+			scope = ['https://www.googleapis.com/auth/cloud-platform']
a27284
+			logging.debug("using credentials from service account")
a27284
+			try:
a27284
+				from google.oauth2.service_account import Credentials as ServiceAccountCredentials
a27284
+				credentials = ServiceAccountCredentials.from_service_account_file(filename=serviceaccount, scopes=scope)
a27284
+			except ImportError:
a27284
+				from oauth2client.service_account import ServiceAccountCredentials
a27284
+				credentials = ServiceAccountCredentials.from_json_keyfile_name(serviceaccount, scope)
a27284
+		else:
a27284
+			try:
a27284
+				from googleapiclient import _auth
a27284
+				credentials = _auth.default_credentials();
a27284
+			except:
a27284
+				credentials = GoogleCredentials.get_application_default()
a27284
+			logging.debug("using application default credentials")
a27284
+
a27284
+		if options.get("--proxyhost") and options.get("--proxyport"):
a27284
+			proxy_info = httplib2.ProxyInfo(
a27284
+				proxy_type=socks.PROXY_TYPE_HTTP,
a27284
+				proxy_host=options.get("--proxyhost"),
a27284
+				proxy_port=int(options.get("--proxyport")))
a27284
+			http = credentials.authorize(httplib2.Http(proxy_info=proxy_info))
a27284
+			conn = googleapiclient.discovery.build(
a27284
+				'compute', 'v1', http=http, cache_discovery=False)
a27284
+		else:
a27284
+			conn = googleapiclient.discovery.build(
a27284
+				'compute', 'v1', credentials=credentials, cache_discovery=False)
a27284
 	except Exception as err:
a27284
-		fail_usage("Failed: Create GCE compute v1 connection: {}".format(str(err)))
a27284
+		fail_fence_agent(options, "Failed: Create GCE compute v1 connection: {}".format(str(err)))
a27284
 
a27284
 	# Get project and zone
a27284
 	if not options.get("--project"):
a27284
 		try:
a27284
 			options["--project"] = get_metadata('project/project-id')
a27284
 		except Exception as err:
a27284
-			fail_usage("Failed retrieving GCE project. Please provide --project option: {}".format(str(err)))
a27284
+			fail_fence_agent(options, "Failed retrieving GCE project. Please provide --project option: {}".format(str(err)))
a27284
 
a27284
-	if not options.get("--zone"):
a27284
-		try:
a27284
-			options["--zone"] = get_zone(conn, options['--project'], options['--plug'])
a27284
-		except Exception as err:
a27284
-			fail_usage("Failed retrieving GCE zone. Please provide --zone option: {}".format(str(err)))
a27284
+	if "--baremetalsolution" in options:
a27284
+		options["--zone"] = "none"
a27284
+
a27284
+	# Populates zone automatically if missing from the command
a27284
+	zones = [] if not "--zone" in options else options["--zone"].split(",")
a27284
+	options["--plugzonemap"] = {}
a27284
+	if "--plug" in options:
a27284
+		for i, instance in enumerate(options["--plug"].split(",")):
a27284
+			if len(zones) == 1:
a27284
+				# If only one zone is specified, use it across all plugs
a27284
+				options["--plugzonemap"][instance] = zones[0]
a27284
+				continue
a27284
+
a27284
+			if len(zones) - 1 >= i:
a27284
+				# If we have enough zones specified with the --zone flag use the zone at
a27284
+				# the same index as the plug
a27284
+				options["--plugzonemap"][instance] = zones[i]
a27284
+				continue
a27284
+
a27284
+			try:
a27284
+				# In this case we do not have a zone specified so we attempt to detect it
a27284
+				options["--plugzonemap"][instance] = get_zone(conn, options, instance)
a27284
+			except Exception as err:
a27284
+				fail_fence_agent(options, "Failed retrieving GCE zone. Please provide --zone option: {}".format(str(err)))
a27284
 
a27284
 	# Operate the fencing device
a27284
 	result = fence_action(conn, options, set_power_status, get_power_status, get_nodes_list, power_cycle)