Blame SOURCES/bz1759113-aws-vpc-route53-1-update.patch

5c52dd
--- ClusterLabs-resource-agents-e711383f/heartbeat/aws-vpc-route53.in	2018-06-29 14:05:02.000000000 +0200
5c52dd
+++ /home/oalbrigt/src/resource-agents/heartbeat/aws-vpc-route53.in	2019-11-07 12:24:18.822111495 +0100
5c52dd
@@ -152,9 +152,15 @@
5c52dd
 END
5c52dd
 }
5c52dd
 
5c52dd
-ec2ip_validate() {
5c52dd
+r53_validate() {
5c52dd
 	ocf_log debug "function: validate"
5c52dd
 
5c52dd
+	# Check for required binaries
5c52dd
+	ocf_log debug "Checking for required binaries"
5c52dd
+	for command in curl dig; do
5c52dd
+		check_binary "$command"
5c52dd
+	done
5c52dd
+
5c52dd
 	# Full name
5c52dd
 	[[ -z "$OCF_RESKEY_fullname" ]] && ocf_log error "Full name parameter not set $OCF_RESKEY_fullname!" && exit $OCF_ERR_CONFIGURED
5c52dd
 
5c52dd
@@ -175,32 +181,111 @@
5c52dd
 	ocf_log debug "ok"
5c52dd
 
5c52dd
 	if [ -n "$OCF_RESKEY_profile" ]; then
5c52dd
-		AWS_PROFILE_OPT="--profile $OCF_RESKEY_profile"
5c52dd
+		AWS_PROFILE_OPT="--profile $OCF_RESKEY_profile --cli-connect-timeout 10"
5c52dd
 	else
5c52dd
-		AWS_PROFILE_OPT="--profile default"
5c52dd
+		AWS_PROFILE_OPT="--profile default --cli-connect-timeout 10"
5c52dd
 	fi
5c52dd
 
5c52dd
 	return $OCF_SUCCESS
5c52dd
 }
5c52dd
 
5c52dd
-ec2ip_monitor() {
5c52dd
-	ec2ip_validate
5c52dd
+r53_monitor() {
5c52dd
+	#
5c52dd
+	# For every start action the agent  will call Route53 API to check for DNS record
5c52dd
+	# otherwise it will try to get results directly bu querying the DNS using "dig".
5c52dd
+	# Due to complexity in some DNS architectures "dig" can fail, and if this happens
5c52dd
+	# the monitor will fallback to the Route53 API call.
5c52dd
+	#
5c52dd
+	# There will be no failure, failover or restart of the agent if the monitor operation fails
5c52dd
+	# hence we only return $OCF_SUCESS in this function
5c52dd
+	#
5c52dd
+	# In case of the monitor operation detects a wrong or non-existent Route53 DNS entry
5c52dd
+	# it will try to fix the existing one, or create it again
5c52dd
+	#
5c52dd
+	#
5c52dd
+	ARECORD=""
5c52dd
+	IPREGEX="^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"
5c52dd
+	r53_validate
5c52dd
 	ocf_log debug "Checking Route53 record sets"
5c52dd
-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
5c52dd
-	ARECORD="$(aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query "ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']" | grep RESOURCERECORDS | /usr/bin/awk '{ print $2 }' )"
5c52dd
-	ocf_log debug "Found IP address: $ARECORD ."
5c52dd
-	if [ "${ARECORD}" == "${IPADDRESS}" ]; then
5c52dd
-		ocf_log debug "ARECORD $ARECORD found"
5c52dd
+	#
5c52dd
+	IPADDRESS="$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)"
5c52dd
+	#
5c52dd
+	if [ "$__OCF_ACTION" = "start" ] || ocf_is_probe ; then
5c52dd
+		#
5c52dd
+		cmd="aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']"
5c52dd
+		ocf_log info "Route53 Agent Starting or probing - executing monitoring API call: $cmd"
5c52dd
+		CLIRES="$($cmd 2>&1)"
5c52dd
+		rc=$?
5c52dd
+		ocf_log debug "awscli returned code: $rc"
5c52dd
+		if [ $rc -ne 0 ]; then
5c52dd
+			CLIRES=$(echo $CLIRES | grep -v '^$')
5c52dd
+			ocf_log warn "Route53 API returned an error: $CLIRES"
5c52dd
+			ocf_log warn "Skipping cluster action due to API call error"
5c52dd
+			return $OCF_ERR_GENERIC
5c52dd
+		fi
5c52dd
+		ARECORD=$(echo $CLIRES | grep RESOURCERECORDS | awk '{ print $5 }')
5c52dd
+		#
5c52dd
+		if ocf_is_probe; then
5c52dd
+			#
5c52dd
+			# Prevent R53 record change during probe
5c52dd
+			#
5c52dd
+			if [[ $ARECORD =~ $IPREGEX ]] && [ "$ARECORD" != "$IPADDRESS" ]; then
5c52dd
+				ocf_log debug "Route53 DNS record $ARECORD found at probing, disregarding"
5c52dd
+				return $OCF_NOT_RUNNING
5c52dd
+			fi
5c52dd
+		fi
5c52dd
+	else
5c52dd
+		#
5c52dd
+		cmd="dig +retries=3 +time=5 +short $OCF_RESKEY_fullname 2>/dev/null"
5c52dd
+		ocf_log info "executing monitoring command : $cmd"
5c52dd
+		ARECORD="$($cmd)"
5c52dd
+		rc=$?
5c52dd
+		ocf_log debug "dig return code: $rc"
5c52dd
+		#
5c52dd
+		if  [[ ! $ARECORD =~ $IPREGEX ]] || [ $rc -ne 0 ]; then
5c52dd
+			ocf_log info "Fallback to Route53 API query due to DNS resolution failure"
5c52dd
+			cmd="aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']"
5c52dd
+			ocf_log debug "executing monitoring API call: $cmd"
5c52dd
+			CLIRES="$($cmd 2>&1)"
5c52dd
+			rc=$?
5c52dd
+			ocf_log debug "awscli return code: $rc"
5c52dd
+			if [ $rc -ne 0 ]; then
5c52dd
+				CLIRES=$(echo $CLIRES | grep -v '^$')
5c52dd
+				ocf_log warn "Route53 API returned an error: $CLIRES"
5c52dd
+				ocf_log warn "Monitor skipping cluster action due to API call error"
5c52dd
+				return $OCF_SUCCESS
5c52dd
+			fi
5c52dd
+			ARECORD=$(echo $CLIRES | grep RESOURCERECORDS | awk '{ print $5 }')
5c52dd
+		fi
5c52dd
+		#
5c52dd
+	fi
5c52dd
+	ocf_log info "Route53 DNS record pointing $OCF_RESKEY_fullname to IP address $ARECORD"
5c52dd
+	#
5c52dd
+	if [ "$ARECORD" == "$IPADDRESS" ]; then
5c52dd
+		ocf_log info "Route53 DNS record $ARECORD found"
5c52dd
+		return $OCF_SUCCESS
5c52dd
+	elif [[ $ARECORD =~ $IPREGEX ]] && [ "$ARECORD" != "$IPADDRESS" ]; then
5c52dd
+		ocf_log info "Route53 DNS record points to a different host, setting DNS record on Route53 to this host"
5c52dd
+		_update_record "UPSERT" "$IPADDRESS"
5c52dd
 		return $OCF_SUCCESS
5c52dd
 	else
5c52dd
-		ocf_log debug "No ARECORD found"
5c52dd
-		return $OCF_NOT_RUNNING
5c52dd
+		ocf_log info "No Route53 DNS record found, setting DNS record on Route53 to this host"
5c52dd
+		_update_record "UPSERT" "$IPADDRESS"
5c52dd
+		return $OCF_SUCCESS
5c52dd
 	fi
5c52dd
 
5c52dd
 	return $OCF_SUCCESS
5c52dd
 }
5c52dd
 
5c52dd
 _update_record() {
5c52dd
+	#
5c52dd
+	# This function is the one that will actually execute Route53's API call
5c52dd
+	# and configure the DNS record using the correct API calls and parameters
5c52dd
+	#
5c52dd
+	# It creates a temporary JSON file under /tmp with the required API payload
5c52dd
+	#
5c52dd
+	# Failures in this function are critical and will cause the agent to fail
5c52dd
+	#
5c52dd
 	update_action="$1"
5c52dd
 	IPADDRESS="$2"
5c52dd
 	ocf_log info "Updating Route53 $OCF_RESKEY_hostedzoneid with $IPADDRESS for $OCF_RESKEY_fullname"
5c52dd
@@ -209,19 +294,19 @@
5c52dd
 		ocf_exit_reason "Failed to create temporary file for record update"
5c52dd
 		exit $OCF_ERR_GENERIC
5c52dd
 	fi
5c52dd
-	cat >>"${ROUTE53RECORD}" <<-EOF
5c52dd
+	cat >>"$ROUTE53RECORD" <<-EOF
5c52dd
 	{
5c52dd
 		  "Comment": "Update record to reflect new IP address for a system ",
5c52dd
 		  "Changes": [
5c52dd
 			  {
5c52dd
-				  "Action": "${update_action}",
5c52dd
+				  "Action": "$update_action",
5c52dd
 				  "ResourceRecordSet": {
5c52dd
-					  "Name": "${OCF_RESKEY_fullname}",
5c52dd
+					  "Name": "$OCF_RESKEY_fullname",
5c52dd
 					  "Type": "A",
5c52dd
-					  "TTL": ${OCF_RESKEY_ttl},
5c52dd
+					  "TTL": $OCF_RESKEY_ttl,
5c52dd
 					  "ResourceRecords": [
5c52dd
 						  {
5c52dd
-							  "Value": "${IPADDRESS}"
5c52dd
+							  "Value": "$IPADDRESS"
5c52dd
 						  }
5c52dd
 					  ]
5c52dd
 				  }
5c52dd
@@ -229,46 +314,53 @@
5c52dd
 		  ]
5c52dd
 	}
5c52dd
 	EOF
5c52dd
-	cmd="aws --profile ${OCF_RESKEY_profile} route53 change-resource-record-sets --hosted-zone-id ${OCF_RESKEY_hostedzoneid} \
5c52dd
-	  --change-batch file://${ROUTE53RECORD} "
5c52dd
+	cmd="aws --profile $OCF_RESKEY_profile route53 change-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --change-batch file://$ROUTE53RECORD "
5c52dd
 	ocf_log debug "Executing command: $cmd"
5c52dd
-	CHANGEID=$($cmd | grep CHANGEINFO |	 /usr/bin/awk -F'\t' '{ print $3 }' )
5c52dd
-	ocf_log debug "Change id: ${CHANGEID}"
5c52dd
-	rmtempfile ${ROUTE53RECORD}
5c52dd
-	CHANGEID=$(echo $CHANGEID |cut -d'/' -f 3 |cut -d'"' -f 1 )
5c52dd
-	ocf_log debug "Change id: ${CHANGEID}"
5c52dd
+	CLIRES="$($cmd 2>&1)"
5c52dd
+	rc=$?
5c52dd
+	ocf_log debug "awscli returned code: $rc"
5c52dd
+	if [ $rc -ne 0 ]; then
5c52dd
+		CLIRES=$(echo $CLIRES | grep -v '^$')
5c52dd
+		ocf_log warn "Route53 API returned an error: $CLIRES"
5c52dd
+		ocf_log warn "Skipping cluster action due to API call error"
5c52dd
+		return $OCF_ERR_GENERIC
5c52dd
+	fi
5c52dd
+	CHANGEID=$(echo $CLIRES | awk '{ print $12 }')
5c52dd
+	ocf_log debug "Change id: $CHANGEID"
5c52dd
+	rmtempfile $ROUTE53RECORD
5c52dd
+	CHANGEID=$(echo $CHANGEID | cut -d'/' -f 3 | cut -d'"' -f 1 )
5c52dd
+	ocf_log debug "Change id: $CHANGEID"
5c52dd
 	STATUS="PENDING"
5c52dd
-	MYSECONDS=2
5c52dd
+	MYSECONDS=20
5c52dd
 	while [ "$STATUS" = 'PENDING' ]; do
5c52dd
-		sleep	${MYSECONDS}
5c52dd
-		STATUS="$(aws --profile ${OCF_RESKEY_profile} route53 get-change --id $CHANGEID | grep CHANGEINFO |  /usr/bin/awk -F'\t' '{ print $4 }' |cut -d'"' -f 2 )"
5c52dd
-		ocf_log debug "Waited for ${MYSECONDS} seconds and checked execution of Route 53 update status: ${STATUS} "
5c52dd
+		sleep $MYSECONDS
5c52dd
+		STATUS="$(aws --profile $OCF_RESKEY_profile route53 get-change --id $CHANGEID | grep CHANGEINFO | awk -F'\t' '{ print $4 }' |cut -d'"' -f 2 )"
5c52dd
+		ocf_log debug "Waited for $MYSECONDS seconds and checked execution of Route 53 update status: $STATUS "
5c52dd
 	done
5c52dd
 }
5c52dd
 
5c52dd
-ec2ip_stop() {
5c52dd
-	ocf_log info "Bringing down Route53 agent. (Will remove ARECORD)"
5c52dd
-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
5c52dd
-	ARECORD="$(aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query "ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']" | grep RESOURCERECORDS | /usr/bin/awk '{ print $2 }' )"
5c52dd
-	ocf_log debug "Found IP address: $ARECORD ."
5c52dd
-	if [ ${ARECORD} != ${IPADDRESS} ]; then
5c52dd
-		ocf_log debug "No ARECORD found"
5c52dd
-		return $OCF_SUCCESS
5c52dd
-	else
5c52dd
-		# determine IP address
5c52dd
-		IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
5c52dd
-		# Patch file
5c52dd
-		ocf_log debug "Deleting IP address to ${IPADDRESS}"
5c52dd
-		return $OCF_SUCCESS
5c52dd
-	fi
5c52dd
-
5c52dd
-	_update_record "DELETE" "$IPADDRESS"
5c52dd
+r53_stop() {
5c52dd
+	#
5c52dd
+	# Stop operation doesn't perform any API call or try to remove the DNS record
5c52dd
+	# this mostly because this is not necessarily mandatory or desired
5c52dd
+	# the start and monitor functions will take care of changing the DNS record
5c52dd
+	# if the agent starts in a different cluster node
5c52dd
+	#
5c52dd
+	ocf_log info "Bringing down Route53 agent. (Will NOT remove Route53 DNS record)"
5c52dd
 	return $OCF_SUCCESS
5c52dd
 }
5c52dd
 
5c52dd
-ec2ip_start() {
5c52dd
-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"
5c52dd
-	_update_record "UPSERT" "$IPADDRESS"
5c52dd
+r53_start() {
5c52dd
+	#
5c52dd
+	# Start agent and config DNS in Route53
5c52dd
+	#
5c52dd
+	ocf_log info "Starting Route53 DNS update...."
5c52dd
+	IPADDRESS="$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)"
5c52dd
+	r53_monitor
5c52dd
+	if [ $? != $OCF_SUCCESS ]; then
5c52dd
+		ocf_log info "Could not start agent - check configurations"
5c52dd
+		return $OCF_ERR_GENERIC
5c52dd
+	fi
5c52dd
 	return $OCF_SUCCESS
5c52dd
 }
5c52dd
 
5c52dd
@@ -284,16 +376,16 @@
5c52dd
 		exit $OCF_SUCCESS
5c52dd
 		;;
5c52dd
 	monitor)
5c52dd
-		ec2ip_monitor
5c52dd
+		r53_monitor
5c52dd
 		;;
5c52dd
 	stop)
5c52dd
-		ec2ip_stop
5c52dd
+		r53_stop
5c52dd
 		;;
5c52dd
 	validate-all)
5c52dd
-		ec2ip_validate
5c52dd
+		r53_validate
5c52dd
 		;;
5c52dd
 	start)
5c52dd
-		ec2ip_start
5c52dd
+		r53_start
5c52dd
 		;;
5c52dd
 	*)
5c52dd
 		usage