--- ClusterLabs-resource-agents-e711383f/heartbeat/aws-vpc-route53.in	2018-06-29 14:05:02.000000000 +0200

+++ /home/oalbrigt/src/resource-agents/heartbeat/aws-vpc-route53.in	2019-11-07 12:24:18.822111495 +0100

@@ -152,9 +152,15 @@

 END

 }

-ec2ip_validate() {

+r53_validate() {

 	ocf_log debug "function: validate"

+	# Check for required binaries

+	ocf_log debug "Checking for required binaries"

+	for command in curl dig; do

+		check_binary "$command"

+	done

+

 	# Full name

 	[[ -z "$OCF_RESKEY_fullname" ]] && ocf_log error "Full name parameter not set $OCF_RESKEY_fullname!" && exit $OCF_ERR_CONFIGURED

@@ -175,32 +181,111 @@

 	ocf_log debug "ok"

 	if [ -n "$OCF_RESKEY_profile" ]; then

-		AWS_PROFILE_OPT="--profile $OCF_RESKEY_profile"

+		AWS_PROFILE_OPT="--profile $OCF_RESKEY_profile --cli-connect-timeout 10"

 	else

-		AWS_PROFILE_OPT="--profile default"

+		AWS_PROFILE_OPT="--profile default --cli-connect-timeout 10"

 	fi

 	return $OCF_SUCCESS

 }

-ec2ip_monitor() {

-	ec2ip_validate

+r53_monitor() {

+	#

+	# For every start action the agent  will call Route53 API to check for DNS record

+	# otherwise it will try to get results directly bu querying the DNS using "dig".

+	# Due to complexity in some DNS architectures "dig" can fail, and if this happens

+	# the monitor will fallback to the Route53 API call.

+	#

+	# There will be no failure, failover or restart of the agent if the monitor operation fails

+	# hence we only return $OCF_SUCESS in this function

+	#

+	# In case of the monitor operation detects a wrong or non-existent Route53 DNS entry

+	# it will try to fix the existing one, or create it again

+	#

+	#

+	ARECORD=""

+	IPREGEX="^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"

+	r53_validate

 	ocf_log debug "Checking Route53 record sets"

-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"

-	ARECORD="$(aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query "ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']" | grep RESOURCERECORDS | /usr/bin/awk '{ print $2 }' )"

-	ocf_log debug "Found IP address: $ARECORD ."

-	if [ "${ARECORD}" == "${IPADDRESS}" ]; then

-		ocf_log debug "ARECORD $ARECORD found"

+	#

+	IPADDRESS="$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)"

+	#

+	if [ "$__OCF_ACTION" = "start" ] || ocf_is_probe ; then

+		#

+		cmd="aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']"

+		ocf_log info "Route53 Agent Starting or probing - executing monitoring API call: $cmd"

+		CLIRES="$($cmd 2>&1)"

+		rc=$?

+		ocf_log debug "awscli returned code: $rc"

+		if [ $rc -ne 0 ]; then

+			CLIRES=$(echo $CLIRES | grep -v '^$')

+			ocf_log warn "Route53 API returned an error: $CLIRES"

+			ocf_log warn "Skipping cluster action due to API call error"

+			return $OCF_ERR_GENERIC

+		fi

+		ARECORD=$(echo $CLIRES | grep RESOURCERECORDS | awk '{ print $5 }')

+		#

+		if ocf_is_probe; then

+			#

+			# Prevent R53 record change during probe

+			#

+			if [[ $ARECORD =~ $IPREGEX ]] && [ "$ARECORD" != "$IPADDRESS" ]; then

+				ocf_log debug "Route53 DNS record $ARECORD found at probing, disregarding"

+				return $OCF_NOT_RUNNING

+			fi

+		fi

+	else

+		#

+		cmd="dig +retries=3 +time=5 +short $OCF_RESKEY_fullname 2>/dev/null"

+		ocf_log info "executing monitoring command : $cmd"

+		ARECORD="$($cmd)"

+		rc=$?

+		ocf_log debug "dig return code: $rc"

+		#

+		if  [[ ! $ARECORD =~ $IPREGEX ]] || [ $rc -ne 0 ]; then

+			ocf_log info "Fallback to Route53 API query due to DNS resolution failure"

+			cmd="aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']"

+			ocf_log debug "executing monitoring API call: $cmd"

+			CLIRES="$($cmd 2>&1)"

+			rc=$?

+			ocf_log debug "awscli return code: $rc"

+			if [ $rc -ne 0 ]; then

+				CLIRES=$(echo $CLIRES | grep -v '^$')

+				ocf_log warn "Route53 API returned an error: $CLIRES"

+				ocf_log warn "Monitor skipping cluster action due to API call error"

+				return $OCF_SUCCESS

+			fi

+			ARECORD=$(echo $CLIRES | grep RESOURCERECORDS | awk '{ print $5 }')

+		fi

+		#

+	fi

+	ocf_log info "Route53 DNS record pointing $OCF_RESKEY_fullname to IP address $ARECORD"

+	#

+	if [ "$ARECORD" == "$IPADDRESS" ]; then

+		ocf_log info "Route53 DNS record $ARECORD found"

+		return $OCF_SUCCESS

+	elif [[ $ARECORD =~ $IPREGEX ]] && [ "$ARECORD" != "$IPADDRESS" ]; then

+		ocf_log info "Route53 DNS record points to a different host, setting DNS record on Route53 to this host"

+		_update_record "UPSERT" "$IPADDRESS"

 		return $OCF_SUCCESS

 	else

-		ocf_log debug "No ARECORD found"

-		return $OCF_NOT_RUNNING

+		ocf_log info "No Route53 DNS record found, setting DNS record on Route53 to this host"

+		_update_record "UPSERT" "$IPADDRESS"

+		return $OCF_SUCCESS

 	fi

 	return $OCF_SUCCESS

 }

 _update_record() {

+	#

+	# This function is the one that will actually execute Route53's API call

+	# and configure the DNS record using the correct API calls and parameters

+	#

+	# It creates a temporary JSON file under /tmp with the required API payload

+	#

+	# Failures in this function are critical and will cause the agent to fail

+	#

 	update_action="$1"

 	IPADDRESS="$2"

 	ocf_log info "Updating Route53 $OCF_RESKEY_hostedzoneid with $IPADDRESS for $OCF_RESKEY_fullname"

@@ -209,19 +294,19 @@

 		ocf_exit_reason "Failed to create temporary file for record update"

 		exit $OCF_ERR_GENERIC

 	fi

-	cat >>"${ROUTE53RECORD}" <<-EOF

+	cat >>"$ROUTE53RECORD" <<-EOF

 	{

 		  "Comment": "Update record to reflect new IP address for a system ",

 		  "Changes": [

 			  {

-				  "Action": "${update_action}",

+				  "Action": "$update_action",

 				  "ResourceRecordSet": {

-					  "Name": "${OCF_RESKEY_fullname}",

+					  "Name": "$OCF_RESKEY_fullname",

 					  "Type": "A",

-					  "TTL": ${OCF_RESKEY_ttl},

+					  "TTL": $OCF_RESKEY_ttl,

 					  "ResourceRecords": [

 						  {

-							  "Value": "${IPADDRESS}"

+							  "Value": "$IPADDRESS"

 						  }

 					  ]

 				  }

@@ -229,46 +314,53 @@

 		  ]

 	}

 	EOF

-	cmd="aws --profile ${OCF_RESKEY_profile} route53 change-resource-record-sets --hosted-zone-id ${OCF_RESKEY_hostedzoneid} \

-	  --change-batch file://${ROUTE53RECORD} "

+	cmd="aws --profile $OCF_RESKEY_profile route53 change-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --change-batch file://$ROUTE53RECORD "

 	ocf_log debug "Executing command: $cmd"

-	CHANGEID=$($cmd | grep CHANGEINFO |	 /usr/bin/awk -F'\t' '{ print $3 }' )

-	ocf_log debug "Change id: ${CHANGEID}"

-	rmtempfile ${ROUTE53RECORD}

-	CHANGEID=$(echo $CHANGEID |cut -d'/' -f 3 |cut -d'"' -f 1 )

-	ocf_log debug "Change id: ${CHANGEID}"

+	CLIRES="$($cmd 2>&1)"

+	rc=$?

+	ocf_log debug "awscli returned code: $rc"

+	if [ $rc -ne 0 ]; then

+		CLIRES=$(echo $CLIRES | grep -v '^$')

+		ocf_log warn "Route53 API returned an error: $CLIRES"

+		ocf_log warn "Skipping cluster action due to API call error"

+		return $OCF_ERR_GENERIC

+	fi

+	CHANGEID=$(echo $CLIRES | awk '{ print $12 }')

+	ocf_log debug "Change id: $CHANGEID"

+	rmtempfile $ROUTE53RECORD

+	CHANGEID=$(echo $CHANGEID | cut -d'/' -f 3 | cut -d'"' -f 1 )

+	ocf_log debug "Change id: $CHANGEID"

 	STATUS="PENDING"

-	MYSECONDS=2

+	MYSECONDS=20

 	while [ "$STATUS" = 'PENDING' ]; do

-		sleep	${MYSECONDS}

-		STATUS="$(aws --profile ${OCF_RESKEY_profile} route53 get-change --id $CHANGEID | grep CHANGEINFO |  /usr/bin/awk -F'\t' '{ print $4 }' |cut -d'"' -f 2 )"

-		ocf_log debug "Waited for ${MYSECONDS} seconds and checked execution of Route 53 update status: ${STATUS} "

+		sleep $MYSECONDS

+		STATUS="$(aws --profile $OCF_RESKEY_profile route53 get-change --id $CHANGEID | grep CHANGEINFO | awk -F'\t' '{ print $4 }' |cut -d'"' -f 2 )"

+		ocf_log debug "Waited for $MYSECONDS seconds and checked execution of Route 53 update status: $STATUS "

 	done

 }

-ec2ip_stop() {

-	ocf_log info "Bringing down Route53 agent. (Will remove ARECORD)"

-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"

-	ARECORD="$(aws $AWS_PROFILE_OPT route53 list-resource-record-sets --hosted-zone-id $OCF_RESKEY_hostedzoneid --query "ResourceRecordSets[?Name=='$OCF_RESKEY_fullname']" | grep RESOURCERECORDS | /usr/bin/awk '{ print $2 }' )"

-	ocf_log debug "Found IP address: $ARECORD ."

-	if [ ${ARECORD} != ${IPADDRESS} ]; then

-		ocf_log debug "No ARECORD found"

-		return $OCF_SUCCESS

-	else

-		# determine IP address

-		IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"

-		# Patch file

-		ocf_log debug "Deleting IP address to ${IPADDRESS}"

-		return $OCF_SUCCESS

-	fi

-

-	_update_record "DELETE" "$IPADDRESS"

+r53_stop() {

+	#

+	# Stop operation doesn't perform any API call or try to remove the DNS record

+	# this mostly because this is not necessarily mandatory or desired

+	# the start and monitor functions will take care of changing the DNS record

+	# if the agent starts in a different cluster node

+	#

+	ocf_log info "Bringing down Route53 agent. (Will NOT remove Route53 DNS record)"

 	return $OCF_SUCCESS

 }

-ec2ip_start() {

-	IPADDRESS="$(ec2metadata aws ip | grep local-ipv4 | /usr/bin/awk '{ print $2 }')"

-	_update_record "UPSERT" "$IPADDRESS"

+r53_start() {

+	#

+	# Start agent and config DNS in Route53

+	#

+	ocf_log info "Starting Route53 DNS update...."

+	IPADDRESS="$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)"

+	r53_monitor

+	if [ $? != $OCF_SUCCESS ]; then

+		ocf_log info "Could not start agent - check configurations"

+		return $OCF_ERR_GENERIC

+	fi

 	return $OCF_SUCCESS

 }

@@ -284,16 +376,16 @@

 		exit $OCF_SUCCESS

 		;;

 	monitor)

-		ec2ip_monitor

+		r53_monitor

 		;;

 	stop)

-		ec2ip_stop

+		r53_stop

 		;;

 	validate-all)

-		ec2ip_validate

+		r53_validate

 		;;

 	start)

-		ec2ip_start

+		r53_start

 		;;

 	*)

 		usage