From ebea4c3620261c529cad908c0e52064df84b0c61 Mon Sep 17 00:00:00 2001
From: Oyvind Albrigtsen <oalbrigt@redhat.com>
Date: Mon, 11 Jul 2022 10:28:11 +0200
Subject: [PATCH] openstack-agents: warn when openstackcli is slow
---
heartbeat/openstack-cinder-volume | 19 +++++++++++--------
heartbeat/openstack-common.sh | 22 ++++++++++++++++++++++
heartbeat/openstack-floating-ip | 17 ++++++++++-------
heartbeat/openstack-info.in | 20 ++++++++++----------
heartbeat/openstack-virtual-ip | 20 ++++++++++----------
5 files changed, 63 insertions(+), 35 deletions(-)
diff --git a/heartbeat/openstack-cinder-volume b/heartbeat/openstack-cinder-volume
index 19bf04faf..116442c41 100755
--- a/heartbeat/openstack-cinder-volume
+++ b/heartbeat/openstack-cinder-volume
@@ -113,11 +113,14 @@ _get_node_id() {
}
osvol_validate() {
+ local result
+
check_binary "$OCF_RESKEY_openstackcli"
get_config
- if ! $OCF_RESKEY_openstackcli volume list|grep -q $OCF_RESKEY_volume_id ; then
+ result=$(run_openstackcli "volume list")
+ if ! echo "$result" | grep -q $OCF_RESKEY_volume_id; then
ocf_exit_reason "volume-id $OCF_RESKEY_volume_id not found"
return $OCF_ERR_CONFIGURED
fi
@@ -156,17 +159,17 @@ osvol_monitor() {
# Is the volue attached?
# We use the API
#
- result=$($OCF_RESKEY_openstackcli volume show \
+ result=$(run_openstackcli "volume show \
--column status \
--column attachments \
--format value \
- $OCF_RESKEY_volume_id)
+ $OCF_RESKEY_volume_id")
- if echo "$result" | grep -q available ; then
+ if echo "$result" | grep -q available; then
ocf_log warn "$OCF_RESKEY_volume_id is not attached to any instance"
return $OCF_NOT_RUNNING
else
- export attached_server_id=$(echo $result|head -n1|
+ export attached_server_id=$(echo "$result"|head -n1|
grep -P -o "'server_id': '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}'"|
grep -P -o "[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}")
ocf_log info "$OCF_RESKEY_volume_id is attached to instance $attached_server_id"
@@ -199,7 +202,7 @@ osvol_stop() {
#
# Detach the volume
#
- if ! $OCF_RESKEY_openstackcli server remove volume $node_id $OCF_RESKEY_volume_id ; then
+ if ! run_openstackcli "server remove volume $node_id $OCF_RESKEY_volume_id"; then
ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $node_id"
return $OCF_ERR_GENERIC
fi
@@ -225,7 +228,7 @@ osvol_start() {
# TODO: make it optional in case multi-attachment is allowed by Cinder
#
if [ ! -z $attached_server_id ] ; then
- if ! $OCF_RESKEY_openstackcli server remove volume $attached_server_id $OCF_RESKEY_volume_id ; then
+ if ! run_openstackcli "server remove volume $attached_server_id $OCF_RESKEY_volume_id"; then
ocf_log error "Couldn't remove volume $OCF_RESKEY_volume_id from instance $attached_server_id"
return $OCF_ERR_GENERIC
fi
@@ -238,7 +241,7 @@ osvol_start() {
#
# Attach the volume
#
- $OCF_RESKEY_openstackcli server add volume $node_id $OCF_RESKEY_volume_id
+ run_openstackcli "server add volume $node_id $OCF_RESKEY_volume_id"
if [ $? != $OCF_SUCCESS ]; then
ocf_log error "Couldn't add volume $OCF_RESKEY_volume_id to instance $node_id"
return $OCF_ERR_GENERIC
diff --git a/heartbeat/openstack-common.sh b/heartbeat/openstack-common.sh
index 4763c90db..b6eec09c2 100644
--- a/heartbeat/openstack-common.sh
+++ b/heartbeat/openstack-common.sh
@@ -145,3 +145,25 @@ get_config() {
OCF_RESKEY_openstackcli="${OCF_RESKEY_openstackcli} --os-project-domain-name $OCF_RESKEY_project_domain_name"
fi
}
+
+run_openstackcli() {
+ local cmd="${OCF_RESKEY_openstackcli} $1"
+ local result
+ local rc
+ local start_time=$(date +%s)
+ local end_time
+ local elapsed_time
+
+ result=$($cmd)
+ rc=$?
+ end_time=$(date +%s)
+ elapsed_time=$(expr $end_time - $start_time)
+
+ if [ $elapsed_time -gt 20 ]; then
+ ocf_log warn "$cmd took ${elapsed_time}s to complete"
+ fi
+
+ echo "$result"
+
+ return $rc
+}
diff --git a/heartbeat/openstack-floating-ip b/heartbeat/openstack-floating-ip
index 6e2895654..7317f19a8 100755
--- a/heartbeat/openstack-floating-ip
+++ b/heartbeat/openstack-floating-ip
@@ -101,11 +101,14 @@ END
}
osflip_validate() {
+ local result
+
check_binary "$OCF_RESKEY_openstackcli"
get_config
- if ! $OCF_RESKEY_openstackcli floating ip list|grep -q $OCF_RESKEY_ip_id ; then
+ result=$(run_openstackcli "floating ip list")
+ if ! echo "$result" | grep -q $OCF_RESKEY_ip_id; then
ocf_exit_reason "ip-id $OCF_RESKEY_ip_id not found"
return $OCF_ERR_CONFIGURED
fi
@@ -132,14 +135,14 @@ osflip_monitor() {
| awk '{gsub("[^ ]*:", "");print}')
# Is the IPÂ active and attached?
- result=$($OCF_RESKEY_openstackcli floating ip show \
+ result=$(run_openstackcli "floating ip show \
--column port_id --column floating_ip_address \
--format yaml \
- $OCF_RESKEY_ip_id)
+ $OCF_RESKEY_ip_id")
for port in $node_port_ids ; do
- if echo $result | grep -q $port ; then
- floating_ip=$(echo $result | awk '/floating_ip_address/ {print $2}')
+ if echo "$result" | grep -q $port ; then
+ floating_ip=$(echo "$result" | awk '/floating_ip_address/ {print $2}')
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_floating_ip -v $floating_ip
return $OCF_SUCCESS
@@ -160,7 +163,7 @@ osflip_stop() {
return $OCF_SUCCESS
fi
- if ! $OCF_RESKEY_openstackcli floating ip unset --port $OCF_RESKEY_ip_id ; then
+ if ! run_openstackcli "floating ip unset --port $OCF_RESKEY_ip_id"; then
return $OCF_ERR_GENERIC
fi
@@ -194,7 +197,7 @@ osflip_start() {
ocf_log info "Moving IP address $OCF_RESKEY_ip_id to port ID $node_port_id"
- $OCF_RESKEY_openstackcli floating ip set --port $node_port_id $OCF_RESKEY_ip_id
+ run_openstackcli "floating ip set --port $node_port_id $OCF_RESKEY_ip_id"
if [ $? != $OCF_SUCCESS ]; then
ocf_log error "$OCF_RESKEY_ip_id Cannot be set to port $node_port_id"
return $OCF_ERR_GENERIC
diff --git a/heartbeat/openstack-info.in b/heartbeat/openstack-info.in
index f3a59fc7a..6502f1df1 100755
--- a/heartbeat/openstack-info.in
+++ b/heartbeat/openstack-info.in
@@ -119,9 +119,7 @@ END
#######################################################################
OSInfoStats() {
- local result
local value
- local node
local node_id
get_config
@@ -141,31 +139,33 @@ OSInfoStats() {
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_id -v "$node_id"
# Nova data: flavor
- value=$($OCF_RESKEY_openstackcli server show \
+ value=$(run_openstackcli "server show \
--format value \
--column flavor \
- $node_id)
+ $node_id")
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_flavor -v "$value"
# Nova data: availability zone
- value=$($OCF_RESKEY_openstackcli server show \
+ value=$(run_openstackcli "server show \
--format value \
--column OS-EXT-AZ:availability_zone \
- $node_id)
+ $node_id")
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -n openstack_az -v "$value"
# Network data: ports
value=""
- for port_id in $($OCF_RESKEY_openstackcli port list \
+ for port_id in $(run_openstackcli "port list \
--format value \
--column id \
- --server $node_id); do
- subnet_id=$($OCF_RESKEY_openstackcli port show \
+ --server $node_id"); do
+ subnet_result=$(run_openstackcli "port show \
--format json \
--column fixed_ips \
- ${port_id} | grep -P '\"subnet_id\": \".*\",$' |
+ ${port_id}")
+ subnet_id=$(echo "$subnet_result" |
+ grep -P '\"subnet_id\": \".*\",$' |
grep -P -o '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}')
value="${value}${subnet_id}:${port_id},"
done
diff --git a/heartbeat/openstack-virtual-ip b/heartbeat/openstack-virtual-ip
index c654d980a..361357d55 100755
--- a/heartbeat/openstack-virtual-ip
+++ b/heartbeat/openstack-virtual-ip
@@ -132,11 +132,11 @@ osvip_monitor() {
node_port_id=$(osvip_port_id)
- result=$($OCF_RESKEY_openstackcli port show \
+ result=$(run_openstackcli "port show \
--format value \
--column allowed_address_pairs \
- ${node_port_id})
- if echo $result | grep -q "$OCF_RESKEY_ip"; then
+ ${node_port_id}")
+ if echo "$result" | grep -q "$OCF_RESKEY_ip"; then
${HA_SBIN_DIR}/attrd_updater ${OCF_RESKEY_delay} -S status -n openstack_virtual_ip -v $OCF_RESKEY_ip
return $OCF_SUCCESS
@@ -158,20 +158,20 @@ osvip_stop() {
return $OCF_SUCCESS
fi
- mac_address=$($OCF_RESKEY_openstackcli port show \
+ mac_address=$(run_openstackcli "port show \
--format value \
--column mac_address \
- $node_port_id)
- echo ${mac_address} | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
+ $node_port_id")
+ echo "${mac_address}" | grep -q -P "^([0-9a-f]{2}:){5}[0-9a-f]{2}$"
if [ $? -ne 0 ]; then
ocf_log error "MAC address '${mac_address}' is not valid."
return $OCF_ERR_GENERIC
fi
- if ! $OCF_RESKEY_openstackcli port unset \
+ if ! run_openstackcli "port unset \
--allowed-address \
ip-address=$OCF_RESKEY_ip,mac-address=${mac_address} \
- $node_port_id; then
+ $node_port_id"; then
return $OCF_ERR_GENERIC
fi
@@ -196,9 +196,9 @@ osvip_start() {
ocf_log info "Moving IP address $OCF_RESKEY_ip to port ID $node_port_id"
- $OCF_RESKEY_openstackcli port set \
+ run_openstackcli "port set \
--allowed-address ip-address=$OCF_RESKEY_ip \
- $node_port_id
+ $node_port_id"
if [ $? != $OCF_SUCCESS ]; then
ocf_log error "$OCF_RESKEY_ip Cannot be set to port $node_port_id"
return $OCF_ERR_GENERIC