|
|
20333d |
From c1a4637e0a6854fd67438bde2dfa7489cbc9e9c4 Mon Sep 17 00:00:00 2001
|
|
|
20333d |
From: Yonit Halperin <yhalperi@redhat.com>
|
|
|
20333d |
Date: Wed, 14 Aug 2013 10:10:37 -0400
|
|
|
20333d |
Subject: [PATCH] red_channel: add option to monitor whether a channel client
|
|
|
20333d |
is alive
|
|
|
20333d |
|
|
|
20333d |
rhbz#994175
|
|
|
20333d |
|
|
|
20333d |
When a client connection is closed surprisingly (i.e., without a FIN
|
|
|
20333d |
segment), we cannot identify it by a socket error (which is the only
|
|
|
20333d |
way by which we identified disconnections so far).
|
|
|
20333d |
This patch allows a channel client to periodically check the state of
|
|
|
20333d |
the connection and identify surprise disconnections.
|
|
|
20333d |
|
|
|
20333d |
https://bugzilla.redhat.com/show_bug.cgi?id=1016790
|
|
|
20333d |
(cherry picked from commit c8b808bb8211e756a5e2280da173010984b71680)
|
|
|
20333d |
---
|
|
|
20333d |
server/red_channel.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
|
20333d |
server/red_channel.h | 14 ++++++
|
|
|
20333d |
2 files changed, 133 insertions(+)
|
|
|
20333d |
|
|
|
20333d |
diff --git a/server/red_channel.c b/server/red_channel.c
|
|
|
20333d |
index bc6ac8d..5df8f14 100644
|
|
|
20333d |
--- a/server/red_channel.c
|
|
|
20333d |
+++ b/server/red_channel.c
|
|
|
20333d |
@@ -57,6 +57,13 @@ enum QosPingState {
|
|
|
20333d |
PING_STATE_LATENCY,
|
|
|
20333d |
};
|
|
|
20333d |
|
|
|
20333d |
+enum ConnectivityState {
|
|
|
20333d |
+ CONNECTIVITY_STATE_CONNECTED,
|
|
|
20333d |
+ CONNECTIVITY_STATE_BLOCKED,
|
|
|
20333d |
+ CONNECTIVITY_STATE_WAIT_PONG,
|
|
|
20333d |
+ CONNECTIVITY_STATE_DISCONNECTED,
|
|
|
20333d |
+};
|
|
|
20333d |
+
|
|
|
20333d |
static void red_channel_client_start_ping_timer(RedChannelClient *rcc, uint32_t timeout);
|
|
|
20333d |
static void red_channel_client_cancel_ping_timer(RedChannelClient *rcc);
|
|
|
20333d |
static void red_channel_client_restart_ping_timer(RedChannelClient *rcc);
|
|
|
20333d |
@@ -66,6 +73,7 @@ static void red_client_add_channel(RedClient *client, RedChannelClient *rcc);
|
|
|
20333d |
static void red_client_remove_channel(RedChannelClient *rcc);
|
|
|
20333d |
static RedChannelClient *red_client_get_channel(RedClient *client, int type, int id);
|
|
|
20333d |
static void red_channel_client_restore_main_sender(RedChannelClient *rcc);
|
|
|
20333d |
+static inline int red_channel_client_waiting_for_ack(RedChannelClient *rcc);
|
|
|
20333d |
|
|
|
20333d |
/*
|
|
|
20333d |
* Lifetime of RedChannel, RedChannelClient and RedClient:
|
|
|
20333d |
@@ -382,11 +390,19 @@ static void red_channel_client_on_output(void *opaque, int n)
|
|
|
20333d |
{
|
|
|
20333d |
RedChannelClient *rcc = opaque;
|
|
|
20333d |
|
|
|
20333d |
+ if (rcc->connectivity_monitor.timer) {
|
|
|
20333d |
+ rcc->connectivity_monitor.out_bytes += n;
|
|
|
20333d |
+ }
|
|
|
20333d |
stat_inc_counter(rcc->channel->out_bytes_counter, n);
|
|
|
20333d |
}
|
|
|
20333d |
|
|
|
20333d |
static void red_channel_client_on_input(void *opaque, int n)
|
|
|
20333d |
{
|
|
|
20333d |
+ RedChannelClient *rcc = opaque;
|
|
|
20333d |
+
|
|
|
20333d |
+ if (rcc->connectivity_monitor.timer) {
|
|
|
20333d |
+ rcc->connectivity_monitor.in_bytes += n;
|
|
|
20333d |
+ }
|
|
|
20333d |
}
|
|
|
20333d |
|
|
|
20333d |
static void red_channel_client_default_peer_on_error(RedChannelClient *rcc)
|
|
|
20333d |
@@ -743,6 +759,97 @@ static void red_channel_client_ping_timer(void *opaque)
|
|
|
20333d |
}
|
|
|
20333d |
}
|
|
|
20333d |
|
|
|
20333d |
+/*
|
|
|
20333d |
+ * When a connection is not alive (and we can't detect it via a socket error), we
|
|
|
20333d |
+ * reach one of these 2 states:
|
|
|
20333d |
+ * (1) Sending msgs is blocked: either writes return EAGAIN
|
|
|
20333d |
+ * or we are missing MSGC_ACK from the client.
|
|
|
20333d |
+ * (2) MSG_PING was sent without receiving a MSGC_PONG in reply.
|
|
|
20333d |
+ *
|
|
|
20333d |
+ * The connectivity_timer callback tests if the channel's state matches one of the above.
|
|
|
20333d |
+ * In case it does, on the next time the timer is called, it checks if the connection has
|
|
|
20333d |
+ * been idle during the time that passed since the previous timer call. If the connection
|
|
|
20333d |
+ * has been idle, we consider the client as disconnected.
|
|
|
20333d |
+ */
|
|
|
20333d |
+static void red_channel_client_connectivity_timer(void *opaque)
|
|
|
20333d |
+{
|
|
|
20333d |
+ RedChannelClient *rcc = opaque;
|
|
|
20333d |
+ RedChannelClientConnectivityMonitor *monitor = &rcc->connectivity_monitor;
|
|
|
20333d |
+ int is_alive = TRUE;
|
|
|
20333d |
+
|
|
|
20333d |
+ if (monitor->state == CONNECTIVITY_STATE_BLOCKED) {
|
|
|
20333d |
+ if (monitor->in_bytes == 0 && monitor->out_bytes == 0) {
|
|
|
20333d |
+ if (!rcc->send_data.blocked && !red_channel_client_waiting_for_ack(rcc)) {
|
|
|
20333d |
+ spice_error("mismatch between rcc-state and connectivity-state");
|
|
|
20333d |
+ }
|
|
|
20333d |
+ spice_debug("rcc is blocked; connection is idle");
|
|
|
20333d |
+ is_alive = FALSE;
|
|
|
20333d |
+ }
|
|
|
20333d |
+ } else if (monitor->state == CONNECTIVITY_STATE_WAIT_PONG) {
|
|
|
20333d |
+ if (monitor->in_bytes == 0) {
|
|
|
20333d |
+ if (rcc->latency_monitor.state != PING_STATE_WARMUP &&
|
|
|
20333d |
+ rcc->latency_monitor.state != PING_STATE_LATENCY) {
|
|
|
20333d |
+ spice_error("mismatch between rcc-state and connectivity-state");
|
|
|
20333d |
+ }
|
|
|
20333d |
+ spice_debug("rcc waits for pong; connection is idle");
|
|
|
20333d |
+ is_alive = FALSE;
|
|
|
20333d |
+ }
|
|
|
20333d |
+ }
|
|
|
20333d |
+
|
|
|
20333d |
+ if (is_alive) {
|
|
|
20333d |
+ monitor->in_bytes = 0;
|
|
|
20333d |
+ monitor->out_bytes = 0;
|
|
|
20333d |
+ if (rcc->send_data.blocked || red_channel_client_waiting_for_ack(rcc)) {
|
|
|
20333d |
+ monitor->state = CONNECTIVITY_STATE_BLOCKED;
|
|
|
20333d |
+ } else if (rcc->latency_monitor.state == PING_STATE_WARMUP ||
|
|
|
20333d |
+ rcc->latency_monitor.state == PING_STATE_LATENCY) {
|
|
|
20333d |
+ monitor->state = CONNECTIVITY_STATE_WAIT_PONG;
|
|
|
20333d |
+ } else {
|
|
|
20333d |
+ monitor->state = CONNECTIVITY_STATE_CONNECTED;
|
|
|
20333d |
+ }
|
|
|
20333d |
+ rcc->channel->core->timer_start(rcc->connectivity_monitor.timer,
|
|
|
20333d |
+ rcc->connectivity_monitor.timeout);
|
|
|
20333d |
+ } else {
|
|
|
20333d |
+ monitor->state = CONNECTIVITY_STATE_DISCONNECTED;
|
|
|
20333d |
+ spice_debug("rcc %p has not been responsive for more than %u ms, disconnecting",
|
|
|
20333d |
+ rcc, monitor->timeout);
|
|
|
20333d |
+ red_channel_client_disconnect(rcc);
|
|
|
20333d |
+ }
|
|
|
20333d |
+}
|
|
|
20333d |
+
|
|
|
20333d |
+void red_channel_client_start_connectivity_monitoring(RedChannelClient *rcc, uint32_t timeout_ms)
|
|
|
20333d |
+{
|
|
|
20333d |
+ if (!red_channel_client_is_connected(rcc)) {
|
|
|
20333d |
+ return;
|
|
|
20333d |
+ }
|
|
|
20333d |
+ spice_debug(NULL);
|
|
|
20333d |
+ spice_assert(timeout_ms > 0);
|
|
|
20333d |
+ /*
|
|
|
20333d |
+ * If latency_monitor is not active, we activate it in order to enable
|
|
|
20333d |
+ * periodic ping messages so that we will be be able to identify a disconnected
|
|
|
20333d |
+ * channel-client even if there are no ongoing channel specific messages
|
|
|
20333d |
+ * on this channel.
|
|
|
20333d |
+ */
|
|
|
20333d |
+ if (rcc->latency_monitor.timer == NULL) {
|
|
|
20333d |
+ rcc->latency_monitor.timer = rcc->channel->core->timer_add(
|
|
|
20333d |
+ red_channel_client_ping_timer, rcc);
|
|
|
20333d |
+ if (!rcc->client->during_target_migrate) {
|
|
|
20333d |
+ red_channel_client_start_ping_timer(rcc, PING_TEST_IDLE_NET_TIMEOUT_MS);
|
|
|
20333d |
+ }
|
|
|
20333d |
+ rcc->latency_monitor.roundtrip = -1;
|
|
|
20333d |
+ }
|
|
|
20333d |
+ if (rcc->connectivity_monitor.timer == NULL) {
|
|
|
20333d |
+ rcc->connectivity_monitor.state = CONNECTIVITY_STATE_CONNECTED;
|
|
|
20333d |
+ rcc->connectivity_monitor.timer = rcc->channel->core->timer_add(
|
|
|
20333d |
+ red_channel_client_connectivity_timer, rcc);
|
|
|
20333d |
+ rcc->connectivity_monitor.timeout = timeout_ms;
|
|
|
20333d |
+ if (!rcc->client->during_target_migrate) {
|
|
|
20333d |
+ rcc->channel->core->timer_start(rcc->connectivity_monitor.timer,
|
|
|
20333d |
+ rcc->connectivity_monitor.timeout);
|
|
|
20333d |
+ }
|
|
|
20333d |
+ }
|
|
|
20333d |
+}
|
|
|
20333d |
+
|
|
|
20333d |
RedChannelClient *red_channel_client_create(int size, RedChannel *channel, RedClient *client,
|
|
|
20333d |
RedsStream *stream,
|
|
|
20333d |
int monitor_latency,
|
|
|
20333d |
@@ -843,6 +950,10 @@ static void red_channel_client_seamless_migration_done(RedChannelClient *rcc)
|
|
|
20333d |
if (rcc->latency_monitor.timer) {
|
|
|
20333d |
red_channel_client_start_ping_timer(rcc, PING_TEST_IDLE_NET_TIMEOUT_MS);
|
|
|
20333d |
}
|
|
|
20333d |
+ if (rcc->connectivity_monitor.timer) {
|
|
|
20333d |
+ rcc->channel->core->timer_start(rcc->connectivity_monitor.timer,
|
|
|
20333d |
+ rcc->connectivity_monitor.timeout);
|
|
|
20333d |
+ }
|
|
|
20333d |
}
|
|
|
20333d |
pthread_mutex_unlock(&rcc->client->lock);
|
|
|
20333d |
}
|
|
|
20333d |
@@ -889,6 +1000,10 @@ void red_channel_client_default_migrate(RedChannelClient *rcc)
|
|
|
20333d |
rcc->channel->core->timer_remove(rcc->latency_monitor.timer);
|
|
|
20333d |
rcc->latency_monitor.timer = NULL;
|
|
|
20333d |
}
|
|
|
20333d |
+ if (rcc->connectivity_monitor.timer) {
|
|
|
20333d |
+ rcc->channel->core->timer_remove(rcc->connectivity_monitor.timer);
|
|
|
20333d |
+ rcc->connectivity_monitor.timer = NULL;
|
|
|
20333d |
+ }
|
|
|
20333d |
red_channel_client_pipe_add_type(rcc, PIPE_ITEM_TYPE_MIGRATE);
|
|
|
20333d |
}
|
|
|
20333d |
|
|
|
20333d |
@@ -1736,6 +1851,10 @@ void red_channel_client_disconnect(RedChannelClient *rcc)
|
|
|
20333d |
rcc->channel->core->timer_remove(rcc->latency_monitor.timer);
|
|
|
20333d |
rcc->latency_monitor.timer = NULL;
|
|
|
20333d |
}
|
|
|
20333d |
+ if (rcc->connectivity_monitor.timer) {
|
|
|
20333d |
+ rcc->channel->core->timer_remove(rcc->connectivity_monitor.timer);
|
|
|
20333d |
+ rcc->connectivity_monitor.timer = NULL;
|
|
|
20333d |
+ }
|
|
|
20333d |
red_channel_remove_client(rcc);
|
|
|
20333d |
rcc->channel->channel_cbs.on_disconnect(rcc);
|
|
|
20333d |
}
|
|
|
20333d |
diff --git a/server/red_channel.h b/server/red_channel.h
|
|
|
20333d |
index 64befff..9e54dce 100644
|
|
|
20333d |
--- a/server/red_channel.h
|
|
|
20333d |
+++ b/server/red_channel.h
|
|
|
20333d |
@@ -236,6 +236,14 @@ typedef struct RedChannelClientLatencyMonitor {
|
|
|
20333d |
int64_t roundtrip;
|
|
|
20333d |
} RedChannelClientLatencyMonitor;
|
|
|
20333d |
|
|
|
20333d |
+typedef struct RedChannelClientConnectivityMonitor {
|
|
|
20333d |
+ int state;
|
|
|
20333d |
+ uint32_t out_bytes;
|
|
|
20333d |
+ uint32_t in_bytes;
|
|
|
20333d |
+ uint32_t timeout;
|
|
|
20333d |
+ SpiceTimer *timer;
|
|
|
20333d |
+} RedChannelClientConnectivityMonitor;
|
|
|
20333d |
+
|
|
|
20333d |
struct RedChannelClient {
|
|
|
20333d |
RingItem channel_link;
|
|
|
20333d |
RingItem client_link;
|
|
|
20333d |
@@ -289,6 +297,7 @@ struct RedChannelClient {
|
|
|
20333d |
int wait_migrate_flush_mark;
|
|
|
20333d |
|
|
|
20333d |
RedChannelClientLatencyMonitor latency_monitor;
|
|
|
20333d |
+ RedChannelClientConnectivityMonitor connectivity_monitor;
|
|
|
20333d |
};
|
|
|
20333d |
|
|
|
20333d |
struct RedChannel {
|
|
|
20333d |
@@ -448,6 +457,11 @@ SpiceMarshaller *red_channel_client_switch_to_urgent_sender(RedChannelClient *rc
|
|
|
20333d |
/* returns -1 if we don't have an estimation */
|
|
|
20333d |
int red_channel_client_get_roundtrip_ms(RedChannelClient *rcc);
|
|
|
20333d |
|
|
|
20333d |
+/*
|
|
|
20333d |
+ * Checks periodically if the connection is still alive
|
|
|
20333d |
+ */
|
|
|
20333d |
+void red_channel_client_start_connectivity_monitoring(RedChannelClient *rcc, uint32_t timeout_ms);
|
|
|
20333d |
+
|
|
|
20333d |
void red_channel_pipe_item_init(RedChannel *channel, PipeItem *item, int type);
|
|
|
20333d |
|
|
|
20333d |
// TODO: add back the channel_pipe_add functionality - by adding reference counting
|