From bcba9c27187a1284e9f0834f7e056324a162d80a Mon Sep 17 00:00:00 2001
From: Humble Devassy Chirammal <hchiramm@redhat.com>
Date: Wed, 29 Apr 2015 11:38:11 +0530
Subject: [PATCH 110/129] rdma: porting rdma to a new message id logging format
Change-Id: I71e940817ae0a9378e82332d5a8569114fc13482
BUG: 1231782
Signed-off-by: Humble Devassy Chirammal <hchiramm@redhat.com>
Reviewed-on: http://review.gluster.org/9868
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/51003
Reviewed-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
Tested-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
---
libglusterfs/src/glfs-message-id.h | 2 +-
rpc/rpc-lib/src/Makefile.am | 3 +-
rpc/rpc-lib/src/rpc-lib-messages.h | 83 +++
rpc/rpc-transport/rdma/src/Makefile.am | 2 +-
rpc/rpc-transport/rdma/src/name.c | 149 ++---
rpc/rpc-transport/rdma/src/rdma.c | 698 +++++++++++--------
.../rdma/src/rpc-trans-rdma-messages.h | 217 ++++++
7 files changed, 781 insertions(+), 373 deletions(-)
create mode 100644 rpc/rpc-lib/src/rpc-lib-messages.h
create mode 100644 rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h
diff --git a/libglusterfs/src/glfs-message-id.h b/libglusterfs/src/glfs-message-id.h
index df7caa2..d67115c 100644
--- a/libglusterfs/src/glfs-message-id.h
+++ b/libglusterfs/src/glfs-message-id.h
@@ -1,5 +1,5 @@
/*
- Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
This file is part of GlusterFS.
This file is licensed to you under your choice of the GNU Lesser
diff --git a/rpc/rpc-lib/src/Makefile.am b/rpc/rpc-lib/src/Makefile.am
index 6a098c9..36eb245 100644
--- a/rpc/rpc-lib/src/Makefile.am
+++ b/rpc/rpc-lib/src/Makefile.am
@@ -8,7 +8,8 @@ libgfrpc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
libgfrpc_la_LDFLAGS = -version-info $(LIBGFRPC_LT_VERSION) $(GF_LDFLAGS)
noinst_HEADERS = rpcsvc.h rpc-transport.h xdr-common.h xdr-rpc.h xdr-rpcclnt.h \
- rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h rpc-clnt-ping.h
+ rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h rpc-clnt-ping.h \
+ rpc-lib-messages.h
AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-I$(top_srcdir)/rpc/xdr/src \
diff --git a/rpc/rpc-lib/src/rpc-lib-messages.h b/rpc/rpc-lib/src/rpc-lib-messages.h
new file mode 100644
index 0000000..51351e8
--- /dev/null
+++ b/rpc/rpc-lib/src/rpc-lib-messages.h
@@ -0,0 +1,83 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#ifndef _RPC_LIB_MESSAGES_H_
+#define _RPC_LIB_MESSAGES_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glfs-message-id.h"
+
+/* NOTE: Rules for message additions
+ * 1) Each instance of a message is _better_ left with a unique message ID, even
+ * if the message format is the same. Reasoning is that, if the message
+ * format needs to change in one instance, the other instances are not
+ * impacted or the new change does not change the ID of the instance being
+ * modified.
+ * 2) Addition of a message,
+ * - Should increment the GLFS_NUM_MESSAGES
+ * - Append to the list of messages defined, towards the end
+ * - Retain macro naming as glfs_msg_X (for redability across developers)
+ * NOTE: Rules for message format modifications
+ * 3) Check acorss the code if the message ID macro in question is reused
+ * anywhere. If reused then then the modifications should ensure correctness
+ * everywhere, or needs a new message ID as (1) above was not adhered to. If
+ * not used anywhere, proceed with the required modification.
+ * NOTE: Rules for message deletion
+ * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
+ * anywhere, then can be deleted, but will leave a hole by design, as
+ * addition rules specify modification to the end of the list and not filling
+ * holes.
+ */
+
+#define GLFS_RPC_LIB_BASE GLFS_MSGID_COMP_RPC_LIB
+#define GLFS_NUM_MESSAGES 12
+#define GLFS_RPC_LIB_MSGID_END (GLFS_RPC_LIB_BASE + GLFS_NUM_MESSAGES + 1)
+
+/* Messages with message IDs */
+
+#define glfs_msg_start_x GLFS_RPC_LIB_BASE, "Invalid: Start of messages"
+
+/*------------*/
+/* First slot is allocated for common transport msg ids */
+
+#define TRANS_MSG_ADDR_FAMILY_NOT_SPECIFIED (GLFS_RPC_LIB_BASE + 1)
+
+#define TRANS_MSG_UNKNOWN_ADDR_FAMILY (GLFS_RPC_LIB_BASE + 2)
+
+#define TRANS_MSG_REMOTE_HOST_ERROR (GLFS_RPC_LIB_BASE + 3)
+
+#define TRANS_MSG_DNS_RESOL_FAILED (GLFS_RPC_LIB_BASE + 4)
+
+#define TRANS_MSG_LISTEN_PATH_ERROR (GLFS_RPC_LIB_BASE + 5)
+
+#define TRANS_MSG_CONNECT_PATH_ERROR (GLFS_RPC_LIB_BASE + 6)
+
+#define TRANS_MSG_GET_ADDR_INFO_FAILED (GLFS_RPC_LIB_BASE + 7)
+
+#define TRANS_MSG_PORT_BIND_FAILED (GLFS_RPC_LIB_BASE + 8)
+
+#define TRANS_MSG_INET_ERROR (GLFS_RPC_LIB_BASE + 9)
+
+#define TRANS_MSG_GET_NAME_INFO_FAILED (GLFS_RPC_LIB_BASE + 10)
+
+#define TRANS_MSG_TRANSPORT_ERROR (GLFS_RPC_LIB_BASE + 11)
+
+#define TRANS_MSG_TIMEOUT_EXCEEDED (GLFS_RPC_LIB_BASE + 12)
+
+/*------------*/
+
+#define glfs_msg_end_x GLFS_RPC_LIB_MSGID_END, "Invalid: End of messages"
+
+#endif /* !_RPC_LIB_MESSAGES_H_ */
+
diff --git a/rpc/rpc-transport/rdma/src/Makefile.am b/rpc/rpc-transport/rdma/src/Makefile.am
index fed139e..6526b6c 100644
--- a/rpc/rpc-transport/rdma/src/Makefile.am
+++ b/rpc/rpc-transport/rdma/src/Makefile.am
@@ -9,7 +9,7 @@ rdma_la_SOURCES = rdma.c name.c
rdma_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-libverbs -lrdmacm
-noinst_HEADERS = rdma.h name.h
+noinst_HEADERS = rdma.h name.h rpc-trans-rdma-messages.h
-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src/ \
-I$(top_srcdir)/xlators/protocol/lib/src/ $(GF_CFLAGS)
diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c
index 52ce5bb..97b2e37 100644
--- a/rpc/rpc-transport/rdma/src/name.c
+++ b/rpc/rpc-transport/rdma/src/name.c
@@ -22,6 +22,8 @@
#include "rpc-transport.h"
#include "rdma.h"
#include "common-utils.h"
+#include "rpc-lib-messages.h"
+#include "rpc-trans-rdma-messages.h"
int32_t
@@ -38,7 +40,7 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
{
int32_t ret = -1;
uint16_t port = ceiling - 1;
- // by default assume none of the ports are blocked and all are available
+ /* by default assume none of the ports are blocked and all are available */
gf_boolean_t ports[1024] = {_gf_false,};
int i = 0;
@@ -48,10 +50,8 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
ports[i] = _gf_false;
}
- while (port)
- {
- switch (sockaddr->sa_family)
- {
+ while (port) {
+ switch (sockaddr->sa_family) {
case AF_INET6:
((struct sockaddr_in6 *)sockaddr)->sin6_port
= htons (port);
@@ -63,7 +63,7 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
= htons (port);
break;
}
- // ignore the reserved ports
+ /* ignore the reserved ports */
if (ports[port] == _gf_true) {
port--;
continue;
@@ -134,7 +134,8 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr)
if (!(remote_host_data || connect_path_data) ||
(remote_host_data && connect_path_data)) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_ADDR_FAMILY_NOT_SPECIFIED,
"address-family not specified and not able to "
"determine the same from other options "
"(remote-host:%s and connect-path:%s)",
@@ -144,14 +145,13 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr)
}
if (remote_host_data) {
- gf_log (this->name, GF_LOG_DEBUG,
- "address-family not specified, guessing it "
- "to be inet/inet6");
+ gf_msg_debug (this->name, 0, "address-family not "
+ "specified, guessing it to be "
+ "inet/inet6");
sockaddr->sa_family = AF_UNSPEC;
} else {
- gf_log (this->name, GF_LOG_DEBUG,
- "address-family not specified, guessing it "
- "to be unix");
+ gf_msg_debug (this->name, 0, "address-family not "
+ "specified, guessing it to be unix");
sockaddr->sa_family = AF_UNIX;
}
@@ -166,7 +166,8 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr)
} else if (!strcasecmp (address_family, "inet-sdp")) {
sockaddr->sa_family = AF_INET_SDP;
} else {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_UNKNOWN_ADDR_FAMILY,
"unknown address-family (%s) specified",
address_family);
sockaddr->sa_family = AF_UNSPEC;
@@ -191,47 +192,40 @@ af_inet_client_get_remote_sockaddr (rpc_transport_t *this,
int32_t ret = 0;
remote_host_data = dict_get (options, "remote-host");
- if (remote_host_data == NULL)
- {
- gf_log (this->name, GF_LOG_ERROR,
- "option remote-host missing in volume %s",
- this->name);
+ if (remote_host_data == NULL) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_REMOTE_HOST_ERROR, "option remote-host "
+ "missing in volume %s", this->name);
ret = -1;
goto err;
}
remote_host = data_to_str (remote_host_data);
- if (remote_host == NULL)
- {
- gf_log (this->name, GF_LOG_ERROR,
- "option remote-host has data NULL in volume %s",
- this->name);
+ if (remote_host == NULL) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_REMOTE_HOST_ERROR, "option remote-host "
+ "has data NULL in volume %s", this->name);
ret = -1;
goto err;
}
if (remote_port == 0) {
remote_port_data = dict_get (options, "remote-port");
- if (remote_port_data == NULL)
- {
- gf_log (this->name, GF_LOG_DEBUG,
- "option remote-port missing in volume %s. "
- "Defaulting to %d",
- this->name, GF_DEFAULT_RDMA_LISTEN_PORT);
+ if (remote_port_data == NULL) {
+ gf_msg_debug (this->name, 0, "option remote-port "
+ "missing in volume %s. Defaulting to %d",
+ this->name, GF_DEFAULT_RDMA_LISTEN_PORT);
remote_port = GF_DEFAULT_RDMA_LISTEN_PORT;
- }
- else
- {
+ } else {
remote_port = data_to_uint16 (remote_port_data);
}
}
- if (remote_port == -1)
- {
- gf_log (this->name, GF_LOG_ERROR,
- "option remote-port has invalid port in volume %s",
- this->name);
+ if (remote_port == -1) {
+ gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+ RDMA_MSG_INVALID_ENTRY, "option remote-port has "
+ "invalid port in volume %s", this->name);
ret = -1;
goto err;
}
@@ -242,7 +236,7 @@ af_inet_client_get_remote_sockaddr (rpc_transport_t *this,
sockaddr->sa_family,
&this->dnscache, &addr_info);
if (ret == -1) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0, TRANS_MSG_DNS_RESOL_FAILED,
"DNS resolution failed on host %s", remote_host);
goto err;
}
@@ -267,32 +261,32 @@ af_unix_client_get_remote_sockaddr (rpc_transport_t *this,
connect_path_data = dict_get (this->options,
"transport.rdma.connect-path");
if (!connect_path_data) {
- gf_log (this->name, GF_LOG_ERROR,
- "option transport.rdma.connect-path not "
- "specified for address-family unix");
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_CONNECT_PATH_ERROR, "option "
+ "transport.rdma.connect-path not specified for "
+ "address-family unix");
ret = -1;
goto err;
}
connect_path = data_to_str (connect_path_data);
if (!connect_path) {
- gf_log (this->name, GF_LOG_ERROR,
- "connect-path is null-string");
+ gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+ RDMA_MSG_INVALID_ENTRY, "connect-path is null-string");
ret = -1;
goto err;
}
if (strlen (connect_path) > UNIX_PATH_MAX) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_CONNECT_PATH_ERROR,
"connect-path value length %"GF_PRI_SIZET" > "
"%d octets", strlen (connect_path), UNIX_PATH_MAX);
ret = -1;
goto err;
}
- gf_log (this->name,
- GF_LOG_DEBUG,
- "using connect-path %s", connect_path);
+ gf_msg_debug (this->name, 0, "using connect-path %s", connect_path);
sockaddr_un = (struct sockaddr_un *)sockaddr;
strcpy (sockaddr_un->sun_path, connect_path);
*sockaddr_len = sizeof (struct sockaddr_un);
@@ -315,7 +309,8 @@ af_unix_server_get_local_sockaddr (rpc_transport_t *this,
listen_path_data = dict_get (this->options,
"transport.rdma.listen-path");
if (!listen_path_data) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_LISTEN_PATH_ERROR,
"missing option listen-path");
ret = -1;
goto err;
@@ -328,8 +323,9 @@ af_unix_server_get_local_sockaddr (rpc_transport_t *this,
#endif
if (strlen (listen_path) > UNIX_PATH_MAX) {
- gf_log (this->name, GF_LOG_ERROR,
- "option listen-path has value length %"GF_PRI_SIZET" > %d",
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_LISTEN_PATH_ERROR, "option listen-path has "
+ "value length %"GF_PRI_SIZET" > %d",
strlen (listen_path), UNIX_PATH_MAX);
ret = -1;
goto err;
@@ -399,7 +395,8 @@ af_inet_server_get_local_sockaddr (rpc_transport_t *this,
ret = getaddrinfo(listen_host, service, &hints, &res);
if (ret != 0) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, ret,
+ TRANS_MSG_GET_ADDR_INFO_FAILED,
"getaddrinfo failed for host %s, service %s (%s)",
listen_host, service, gai_strerror (ret));
ret = -1;
@@ -422,8 +419,7 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
int ret = 0;
*sockaddr_len = sizeof (struct sockaddr_in6);
- switch (sockaddr->sa_family)
- {
+ switch (sockaddr->sa_family) {
case AF_INET_SDP:
case AF_INET:
*sockaddr_len = sizeof (struct sockaddr_in);
@@ -433,7 +429,8 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
*sockaddr_len,
GF_CLIENT_PORT_CEILING);
if (ret == -1) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_PORT_BIND_FAILED,
"cannot bind rdma_cm_id to port "
"less than %d (%s)", GF_CLIENT_PORT_CEILING,
strerror (errno));
@@ -457,7 +454,8 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
break;
default:
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_UNKNOWN_ADDR_FAMILY,
"unknown address family %d", sockaddr->sa_family);
ret = -1;
break;
@@ -481,8 +479,7 @@ gf_rdma_client_get_remote_sockaddr (rpc_transport_t *this,
goto err;
}
- switch (sockaddr->sa_family)
- {
+ switch (sockaddr->sa_family) {
case AF_INET_SDP:
sockaddr->sa_family = AF_INET;
is_inet_sdp = 1;
@@ -508,7 +505,8 @@ gf_rdma_client_get_remote_sockaddr (rpc_transport_t *this,
break;
default:
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_UNKNOWN_ADDR_FAMILY,
"unknown address-family %d", sockaddr->sa_family);
ret = -1;
}
@@ -541,22 +539,20 @@ gf_rdma_server_get_local_sockaddr (rpc_transport_t *this,
} else if (!strcasecmp (address_family, "unix")) {
addr->sa_family = AF_UNIX;
} else {
- gf_log (this->name, GF_LOG_ERROR,
- "unknown address family (%s) specified",
- address_family);
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_UNKNOWN_ADDR_FAMILY, "unknown address"
+ " family (%s) specified", address_family);
addr->sa_family = AF_UNSPEC;
ret = -1;
goto err;
}
} else {
- gf_log (this->name, GF_LOG_DEBUG,
- "option address-family not specified, defaulting "
- "to inet");
+ gf_msg_debug (this->name, 0, "option address-family not "
+ "specified, defaulting to inet");
addr->sa_family = AF_INET;
}
- switch (addr->sa_family)
- {
+ switch (addr->sa_family) {
case AF_INET_SDP:
is_inet_sdp = 1;
addr->sa_family = AF_INET;
@@ -639,8 +635,8 @@ fill_inet6_inet_identifiers (rpc_transport_t *this, struct sockaddr_storage *add
service, sizeof (service),
NI_NUMERICHOST | NI_NUMERICSERV);
if (ret != 0) {
- gf_log (this->name,
- GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, ret,
+ TRANS_MSG_GET_NAME_INFO_FAILED,
"getnameinfo failed (%s)", gai_strerror (ret));
}
@@ -655,21 +651,20 @@ gf_rdma_get_transport_identifiers (rpc_transport_t *this)
int32_t ret = 0;
char is_inet_sdp = 0;
- switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family)
- {
+ switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family) {
case AF_INET_SDP:
is_inet_sdp = 1;
((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET;
case AF_INET:
- case AF_INET6:
- {
+ case AF_INET6: {
ret = fill_inet6_inet_identifiers (this,
&this->myinfo.sockaddr,
this->myinfo.sockaddr_len,
this->myinfo.identifier);
if (ret == -1) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_INET_ERROR,
"can't fill inet/inet6 identifier for server");
goto err;
}
@@ -679,7 +674,8 @@ gf_rdma_get_transport_identifiers (rpc_transport_t *this)
this->peerinfo.sockaddr_len,
this->peerinfo.identifier);
if (ret == -1) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_INET_ERROR,
"can't fill inet/inet6 identifier for client");
goto err;
}
@@ -703,7 +699,8 @@ gf_rdma_get_transport_identifiers (rpc_transport_t *this)
break;
default:
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ TRANS_MSG_UNKNOWN_ADDR_FAMILY,
"unknown address family (%d)",
((struct sockaddr *) &this->myinfo.sockaddr)->sa_family);
ret = -1;
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index f62f401..89fb6cb 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -22,6 +22,8 @@
#include "byte-order.h"
#include "xlator.h"
#include "xdr-rpc.h"
+#include "rpc-lib-messages.h"
+#include "rpc-trans-rdma-messages.h"
#include <signal.h>
#define GF_RDMA_LOG_NAME "rpc-transport/rdma"
@@ -116,7 +118,8 @@ gf_rdma_new_post (rpc_transport_t *this, gf_rdma_device_t *device, int32_t len,
post->buf_size,
IBV_ACCESS_LOCAL_WRITE);
if (!post->mr) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_MR_ALOC_FAILED,
"memory registration failed (%s)",
strerror (errno));
goto out;
@@ -324,12 +327,13 @@ __gf_rdma_create_posts (rpc_transport_t *this, int32_t count, int32_t size,
priv = this->private;
device = priv->device;
- for (i=0 ; i<count ; i++) {
+ for (i = 0 ; i < count ; i++) {
gf_rdma_post_t *post = NULL;
post = gf_rdma_new_post (this, device, size + 2048, type);
if (!post) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_POST_CREATION_FAILED,
"post creation failed");
ret = -1;
break;
@@ -399,9 +403,10 @@ gf_rdma_deregister_arena (struct list_head **mr_list,
list_for_each_entry_safe (tmp, dummy, mr_list[i], list) {
if (tmp->iobuf_arena == iobuf_arena) {
if (ibv_dereg_mr(tmp->mr)) {
- gf_log("rdma", GF_LOG_WARNING,
- "deallocation of memory region "
- "failed");
+ gf_msg ("rdma", GF_LOG_WARNING, 0,
+ RDMA_MSG_DEREGISTER_ARENA_FAILED,
+ "deallocation of memory region "
+ "failed");
return -1;
}
list_del(&tmp->list);
@@ -431,8 +436,10 @@ gf_rdma_register_arena (void **arg1, void *arg2)
new = GF_CALLOC(1, sizeof(gf_rdma_arena_mr),
gf_common_mt_rdma_arena_mr);
if (new == NULL) {
- gf_log ("rdma", GF_LOG_INFO, "Out of memory:"
- " registering pre allocated buffer wth rdma device failed.");
+ gf_msg ("rdma", GF_LOG_INFO, ENOMEM,
+ RDMA_MSG_MR_ALOC_FAILED, "Out of "
+ "memory: registering pre allocated buffer "
+ "with rdma device failed.");
return -1;
}
INIT_LIST_HEAD (&new->list);
@@ -445,8 +452,9 @@ gf_rdma_register_arena (void **arg1, void *arg2)
IBV_ACCESS_REMOTE_WRITE
);
if (!mr)
- gf_log("rdma", GF_LOG_WARNING,
- "allocation of mr failed");
+ gf_msg ("rdma", GF_LOG_WARNING, 0,
+ RDMA_MSG_MR_ALOC_FAILED, "allocation of mr "
+ "failed");
new->mr = mr;
list_add (&new->list, &device[i]->all_mr);
@@ -473,8 +481,10 @@ gf_rdma_register_iobuf_pool (gf_rdma_device_t *device,
new = GF_CALLOC(1, sizeof(gf_rdma_arena_mr),
gf_common_mt_rdma_arena_mr);
if (new == NULL) {
- gf_log ("rdma", GF_LOG_INFO, "Out of memory:"
- " registering pre allocated buffer with rdma device failed.");
+ gf_msg ("rdma", GF_LOG_INFO, ENOMEM,
+ RDMA_MSG_MR_ALOC_FAILED, "Out of "
+ "memory: registering pre allocated "
+ "buffer with rdma device failed.");
return;
}
INIT_LIST_HEAD (&new->list);
@@ -486,8 +496,9 @@ gf_rdma_register_iobuf_pool (gf_rdma_device_t *device,
IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE);
if (!mr) {
- gf_log ("rdma", GF_LOG_WARNING, "failed to pre"
- " register buffers with rdma "
+ gf_msg ("rdma", GF_LOG_WARNING, 0,
+ RDMA_MSG_MR_ALOC_FAILED, "failed"
+ " to pre register buffers with rdma "
"devices.");
}
@@ -557,7 +568,7 @@ gf_rdma_create_posts (rpc_transport_t *this)
GF_RDMA_RECV_POST);
if (!ret) {
- for (i=0 ; i<options->recv_count ; i++) {
+ for (i = 0 ; i < options->recv_count ; i++) {
post = gf_rdma_get_post (&device->recvq);
if (gf_rdma_post_recv (device->srq, post) != 0) {
ret = -1;
@@ -614,14 +625,15 @@ gf_rdma_create_cq (rpc_transport_t *this)
device->recv_chan,
0);
if (!device->recv_cq) {
- gf_log (this->name, GF_LOG_ERROR,
- "creation of CQ for device %s failed",
- device->device_name);
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_CQ_CREATION_FAILED, "creation of CQ for "
+ "device %s failed", device->device_name);
ret = -1;
goto out;
} else if (ibv_req_notify_cq (device->recv_cq, 0)) {
- gf_log (this->name, GF_LOG_ERROR,
- "ibv_req_notify_cq on recv CQ of device %s failed",
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_REQ_NOTIFY_CQ_REVQ_FAILED, "ibv_req_notify_"
+ "cq on recv CQ of device %s failed",
device->device_name);
ret = -1;
goto out;
@@ -630,8 +642,9 @@ gf_rdma_create_cq (rpc_transport_t *this)
do {
ret = ibv_query_device (priv->device->context, &device_attr);
if (ret != 0) {
- gf_log (this->name, GF_LOG_ERROR,
- "ibv_query_device on %s returned %d (%s)",
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_QUERY_DEVICE_FAILED, "ibv_query_"
+ "device on %s returned %d (%s)",
priv->device->device_name, ret,
(ret > 0) ? strerror (ret) : "");
ret = -1;
@@ -647,15 +660,17 @@ gf_rdma_create_cq (rpc_transport_t *this)
send_cqe, device,
device->send_chan, 0);
if (!device->send_cq) {
- gf_log (this->name, GF_LOG_ERROR,
- "creation of send_cq for device %s failed",
- device->device_name);
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_CQ_CREATION_FAILED,
+ "creation of send_cq "
+ "for device %s failed", device->device_name);
ret = -1;
goto out;
}
if (ibv_req_notify_cq (device->send_cq, 0)) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_REQ_NOTIFY_CQ_SENDQ_FAILED,
"ibv_req_notify_cq on send_cq for device %s"
" failed", device->device_name);
ret = -1;
@@ -734,16 +749,18 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
trav->send_chan = ibv_create_comp_channel (trav->context);
if (!trav->send_chan) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not create send completion channel for "
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_SEND_COMP_CHAN_FAILED, "could not "
+ "create send completion channel for "
"device (%s)", device_name);
goto out;
}
trav->recv_chan = ibv_create_comp_channel (trav->context);
if (!trav->recv_chan) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not create recv completion channel for "
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_RECV_COMP_CHAN_FAILED, "could not "
+ "create recv completion channel for "
"device (%s)", device_name);
/* TODO: cleanup current mess */
@@ -751,7 +768,8 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
}
if (gf_rdma_create_cq (this) < 0) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_CQ_CREATION_FAILED,
"could not create CQ for device (%s)",
device_name);
goto out;
@@ -761,9 +779,10 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
trav->pd = ibv_alloc_pd (trav->context);
if (!trav->pd) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not allocate protection domain for "
- "device (%s)", device_name);
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_ALOC_PROT_DOM_FAILED, "could not "
+ "allocate protection domain for device (%s)",
+ device_name);
goto out;
}
@@ -777,8 +796,9 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
trav->srq = ibv_create_srq (trav->pd, &attr);
if (!trav->srq) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not create SRQ for device (%s)",
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_CRE_SRQ_FAILED, "could not create SRQ"
+ " for device (%s)",
device_name);
goto out;
}
@@ -791,9 +811,9 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
gf_rdma_register_iobuf_pool(trav, iobuf_pool);
if (gf_rdma_create_posts (this) < 0) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not allocate posts for device (%s)",
- device_name);
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_ALOC_POST_FAILED, "could not allocate"
+ "posts for device (%s)", device_name);
goto out;
}
@@ -802,7 +822,8 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
gf_rdma_send_completion_proc,
trav->send_chan);
if (ret) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_SEND_COMP_THREAD_FAILED,
"could not create send completion thread for "
"device (%s)", device_name);
goto out;
@@ -812,7 +833,8 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
gf_rdma_recv_completion_proc,
trav->recv_chan);
if (ret) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_RECV_COMP_THREAD_FAILED,
"could not create recv completion thread "
"for device (%s)", device_name);
return NULL;
@@ -822,14 +844,15 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx,
gf_rdma_async_event_thread,
ibctx);
if (ret) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ RDMA_MSG_ASYNC_EVENT_THEAD_FAILED,
"could not create async_event_thread");
return NULL;
}
/* qpreg */
pthread_mutex_init (&trav->qpreg.lock, NULL);
- for (i=0; i<42; i++) {
+ for (i = 0; i < 42; i++) {
trav->qpreg.ents[i].next = &trav->qpreg.ents[i];
trav->qpreg.ents[i].prev = &trav->qpreg.ents[i];
}
@@ -912,19 +935,20 @@ gf_rdma_transport_new (rpc_transport_t *listener, struct rdma_cm_id *cm_id)
device_name = (char *)ibv_get_device_name (cm_id->verbs->device);
if (device_name == NULL) {
- gf_log (listener->name, GF_LOG_WARNING,
- "cannot get device name (peer:%s me:%s)",
- this->peerinfo.identifier, this->myinfo.identifier);
+ gf_msg (listener->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_GET_DEVICE_NAME_FAILED, "cannot get device "
+ "name (peer:%s me:%s)", this->peerinfo.identifier,
+ this->myinfo.identifier);
goto out;
}
priv->device = gf_rdma_get_device (this, cm_id->verbs,
device_name);
if (priv->device == NULL) {
- gf_log (listener->name, GF_LOG_WARNING,
- "cannot get infiniband device %s (peer:%s me:%s)",
- device_name, this->peerinfo.identifier,
- this->myinfo.identifier);
+ gf_msg (listener->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_GET_IB_DEVICE_FAILED, "cannot get infiniband"
+ " device %s (peer:%s me:%s)", device_name,
+ this->peerinfo.identifier, this->myinfo.identifier);
goto out;
}
@@ -978,23 +1002,25 @@ gf_rdma_cm_handle_connect_request (struct rdma_cm_event *event)
this = gf_rdma_transport_new (listener, child_cm_id);
if (this == NULL) {
- gf_log (listener->name, GF_LOG_WARNING,
- "could not create a transport for incoming connection"
+ gf_msg (listener->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_CREAT_INC_TRANS_FAILED, "could not create "
+ "a transport for incoming connection"
" (me.name:%s me.identifier:%s)", listener->name,
listener->myinfo.identifier);
rdma_destroy_id (child_cm_id);
goto out;
}
- gf_log (listener->name, GF_LOG_TRACE,
- "got a connect request (me:%s peer:%s)",
- listener->myinfo.identifier, this->peerinfo.identifier);
+ gf_msg_trace (listener->name, 0, "got a connect request (me:%s peer:"
+ "%s)", listener->myinfo.identifier,
+ this->peerinfo.identifier);
ret = gf_rdma_create_qp (this);
if (ret < 0) {
- gf_log (listener->name, GF_LOG_WARNING,
- "could not create QP (peer:%s me:%s)",
- this->peerinfo.identifier, this->myinfo.identifier);
+ gf_msg (listener->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_CREAT_QP_FAILED, "could not create QP "
+ "(peer:%s me:%s)", this->peerinfo.identifier,
+ this->myinfo.identifier);
gf_rdma_cm_handle_disconnect (this);
goto out;
}
@@ -1006,8 +1032,9 @@ gf_rdma_cm_handle_connect_request (struct rdma_cm_event *event)
ret = rdma_accept(child_cm_id, &conn_param);
if (ret < 0) {
- gf_log (listener->name, GF_LOG_WARNING, "rdma_accept failed "
- "peer:%s me:%s (%s)", this->peerinfo.identifier,
+ gf_msg (listener->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_ACCEPT_FAILED, "rdma_accept failed peer:%s "
+ "me:%s (%s)", this->peerinfo.identifier,
this->myinfo.identifier, strerror (errno));
gf_rdma_cm_handle_disconnect (this);
goto out;
@@ -1042,9 +1069,10 @@ gf_rdma_cm_handle_route_resolved (struct rdma_cm_event *event)
ret = gf_rdma_create_qp (this);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
- "could not create QP (peer:%s me:%s)",
- this->peerinfo.identifier, this->myinfo.identifier);
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_CREAT_QP_FAILED, "could not create QP "
+ "(peer:%s me:%s)", this->peerinfo.identifier,
+ this->myinfo.identifier);
gf_rdma_cm_handle_disconnect (this);
goto out;
}
@@ -1057,14 +1085,15 @@ gf_rdma_cm_handle_route_resolved (struct rdma_cm_event *event)
ret = rdma_connect(peer->cm_id, &conn_param);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_CONNECT_FAILED,
"rdma_connect failed (%s)", strerror (errno));
gf_rdma_cm_handle_disconnect (this);
goto out;
}
- gf_log (this->name, GF_LOG_TRACE, "route resolved (me:%s peer:%s)",
- this->myinfo.identifier, this->peerinfo.identifier);
+ gf_msg_trace (this->name, 0, "route resolved (me:%s peer:%s)",
+ this->myinfo.identifier, this->peerinfo.identifier);
ret = 0;
out:
@@ -1099,15 +1128,16 @@ gf_rdma_cm_handle_addr_resolved (struct rdma_cm_event *event)
ret = rdma_resolve_route(peer->cm_id, 2000);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
- "rdma_resolve_route failed (me:%s peer:%s) (%s)",
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_ROUTE_RESOLVE_FAILED, "rdma_resolve_route "
+ "failed (me:%s peer:%s) (%s)",
this->myinfo.identifier, this->peerinfo.identifier,
strerror (errno));
gf_rdma_cm_handle_disconnect (this);
}
- gf_log (this->name, GF_LOG_TRACE, "Address resolved (me:%s peer:%s)",
- this->myinfo.identifier, this->peerinfo.identifier);
+ gf_msg_trace (this->name, 0, "Address resolved (me:%s peer:%s)",
+ this->myinfo.identifier, this->peerinfo.identifier);
return ret;
}
@@ -1120,8 +1150,7 @@ gf_rdma_cm_handle_disconnect (rpc_transport_t *this)
char need_unref = 0;
priv = this->private;
- gf_log (this->name, GF_LOG_DEBUG,
- "peer disconnected, cleaning up");
+ gf_msg_debug (this->name, 0, "peer disconnected, cleaning up");
pthread_mutex_lock (&priv->write_mutex);
{
@@ -1155,9 +1184,10 @@ gf_rdma_cm_handle_connect_init (struct rdma_cm_event *event)
priv = this->private;
if (priv->connected == 1) {
- gf_log (this->name, GF_LOG_TRACE,
- "received event RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)",
- this->myinfo.identifier, this->peerinfo.identifier);
+ gf_msg_trace (this->name, 0, "received event "
+ "RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)",
+ this->myinfo.identifier,
+ this->peerinfo.identifier);
return ret;
}
@@ -1171,9 +1201,10 @@ gf_rdma_cm_handle_connect_init (struct rdma_cm_event *event)
pthread_mutex_unlock (&priv->write_mutex);
if (priv->entity == GF_RDMA_CLIENT) {
- gf_log (this->name, GF_LOG_TRACE,
- "received event RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)",
- this->myinfo.identifier, this->peerinfo.identifier);
+ gf_msg_trace (this->name, 0, "received event "
+ "RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)",
+ this->myinfo.identifier,
+ this->peerinfo.identifier);
ret = rpc_transport_notify (this, RPC_TRANSPORT_CONNECT, this);
} else if (priv->entity == GF_RDMA_SERVER) {
@@ -1224,7 +1255,8 @@ gf_rdma_cm_event_handler (void *data)
while (1) {
ret = rdma_get_cm_event (event_channel, &event);
if (ret != 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno,
+ RDMA_MSG_CM_EVENT_FAILED,
"rdma_cm_get_event failed (%s)",
strerror (errno));
break;
@@ -1254,10 +1286,9 @@ gf_rdma_cm_event_handler (void *data)
case RDMA_CM_EVENT_REJECTED:
this = event->id->context;
- gf_log (this->name, GF_LOG_WARNING,
- "cma event %s, error %d (me:%s peer:%s)\n "
- "Host Unreachable, Check your connection "
- "with IPoIB",
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_CM_EVENT_FAILED, "cma event %s, "
+ "error %d (me:%s peer:%s)\n",
rdma_event_str(event->event), event->status,
this->myinfo.identifier,
this->peerinfo.identifier);
@@ -1271,10 +1302,10 @@ gf_rdma_cm_event_handler (void *data)
case RDMA_CM_EVENT_DISCONNECTED:
this = event->id->context;
- gf_log (this->name, GF_LOG_DEBUG,
- "received disconnect (me:%s peer:%s)\n",
- this->myinfo.identifier,
- this->peerinfo.identifier);
+ gf_msg_debug (this->name, 0, "received disconnect "
+ "(me:%s peer:%s)\n",
+ this->myinfo.identifier,
+ this->peerinfo.identifier);
rdma_ack_cm_event (event);
event = NULL;
@@ -1283,13 +1314,15 @@ gf_rdma_cm_event_handler (void *data)
continue;
case RDMA_CM_EVENT_DEVICE_REMOVAL:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "device removed");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_CM_EVENT_FAILED, "device "
+ "removed");
gf_rdma_cm_handle_device_removal (event);
break;
default:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_CM_EVENT_FAILED,
"unhandled event: %s, ignoring",
rdma_event_str(event->event));
break;
@@ -1367,8 +1400,8 @@ __gf_rdma_send_error (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
len = __gf_rdma_encode_error (peer, reply_info, entry->rpchdr,
(gf_rdma_header_t *)post->buf, err);
if (len == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
- "encode error returned -1");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, 0,
+ RDMA_MSG_ENCODE_ERROR, "encode error returned -1");
goto out;
}
@@ -1378,7 +1411,8 @@ __gf_rdma_send_error (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if (!ret) {
ret = len;
} else {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POST_SEND_FAILED,
"gf_rdma_post_send (to %s) failed with ret = %d (%s)",
peer->trans->peerinfo.identifier, ret,
(ret > 0) ? strerror (ret) : "");
@@ -1428,7 +1462,8 @@ __gf_rdma_create_read_chunks_from_vector (gf_rdma_peer_t *peer,
IBV_ACCESS_REMOTE_READ);
}
if (!mr) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno,
+ RDMA_MSG_MR_ALOC_FAILED,
"memory registration failed (%s) (peer:%s)",
strerror (errno),
peer->trans->peerinfo.identifier);
@@ -1480,7 +1515,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->rpchdr_count,
request_ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_READ_CHUNK_VECTOR_FAILED,
"cannot create read chunks from vector "
"entry->rpchdr");
goto out;
@@ -1493,7 +1529,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->proghdr_count,
request_ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_READ_CHUNK_VECTOR_FAILED,
"cannot create read chunks from vector "
"entry->proghdr");
}
@@ -1506,7 +1543,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->prog_payload_count,
request_ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_READ_CHUNK_VECTOR_FAILED,
"cannot create read chunks from vector"
" entry->prog_payload");
}
@@ -1520,7 +1558,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->prog_payload_count,
request_ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_READ_CHUNK_VECTOR_FAILED,
"cannot create read chunks from vector "
"entry->prog_payload");
}
@@ -1570,8 +1609,9 @@ __gf_rdma_create_write_chunks_from_vector (gf_rdma_peer_t *peer,
}
if (!mr) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "memory registration failed (%s) (peer:%s)",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno,
+ RDMA_MSG_MR_ALOC_FAILED, "memory "
+ "registration failed (%s) (peer:%s)",
strerror (errno),
peer->trans->peerinfo.identifier);
goto out;
@@ -1612,7 +1652,8 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if ((chunk_type == gf_rdma_replych)
&& ((entry->msg.request.rsphdr_count != 1) ||
(entry->msg.request.rsphdr_vec[0].iov_base == NULL))) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_BUFFER_ERROR,
(entry->msg.request.rsphdr_count == 1)
? "chunktype specified as reply chunk but the vector "
"specifying the buffer to be used for holding reply"
@@ -1648,7 +1689,8 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->msg.request.rsp_payload_count,
request_ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_CHUNK_VECTOR_FAILED,
"cannot create write chunks from vector "
"entry->rpc_payload");
goto out;
@@ -1678,7 +1720,8 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->msg.request.rsphdr_count,
request_ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_CHUNK_VECTOR_FAILED,
"cannot create write chunks from vector "
"entry->rpchdr");
goto out;
@@ -1783,9 +1826,7 @@ __gf_rdma_request_context_destroy (gf_rdma_request_context_t *context)
if (priv->connected) {
ret = __gf_rdma_quota_put (peer);
if (ret < 0) {
- gf_log ("rdma", GF_LOG_DEBUG,
- "failed to send "
- "message");
+ gf_msg_debug ("rdma", 0, "failed to send message");
mem_put (context);
__gf_rdma_disconnect (peer->trans);
goto out;
@@ -1924,7 +1965,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if ((entry->msg.request.rsphdr_count != 0)
&& (entry->msg.request.rsp_payload_count != 0)) {
ret = -1;
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_REPLY_CHUNCK_CONFLICT,
"both write-chunklist and reply-chunk cannot be "
"present");
goto out;
@@ -1976,7 +2018,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if (chunk_count > GF_RDMA_MAX_SEGMENTS) {
ret = -1;
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_CHUNK_COUNT_GREAT_MAX_SEGMENTS,
"chunk count(%d) exceeding maximum allowed RDMA "
"segment count(%d)", chunk_count, GF_RDMA_MAX_SEGMENTS);
goto out;
@@ -2022,7 +2065,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
&chunkptr,
request_ctx);
if (ret != 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_CREATE_READ_CHUNK_FAILED,
"creation of read chunks failed");
goto out;
}
@@ -2035,7 +2079,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
&chunkptr,
request_ctx);
if (ret != 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_CREATE_WRITE_REPLAY_FAILED,
"creation of write/reply chunk failed");
goto out;
}
@@ -2069,7 +2114,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if (!ret) {
ret = len;
} else {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POST_SEND_FAILED,
"gf_rdma_post_send (to %s) failed with ret = %d (%s)",
peer->trans->peerinfo.identifier, ret,
(ret > 0) ? strerror (ret) : "");
@@ -2141,7 +2187,8 @@ __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
ret = __gf_rdma_send_error (peer, entry, post, reply_info,
ERR_CHUNK);
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_SEND_SIZE_GREAT_INLINE_THRESHOLD,
"msg size (%d) is greater than maximum size "
"of msg that can be sent inlined (%d)",
send_size, GLUSTERFS_RDMA_INLINE_THRESHOLD);
@@ -2178,8 +2225,9 @@ __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
if (!ret) {
ret = send_size;
} else {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "posting send (to %s) failed with ret = %d (%s)",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POST_SEND_FAILED, "posting send (to %s) "
+ "failed with ret = %d (%s)",
peer->trans->peerinfo.identifier, ret,
(ret > 0) ? strerror (ret) : "");
gf_rdma_post_unref (post);
@@ -2212,9 +2260,9 @@ __gf_rdma_reply_encode_write_chunks (gf_rdma_peer_t *peer,
}
if (chunk_size < payload_size) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
- "length of payload (%d) is exceeding the total "
- "write chunk length (%d)", payload_size, chunk_size);
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0, "length of payload (%d) is "
+ "exceeding the total write chunk length (%d)",
+ payload_size, chunk_size);
goto out;
}
@@ -2280,9 +2328,10 @@ __gf_rdma_register_local_mr_for_rdma (gf_rdma_peer_t *peer,
IBV_ACCESS_LOCAL_WRITE);
}
if (ctx->mr[ctx->mr_count] == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "registering memory for IBV_ACCESS_LOCAL_WRITE "
- "failed (%s)", strerror (errno));
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno,
+ RDMA_MSG_MR_ALOC_FAILED,
+ "registering memory for IBV_ACCESS_LOCAL_WRITE"
+ " failed (%s)", strerror (errno));
goto out;
}
@@ -2330,9 +2379,9 @@ __gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *vec,
for ((i = *idx), (num_sge = 0); (xfer_len != 0); i++, num_sge++) {
size = min (xfer_len, vec[i].iov_len);
- sg_list [num_sge].addr = (unsigned long)vec[i].iov_base;
- sg_list [num_sge].length = size;
- sg_list [num_sge].lkey = post->ctx.mr[i]->lkey;
+ sg_list[num_sge].addr = (unsigned long)vec[i].iov_base;
+ sg_list[num_sge].length = size;
+ sg_list[num_sge].lkey = post->ctx.mr[i]->lkey;
xfer_len -= size;
}
@@ -2353,8 +2402,8 @@ __gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *vec,
ret = ibv_post_send(peer->qp, &wr, &bad_wr);
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma write to "
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_CLIENT_ERROR, "rdma write to "
"client (%s) failed with ret = %d (%s)",
peer->trans->peerinfo.identifier, ret,
(ret > 0) ? strerror (ret) : "");
@@ -2389,7 +2438,8 @@ __gf_rdma_do_gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
ret = __gf_rdma_register_local_mr_for_rdma (peer, vector, count,
&post->ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_MR_ALOC_FAILED,
"registering memory region for rdma failed");
goto out;
}
@@ -2406,8 +2456,9 @@ __gf_rdma_do_gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
&payload_idx,
&reply_info->wc_array->wc_array[i]);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma write to client (%s) failed",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_CLIENT_ERROR, "rdma write to "
+ "client (%s) failed",
peer->trans->peerinfo.identifier);
goto out;
}
@@ -2450,8 +2501,8 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
reply_info,
(uint32_t **)&buf);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "encoding write chunks failed");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_ENCODE_ERROR, "encoding write chunks failed");
ret = __gf_rdma_send_error (peer, entry, post, reply_info,
ERR_CHUNK);
goto out;
@@ -2470,18 +2521,19 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
ret = __gf_rdma_do_gf_rdma_write (peer, post, vector, count,
entry->iobref, reply_info);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma write to peer (%s) failed",
- peer->trans->peerinfo.identifier);
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_PEER_FAILED, "rdma write to peer "
+ "(%s) failed", peer->trans->peerinfo.identifier);
gf_rdma_post_unref (post);
goto out;
}
ret = gf_rdma_post_send (peer->qp, post, (buf - post->buf));
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "posting a send request to client (%s) failed with "
- "ret = %d (%s)", peer->trans->peerinfo.identifier, ret,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POST_SEND_FAILED, "posting a send request "
+ "to client (%s) failed with ret = %d (%s)",
+ peer->trans->peerinfo.identifier, ret,
(ret > 0) ? strerror (ret) : "");
ret = -1;
gf_rdma_post_unref (post);
@@ -2509,7 +2561,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
+ GLUSTERFS_RDMA_MAX_HEADER_SIZE;
if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_SEND_SIZE_GREAT_INLINE_THRESHOLD,
"client has provided only write chunks, but the "
"combined size of rpc and program header (%d) is "
"exceeding the size of msg that can be sent using "
@@ -2534,8 +2587,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
reply_info,
(uint32_t **)&ptr);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "encoding write chunks failed");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_ENCODE_ERROR, "encoding write chunks failed");
ret = __gf_rdma_send_error (peer, entry, post, reply_info,
ERR_CHUNK);
goto out;
@@ -2550,7 +2603,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
entry->prog_payload_count,
entry->iobref, reply_info);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, "rdma write to peer "
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_PEER_FAILED, "rdma write to peer "
"(%s) failed", peer->trans->peerinfo.identifier);
gf_rdma_post_unref (post);
goto out;
@@ -2564,7 +2618,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
ret = gf_rdma_post_send (peer->qp, post, (ptr - post->buf));
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_SEND_CLIENT_ERROR,
"rdma send to client (%s) failed with ret = %d (%s)",
peer->trans->peerinfo.identifier, ret,
(ret > 0) ? strerror (ret) : "");
@@ -2640,7 +2695,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
ret = __gf_rdma_send_reply_inline (peer, entry, post,
reply_info);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_SEND_REPLY_FAILED,
"failed to send reply to peer (%s) as an "
"inlined rdma msg",
peer->trans->peerinfo.identifier);
@@ -2651,7 +2707,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
ret = __gf_rdma_send_reply_type_nomsg (peer, entry, post,
reply_info);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_SEND_REPLY_FAILED,
"failed to send reply to peer (%s) as "
"RDMA_NOMSG", peer->trans->peerinfo.identifier);
}
@@ -2661,7 +2718,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
ret = __gf_rdma_send_reply_type_msg (peer, entry, post,
reply_info);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_SEND_REPLY_FAILED,
"failed to send reply with write chunks "
"to peer (%s)",
peer->trans->peerinfo.identifier);
@@ -2669,7 +2727,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry,
break;
default:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_INVALID_CHUNK_TYPE,
"invalid chunktype (%d) specified for sending reply "
" (peer:%s)", type, peer->trans->peerinfo.identifier);
break;
@@ -2715,7 +2774,8 @@ __gf_rdma_ioq_churn_entry (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry)
if (entry->is_request) {
ret = __gf_rdma_ioq_churn_request (peer, entry, post);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_PROC_IOQ_ENTRY_FAILED,
"failed to process request ioq entry "
"to peer(%s)",
peer->trans->peerinfo.identifier);
@@ -2723,7 +2783,8 @@ __gf_rdma_ioq_churn_entry (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry)
} else {
ret = __gf_rdma_ioq_churn_reply (peer, entry, post);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_PROC_IOQ_ENTRY_FAILED,
"failed to process reply ioq entry "
"to peer (%s)",
peer->trans->peerinfo.identifier);
@@ -2748,8 +2809,7 @@ __gf_rdma_ioq_churn (gf_rdma_peer_t *peer)
gf_rdma_ioq_t *entry = NULL;
int32_t ret = 0;
- while (!list_empty (&peer->ioq))
- {
+ while (!list_empty (&peer->ioq)) {
/* pick next entry */
entry = peer->ioq_next;
@@ -2783,7 +2843,8 @@ gf_rdma_writev (rpc_transport_t *this, gf_rdma_ioq_t *entry)
pthread_mutex_lock (&priv->write_mutex);
{
if (!priv->connected) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_PEER_DISCONNECTED,
"rdma is not connected to peer (%s)",
this->peerinfo.identifier);
ret = -1;
@@ -2797,9 +2858,10 @@ gf_rdma_writev (rpc_transport_t *this, gf_rdma_ioq_t *entry)
need_append = 0;
if (ret < 0) {
- gf_log (this->name, GF_LOG_WARNING,
- "processing ioq entry destined "
- "to (%s) failed",
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_PROC_IOQ_ENTRY_FAILED,
+ "processing ioq entry destined"
+ " to (%s) failed",
this->peerinfo.identifier);
}
}
@@ -2942,7 +3004,8 @@ gf_rdma_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
entry = gf_rdma_ioq_new (this, &data);
if (entry == NULL) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_NEW_IOQ_ENTRY_FAILED,
"getting a new ioq entry failed (peer:%s)",
this->peerinfo.identifier);
goto out;
@@ -2953,7 +3016,8 @@ gf_rdma_submit_request (rpc_transport_t *this, rpc_transport_req_t *req)
if (ret > 0) {
ret = 0;
} else if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_PEER_FAILED,
"sending request to peer (%s) failed",
this->peerinfo.identifier);
rpc_transport_disconnect (this);
@@ -2978,7 +3042,8 @@ gf_rdma_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
entry = gf_rdma_ioq_new (this, &data);
if (entry == NULL) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_NEW_IOQ_ENTRY_FAILED,
"getting a new ioq entry failed (peer:%s)",
this->peerinfo.identifier);
goto out;
@@ -2988,7 +3053,8 @@ gf_rdma_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply)
if (ret > 0) {
ret = 0;
} else if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_WRITE_PEER_FAILED,
"sending request to peer (%s) failed",
this->peerinfo.identifier);
rpc_transport_disconnect (this);
@@ -3129,7 +3195,9 @@ gf_rdma_create_qp (rpc_transport_t *this)
device_name = (char *)ibv_get_device_name (peer->cm_id->verbs->device);
if (device_name == NULL) {
ret = -1;
- gf_log (this->name, GF_LOG_WARNING, "cannot get device_name");
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_GET_DEVICE_NAME_FAILED, "cannot get "
+ "device_name");
goto out;
}
@@ -3137,7 +3205,8 @@ gf_rdma_create_qp (rpc_transport_t *this)
device_name);
if (device == NULL) {
ret = -1;
- gf_log (this->name, GF_LOG_WARNING, "cannot get device for "
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_GET_DEVICE_FAILED, "cannot get device for "
"device %s", device_name);
goto out;
}
@@ -3161,9 +3230,9 @@ gf_rdma_create_qp (rpc_transport_t *this)
ret = rdma_create_qp(peer->cm_id, device->pd, &init_attr);
if (ret != 0) {
- gf_log (peer->trans->name, GF_LOG_CRITICAL,
- "%s: could not create QP (%s)", this->name,
- strerror (errno));
+ gf_msg (peer->trans->name, GF_LOG_CRITICAL, errno,
+ RDMA_MSG_CREAT_QP_FAILED, "%s: could not create QP "
+ "(%s)", this->name, strerror (errno));
ret = -1;
goto out;
}
@@ -3358,8 +3427,9 @@ gf_rdma_decode_error_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
ret = rpc_reply_to_xdr (&rpc_msg, iobuf_ptr (iobuf),
iobuf_pagesize (iobuf), &post->ctx.vector[0]);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "Failed to create RPC reply");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_RPC_REPLY_CREATE_FAILED, "Failed to create "
+ "RPC reply");
goto out;
}
@@ -3400,8 +3470,9 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
ret = gf_rdma_get_read_chunklist (&ptr, readch);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "cannot get read chunklist from msg");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_GET_READ_CHUNK_FAILED, "cannot get read "
+ "chunklist from msg");
goto out;
}
@@ -3410,8 +3481,9 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
ret = gf_rdma_get_write_chunklist (&ptr, &write_ary);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "cannot get write chunklist from msg");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_GET_WRITE_CHUNK_FAILED, "cannot get write "
+ "chunklist from msg");
goto out;
}
@@ -3421,7 +3493,8 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
if (write_ary != NULL) {
reply_info = gf_rdma_reply_info_alloc (peer);
if (reply_info == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_REPLY_INFO_ALLOC_FAILED,
"reply_info_alloc failed");
ret = -1;
goto out;
@@ -3433,15 +3506,17 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
} else {
ret = gf_rdma_get_write_chunklist (&ptr, &write_ary);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "cannot get reply chunklist from msg");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_CHUNKLIST_ERROR, "cannot get reply "
+ "chunklist from msg");
goto out;
}
if (write_ary != NULL) {
reply_info = gf_rdma_reply_info_alloc (peer);
if (reply_info == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_REPLY_INFO_ALLOC_FAILED,
"reply_info_alloc_failed");
ret = -1;
goto out;
@@ -3507,36 +3582,38 @@ gf_rdma_decode_header (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
case GF_RDMA_NOMSG:
ret = gf_rdma_decode_msg (peer, post, readch, bytes_in_post);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "cannot decode msg of type (%d)",
- header->rm_type);
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_ENCODE_ERROR, "cannot decode msg of "
+ "type (%d)", header->rm_type);
}
break;
case GF_RDMA_MSGP:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma msg of msg-type GF_RDMA_MSGP should not have "
- "been received");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_INVALID_ENTRY, "rdma msg of msg-type "
+ "GF_RDMA_MSGP should not have been received");
ret = -1;
break;
case GF_RDMA_DONE:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma msg of msg-type GF_RDMA_DONE should not have "
- "been received");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_INVALID_ENTRY, "rdma msg of msg-type "
+ "GF_RDMA_DONE should not have been received");
ret = -1;
break;
case GF_RDMA_ERROR:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "received a msg of type RDMA_ERROR");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_RDMA_ERROR_RECEIVED, "received a msg of type"
+ " RDMA_ERROR");
ret = gf_rdma_decode_error_msg (peer, post, bytes_in_post);
break;
default:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "unknown rdma msg-type (%d)", header->rm_type);
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_INVALID_ENTRY, "unknown rdma msg-type (%d)",
+ header->rm_type);
}
return ret;
@@ -3562,9 +3639,10 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
}
if (i == 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "message type specified as rdma-read but there are no "
- "rdma read-chunks present");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_INVALID_CHUNK_TYPE, "message type specified "
+ "as rdma-read but there are no rdma read-chunks "
+ "present");
goto out;
}
@@ -3592,9 +3670,9 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
pthread_mutex_lock (&priv->write_mutex);
{
if (!priv->connected) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "transport not connected to peer (%s), "
- "not doing rdma reads",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_PEER_DISCONNECTED, "transport not "
+ "connected to peer (%s), not doing rdma reads",
peer->trans->peerinfo.identifier);
goto unlock;
}
@@ -3623,7 +3701,8 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
ret = __gf_rdma_register_local_mr_for_rdma (peer,
&post->ctx.vector[count], 1, &post->ctx);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_MR_ALOC_FAILED,
"registering local memory"
" for rdma read failed");
goto unlock;
@@ -3652,9 +3731,9 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
wr[i-1].next = NULL;
ret = ibv_post_send (peer->qp, wr, &bad_wr);
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma read from client "
- "(%s) failed with ret = %d (%s)",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_READ_CLIENT_ERROR, "rdma read from "
+ "client (%s) failed with ret = %d (%s)",
peer->trans->peerinfo.identifier,
ret, (ret > 0) ? strerror (ret) : "");
@@ -3746,16 +3825,15 @@ gf_rdma_pollin_notify (gf_rdma_peer_t *peer, gf_rdma_post_t *post)
RPC_TRANSPORT_MAP_XID_REQUEST,
&request_info);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
- "cannot get request information from rpc "
- "layer");
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0, "cannot get request"
+ "information from rpc layer");
goto out;
}
rpc_req = request_info.rpc_req;
if (rpc_req == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
- "rpc request structure not found");
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0, "rpc request "
+ "structure not found");
ret = -1;
goto out;
}
@@ -3780,8 +3858,8 @@ gf_rdma_pollin_notify (gf_rdma_peer_t *peer, gf_rdma_post_t *post)
ret = rpc_transport_notify (peer->trans, RPC_TRANSPORT_MSG_RECEIVED,
pollin);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "transport_notify failed");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ TRANS_MSG_TRANSPORT_ERROR, "transport_notify failed");
}
out:
@@ -3843,16 +3921,18 @@ gf_rdma_recv_reply (gf_rdma_peer_t *peer, gf_rdma_post_t *post)
RPC_TRANSPORT_MAP_XID_REQUEST,
&request_info);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "cannot get request information (peer:%s) from rpc "
- "layer", peer->trans->peerinfo.identifier);
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ TRANS_MSG_TRANSPORT_ERROR, "cannot get request "
+ "information (peer:%s) from rpc layer",
+ peer->trans->peerinfo.identifier);
goto out;
}
rpc_req = request_info.rpc_req;
if (rpc_req == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rpc request structure not found");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_RPC_ST_ERROR, "rpc request structure not "
+ "found");
ret = -1;
goto out;
}
@@ -3870,7 +3950,8 @@ out:
if (ret == 0) {
ret = gf_rdma_pollin_notify (peer, post);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POLL_IN_NOTIFY_FAILED,
"pollin notify failed");
}
}
@@ -3888,14 +3969,16 @@ gf_rdma_recv_request (gf_rdma_peer_t *peer, gf_rdma_post_t *post,
if (readch != NULL) {
ret = gf_rdma_do_reads (peer, post, readch);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_PEER_READ_FAILED,
"rdma read from peer (%s) failed",
peer->trans->peerinfo.identifier);
}
} else {
ret = gf_rdma_pollin_notify (peer, post);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POLL_IN_NOTIFY_FAILED,
"pollin notification failed");
}
}
@@ -3916,15 +3999,17 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc)
post = (gf_rdma_post_t *) (long) wc->wr_id;
if (post == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "no post found in successful work completion element");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_POST_MISSING, "no post found in successful "
+ "work completion element");
goto out;
}
ret = gf_rdma_decode_header (peer, post, &readch, wc->byte_len);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "decoding of header failed");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_HEADER_DECODE_FAILED, "decoding of header "
+ "failed");
goto out;
}
@@ -3945,7 +4030,7 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc)
* instead of directly setting it to header->rm_credit.
*/
priv->peer.quota = header->rm_credit
- - ( 1 - priv->peer.quota);
+ - (1 - priv->peer.quota);
}
}
pthread_mutex_unlock (&priv->write_mutex);
@@ -3966,7 +4051,8 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc)
case GF_RDMA_ERROR:
if (header->rm_body.rm_error.rm_type == ERR_CHUNK) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_RDMA_ERROR_RECEIVED,
"peer (%s), couldn't encode or decode the msg "
"properly or write chunks were not provided "
"for replies that were bigger than "
@@ -3975,36 +4061,40 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc)
GLUSTERFS_RDMA_INLINE_THRESHOLD);
ret = gf_rdma_pollin_notify (peer, post);
if (ret == -1) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
- "pollin notification failed");
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0, "pollin "
+ "notification failed");
}
goto out;
} else {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
- "an error has happened while transmission of "
- "msg, disconnecting the transport");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, 0,
+ TRANS_MSG_TRANSPORT_ERROR, "an error has "
+ "happened while transmission of msg, "
+ "disconnecting the transport");
ret = -1;
goto out;
}
default:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "invalid rdma msg-type (%d)", header->rm_type);
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_INVALID_ENTRY, "invalid rdma msg-type (%d)",
+ header->rm_type);
goto out;
}
if (msg_type == CALL) {
ret = gf_rdma_recv_request (peer, post, readch);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "receiving a request from peer (%s) failed",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_PEER_REQ_FAILED, "receiving a request"
+ " from peer (%s) failed",
peer->trans->peerinfo.identifier);
}
} else {
ret = gf_rdma_recv_reply (peer, post);
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "receiving a reply from peer (%s) failed",
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_PEER_REP_FAILED, "receiving a reply "
+ "from peer (%s) failed",
peer->trans->peerinfo.identifier);
}
}
@@ -4029,21 +4119,22 @@ gf_rdma_async_event_thread (void *context)
&event);
if (ret && errno != EINTR) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "Error getting event (%s)",
- strerror (errno));
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno,
+ RDMA_MSG_EVENT_ERROR, "Error getting "
+ "event (%s)", strerror (errno));
}
- } while(ret && errno == EINTR);
+ } while (ret && errno == EINTR);
switch (event.event_type) {
case IBV_EVENT_SRQ_LIMIT_REACHED:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "received srq_limit reached");
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_EVENT_SRQ_LIMIT_REACHED, "received "
+ "srq_limit reached");
break;
default:
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
- "event (%d) received", event.event_type);
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0, "event (%d) "
+ "received", event.event_type);
break;
}
@@ -4074,7 +4165,8 @@ gf_rdma_recv_completion_proc (void *data)
failed = 0;
ret = ibv_get_cq_event (chan, &event_cq, &event_ctx);
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno,
+ RDMA_MSG_IBV_GET_CQ_FAILED,
"ibv_get_cq_event failed, terminating recv "
"thread %d (%d)", ret, errno);
continue;
@@ -4084,7 +4176,8 @@ gf_rdma_recv_completion_proc (void *data)
ret = ibv_req_notify_cq (event_cq, 0);
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno,
+ RDMA_MSG_IBV_REQ_NOTIFY_CQ_FAILED,
"ibv_req_notify_cq on %s failed, terminating "
"recv thread: %d (%d)",
device->device_name, ret, errno);
@@ -4119,9 +4212,10 @@ gf_rdma_recv_completion_proc (void *data)
pthread_mutex_unlock (&device->qpreg.lock);
if (wc[index].status != IBV_WC_SUCCESS) {
- gf_log (GF_RDMA_LOG_NAME,
- GF_LOG_ERROR, "recv work request "
- "on `%s' returned error (%d)",
+ gf_msg (GF_RDMA_LOG_NAME,
+ GF_LOG_ERROR, 0,
+ RDMA_MSG_RECV_ERROR, "recv work "
+ "request on `%s' returned error (%d)",
device->device_name,
wc[index].status);
failed = 1;
@@ -4142,10 +4236,10 @@ gf_rdma_recv_completion_proc (void *data)
&wc[index]);
rpc_transport_unref (peer->trans);
} else {
- gf_log (GF_RDMA_LOG_NAME,
- GF_LOG_DEBUG,
- "could not lookup peer for qp_num: %d",
- wc[index].qp_num);
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0,
+ "could not lookup peer "
+ "for qp_num: %d",
+ wc[index].qp_num);
}
gf_rdma_post_unref (post);
@@ -4153,8 +4247,8 @@ gf_rdma_recv_completion_proc (void *data)
}
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME,
- GF_LOG_ERROR,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno,
+ RDMA_MSG_IBV_POLL_CQ_ERROR,
"ibv_poll_cq on `%s' returned error "
"(ret = %d, errno = %d)",
device->device_name, ret, errno);
@@ -4185,7 +4279,8 @@ gf_rdma_handle_failed_send_completion (gf_rdma_peer_t *peer, struct ibv_wc *wc)
post = (gf_rdma_post_t *) (long) wc->wr_id;
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_RDMA_HANDLE_FAILED,
"send work request on `%s' returned error "
"wc.status = %d, wc.vendor_err = %d, post->buf = %p, "
"wc.byte_len = %d, post->reused = %d",
@@ -4193,12 +4288,13 @@ gf_rdma_handle_failed_send_completion (gf_rdma_peer_t *peer, struct ibv_wc *wc)
wc->vendor_err, post->buf, wc->byte_len, post->reused);
if (wc->status == IBV_WC_RETRY_EXC_ERR) {
- gf_log ("rdma", GF_LOG_ERROR, "connection between client and"
- " server not working. check by running "
- "'ibv_srq_pingpong'. also make sure subnet manager"
- " is running (eg: 'opensm'), or check if rdma port is "
- "valid (or active) by running 'ibv_devinfo'. contact "
- "Gluster Support Team if the problem persists.");
+ gf_msg ("rdma", GF_LOG_ERROR, 0, TRANS_MSG_TIMEOUT_EXCEEDED,
+ "connection between client and server not working. "
+ "check by running 'ibv_srq_pingpong'. also make sure "
+ "subnet manager is running (eg: 'opensm'), or check "
+ "if rdma port is valid (or active) by running "
+ "'ibv_devinfo'. contact Gluster Support Team if the "
+ "problem persists.");
}
if (peer) {
@@ -4279,7 +4375,8 @@ gf_rdma_send_completion_proc (void *data)
failed = 0;
ret = ibv_get_cq_event (chan, &event_cq, &event_ctx);
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno,
+ RDMA_MSG_IBV_GET_CQ_FAILED,
"ibv_get_cq_event on failed, terminating "
"send thread: %d (%d)", ret, errno);
continue;
@@ -4289,7 +4386,8 @@ gf_rdma_send_completion_proc (void *data)
ret = ibv_req_notify_cq (event_cq, 0);
if (ret) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno,
+ RDMA_MSG_IBV_REQ_NOTIFY_CQ_FAILED,
"ibv_req_notify_cq on %s failed, terminating "
"send thread: %d (%d)",
device->device_name, ret, errno);
@@ -4351,10 +4449,9 @@ gf_rdma_send_completion_proc (void *data)
quota_ret = gf_rdma_quota_put
(peer);
if (quota_ret < 0) {
- gf_log ("rdma",
- GF_LOG_DEBUG,
- "failed to send "
- "message");
+ gf_msg_debug ("rdma",
+ 0, "failed to send "
+ "message");
}
}
}
@@ -4362,7 +4459,7 @@ gf_rdma_send_completion_proc (void *data)
if (peer) {
rpc_transport_unref (peer->trans);
} else {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG,
+ gf_msg_debug (GF_RDMA_LOG_NAME, 0,
"could not lookup peer for qp_num: %d",
wc[index].qp_num);
@@ -4371,7 +4468,8 @@ gf_rdma_send_completion_proc (void *data)
}
if (ret < 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR,
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno,
+ RDMA_MSG_IBV_POLL_CQ_ERROR,
"ibv_poll_cq on `%s' returned error (ret = %d,"
" errno = %d)",
device->device_name, ret, errno);
@@ -4442,26 +4540,30 @@ gf_rdma_options_init (rpc_transport_t *this)
if (temp)
mtu = data_to_int32 (temp);
switch (mtu) {
+
case 256: options->mtu = IBV_MTU_256;
break;
+
case 512: options->mtu = IBV_MTU_512;
break;
+
case 1024: options->mtu = IBV_MTU_1024;
break;
+
case 2048: options->mtu = IBV_MTU_2048;
break;
+
case 4096: options->mtu = IBV_MTU_4096;
break;
default:
if (temp)
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "%s: unrecognized MTU value '%s', defaulting "
- "to '2048'", this->name,
- data_to_str (temp));
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0,
+ RDMA_MSG_UNRECG_MTU_VALUE, "%s: unrecognized "
+ "MTU value '%s', defaulting to '2048'",
+ this->name, data_to_str (temp));
else
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_TRACE,
- "%s: defaulting MTU to '2048'",
- this->name);
+ gf_msg_trace (GF_RDMA_LOG_NAME, 0, "%s: defaulting "
+ "MTU to '2048'", this->name);
options->mtu = IBV_MTU_2048;
break;
}
@@ -4488,9 +4590,9 @@ __gf_rdma_ctx_create (void)
pthread_mutex_init (&rdma_ctx->lock, NULL);
rdma_ctx->rdma_cm_event_channel = rdma_create_event_channel ();
if (rdma_ctx->rdma_cm_event_channel == NULL) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "rdma_cm event channel creation failed (%s)",
- strerror (errno));
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno,
+ RDMA_MSG_CM_EVENT_FAILED, "rdma_cm event channel "
+ "creation failed (%s)", strerror (errno));
goto out;
}
@@ -4498,9 +4600,9 @@ __gf_rdma_ctx_create (void)
gf_rdma_cm_event_handler,
rdma_ctx->rdma_cm_event_channel);
if (ret != 0) {
- gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING,
- "creation of thread to handle rdma-cm events "
- "failed (%s)", strerror (ret));
+ gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, ret,
+ RDMA_MSG_CM_EVENT_FAILED, "creation of thread to "
+ "handle rdma-cm events failed (%s)", strerror (ret));
goto out;
}
@@ -4525,7 +4627,7 @@ gf_rdma_init (rpc_transport_t *this)
glusterfs_ctx_t *ctx = NULL;
gf_rdma_options_t *options = NULL;
- ctx= this->ctx;
+ ctx = this->ctx;
priv = this->private;
@@ -4602,8 +4704,8 @@ gf_rdma_connect (struct rpc_transport *this, int port)
&sock_union.sa,
&sockaddr_len, port);
if (ret != 0) {
- gf_log (this->name, GF_LOG_DEBUG,
- "cannot get remote address to connect");
+ gf_msg_debug (this->name, 0, "cannot get remote address to "
+ "connect");
goto out;
}
@@ -4623,9 +4725,9 @@ gf_rdma_connect (struct rpc_transport *this, int port)
ret = rdma_create_id (rdma_ctx->rdma_cm_event_channel,
&peer->cm_id, this, RDMA_PS_TCP);
if (ret != 0) {
- gf_log (this->name, GF_LOG_ERROR,
- "creation of rdma_cm_id failed (%s)",
- strerror (errno));
+ gf_msg (this->name, GF_LOG_ERROR, errno,
+ RDMA_MSG_CM_EVENT_FAILED, "creation of "
+ "rdma_cm_id failed (%s)", strerror (errno));
ret = -errno;
goto unlock;
}
@@ -4645,7 +4747,8 @@ gf_rdma_connect (struct rpc_transport *this, int port)
&this->myinfo.sockaddr_len,
peer->cm_id);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_CLIENT_BIND_FAILED,
"client bind failed: %s", strerror (errno));
goto unlock;
}
@@ -4653,7 +4756,8 @@ gf_rdma_connect (struct rpc_transport *this, int port)
ret = rdma_resolve_addr (peer->cm_id, NULL, &sock_union.sa,
2000);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_RDMA_RESOLVE_ADDR_FAILED,
"rdma_resolve_addr failed (%s)",
strerror (errno));
goto unlock;
@@ -4699,7 +4803,8 @@ gf_rdma_listen (rpc_transport_t *this)
ret = gf_rdma_server_get_local_sockaddr (this, &sock_union.sa,
&sockaddr_len);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_NW_ADDR_UNKNOWN,
"cannot find network address of server to bind to");
goto err;
}
@@ -4707,9 +4812,9 @@ gf_rdma_listen (rpc_transport_t *this)
ret = rdma_create_id (rdma_ctx->rdma_cm_event_channel,
&peer->cm_id, this, RDMA_PS_TCP);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
- "creation of rdma_cm_id failed (%s)",
- strerror (errno));
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_CM_EVENT_FAILED, "creation of rdma_cm_id "
+ "failed (%s)", strerror (errno));
goto err;
}
@@ -4722,7 +4827,8 @@ gf_rdma_listen (rpc_transport_t *this)
service, sizeof (service),
NI_NUMERICHOST);
if (ret != 0) {
- gf_log (this->name, GF_LOG_ERROR,
+ gf_msg (this->name, GF_LOG_ERROR, ret,
+ TRANS_MSG_GET_NAME_INFO_FAILED,
"getnameinfo failed (%s)", gai_strerror (ret));
goto err;
}
@@ -4733,21 +4839,24 @@ gf_rdma_listen (rpc_transport_t *this)
RDMA_OPTION_ID_REUSEADDR,
(void *)&optval, sizeof(optval));
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
- "rdma option set failed (%s)", strerror (errno));
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_OPTION_SET_FAILED, "rdma option set failed "
+ "(%s)", strerror (errno));
goto err;
}
ret = rdma_bind_addr (peer->cm_id, &sock_union.sa);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_RDMA_BIND_ADDR_FAILED,
"rdma_bind_addr failed (%s)", strerror (errno));
goto err;
}
ret = rdma_listen (peer->cm_id, 10);
if (ret != 0) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, errno,
+ RDMA_MSG_LISTEN_FAILED,
"rdma_listen failed (%s)", strerror (errno));
goto err;
}
@@ -4789,7 +4898,8 @@ init (rpc_transport_t *this)
this->private = priv;
if (gf_rdma_init (this)) {
- gf_log (this->name, GF_LOG_WARNING,
+ gf_msg (this->name, GF_LOG_WARNING, 0,
+ RDMA_MSG_INIT_IB_DEVICE_FAILED,
"Failed to initialize IB Device");
this->private = NULL;
GF_FREE (priv);
@@ -4830,8 +4940,8 @@ fini (struct rpc_transport *this)
pthread_mutex_destroy (&priv->recv_mutex);
pthread_mutex_destroy (&priv->write_mutex);
- gf_log (this->name, GF_LOG_TRACE,
- "called fini on transport: %p", this);
+ gf_msg_trace (this->name, 0,
+ "called fini on transport: %p", this);
GF_FREE (priv);
}
diff --git a/rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h b/rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h
new file mode 100644
index 0000000..0957de2
--- /dev/null
+++ b/rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h
@@ -0,0 +1,217 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+
+#ifndef _RPC_TRANS_RDMA_MESSAGES_H_
+#define _RPC_TRANS_RDMA_MESSAGES_H_
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glfs-message-id.h"
+
+/* NOTE: Rules for message additions
+ * 1) Each instance of a message is _better_ left with a unique message ID, even
+ * if the message format is the same. Reasoning is that, if the message
+ * format needs to change in one instance, the other instances are not
+ * impacted or the new change does not change the ID of the instance being
+ * modified.
+ * 2) Addition of a message,
+ * - Should increment the GLFS_NUM_MESSAGES_RDMA
+ * - Append to the list of messages defined, towards the end
+ * - Retain macro naming as glfs_msg_X (for redability across developers)
+ * NOTE: Rules for message format modifications
+ * 3) Check acorss the code if the message ID macro in question is reused
+ * anywhere. If reused then then the modifications should ensure correctness
+ * everywhere, or needs a new message ID as (1) above was not adhered to. If
+ * not used anywhere, proceed with the required modification.
+ * NOTE: Rules for message deletion
+ * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
+ * anywhere, then can be deleted, but will leave a hole by design, as
+ * addition rules specify modification to the end of the list and not filling
+ * holes.
+ */
+
+#define GLFS_RPC_TRANS_RDMA_BASE GLFS_MSGID_COMP_RPC_TRANS_RDMA
+#define GLFS_NUM_MESSAGES_RDMA 77
+#define GLFS_RPC_TRANS_RDMA_MSGID_END (GLFS_RPC_TRANS_RDMA_BASE + GLFS_NUM_MESSAGES_RDMA + 1)
+
+/* Messages with message IDs */
+
+#define glfs_msg_start_rdma GLFS_RPC_TRANS_RDMA_BASE, "Invalid: Start of messages"
+
+/* This slot/segment is allocated for RDMA message IDs
+ * The allocation starts from BASE.
+
+*/
+
+#define RDMA_MSG_PORT_BIND_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 1)
+
+#define RDMA_MSG_POST_CREATION_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 2)
+
+#define RDMA_MSG_DEREGISTER_ARENA_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 3)
+
+#define RDMA_MSG_MR_ALOC_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 4)
+
+#define RDMA_MSG_PREREG_BUFFER_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 5)
+
+#define RDMA_MSG_CQ_CREATION_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 6)
+
+#define RDMA_MSG_REQ_NOTIFY_CQ_REVQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 7)
+
+#define RDMA_MSG_QUERY_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 8)
+
+#define RDMA_MSG_REQ_NOTIFY_CQ_SENDQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 9)
+
+#define RDMA_MSG_SEND_COMP_CHAN_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 10)
+
+#define RDMA_MSG_RECV_COMP_CHAN_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 11)
+
+#define RDMA_MSG_ALOC_PROT_DOM_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 12)
+
+#define RDMA_MSG_CRE_SRQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 13)
+
+#define RDMA_MSG_ALOC_POST_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 14)
+
+#define RDMA_MSG_SEND_COMP_THREAD_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 15)
+
+#define RDMA_MSG_RECV_COMP_THREAD_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 16)
+
+#define RDMA_MSG_ASYNC_EVENT_THEAD_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 17)
+
+#define RDMA_MSG_GET_DEVICE_NAME_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 18)
+
+#define RDMA_MSG_GET_IB_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 19)
+
+#define RDMA_MSG_CREAT_INC_TRANS_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 20)
+
+#define RDMA_MSG_CREAT_QP_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 21)
+
+#define RDMA_MSG_ACCEPT_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 22)
+
+#define RDMA_MSG_CONNECT_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 23)
+
+#define RDMA_MSG_ROUTE_RESOLVE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 24)
+
+#define RDMA_MSG_GET_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 25)
+
+#define RDMA_MSG_PEER_DISCONNECTED (GLFS_RPC_TRANS_RDMA_BASE + 26)
+
+#define RDMA_MSG_ENCODE_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 27)
+
+#define RDMA_MSG_POST_SEND_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 28)
+
+#define RDMA_MSG_READ_CHUNK_VECTOR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 29)
+
+#define RDMA_MSG_WRITE_CHUNK_VECTOR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 30)
+
+#define RDMA_MSG_WRITE_REPLY_CHUNCK_CONFLICT (GLFS_RPC_TRANS_RDMA_BASE + 31)
+
+#define RDMA_MSG_CHUNK_COUNT_GREAT_MAX_SEGMENTS (GLFS_RPC_TRANS_RDMA_BASE + 32)
+
+#define RDMA_MSG_CREATE_READ_CHUNK_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 33)
+
+#define RDMA_MSG_CREATE_WRITE_REPLAY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 34)
+
+#define RDMA_MSG_SEND_SIZE_GREAT_INLINE_THRESHOLD (GLFS_RPC_TRANS_RDMA_BASE + 35)
+
+#define RDMA_MSG_REG_ACCESS_LOCAL_WRITE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 36)
+
+#define RDMA_MSG_WRITE_PEER_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 37)
+
+#define RDMA_MSG_SEND_REPLY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 38)
+
+#define RDMA_MSG_INVALID_CHUNK_TYPE (GLFS_RPC_TRANS_RDMA_BASE + 39)
+
+#define RDMA_MSG_PROC_IOQ_ENTRY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 40)
+
+#define RDMA_MSG_NEW_IOQ_ENTRY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 41)
+
+#define RDMA_MSG_RPC_REPLY_CREATE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 42)
+
+#define RDMA_MSG_GET_READ_CHUNK_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 43)
+
+#define RDMA_MSG_GET_WRITE_CHUNK_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 44)
+
+#define RDMA_MSG_REPLY_INFO_ALLOC_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 45)
+
+#define RDMA_MSG_RDMA_ERROR_RECEIVED (GLFS_RPC_TRANS_RDMA_BASE + 46)
+
+#define RDMA_MSG_GET_REQ_INFO_RPC_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 47)
+
+#define RDMA_MSG_POLL_IN_NOTIFY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 48)
+
+#define RDMA_MSG_HEADER_DECODE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 49)
+
+#define RDMA_MSG_EVENT_SRQ_LIMIT_REACHED (GLFS_RPC_TRANS_RDMA_BASE + 50)
+
+#define RDMA_MSG_UNRECG_MQ_VALUE (GLFS_RPC_TRANS_RDMA_BASE + 51)
+
+#define RDMA_MSG_BUFFER_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 52)
+
+#define RDMA_MSG_OPTION_SET_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 53)
+
+#define RDMA_MSG_LISTEN_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 54)
+
+#define RDMA_MSG_INIT_IB_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 55)
+
+#define RDMA_MSG_WRITE_CLIENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 56)
+
+#define RDMA_MSG_CHUNKLIST_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 57)
+
+#define RDMA_MSG_INVALID_ENTRY (GLFS_RPC_TRANS_RDMA_BASE + 58)
+
+#define RDMA_MSG_READ_CLIENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 59)
+
+#define RDMA_MSG_RPC_ST_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 60)
+
+#define RDMA_MSG_PEER_READ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 61)
+
+#define RDMA_MSG_POST_MISSING (GLFS_RPC_TRANS_RDMA_BASE + 62)
+
+#define RDMA_MSG_PEER_REQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 63)
+
+#define RDMA_MSG_PEER_REP_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 64)
+
+#define RDMA_MSG_EVENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 65)
+
+#define RDMA_MSG_IBV_GET_CQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 66)
+
+#define RDMA_MSG_IBV_REQ_NOTIFY_CQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 67)
+
+#define RDMA_MSG_RECV_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 68)
+
+#define RDMA_MSG_IBV_POLL_CQ_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 69)
+
+#define RDMA_MSG_RDMA_HANDLE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 70)
+
+#define RDMA_MSG_CM_EVENT_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 71)
+
+#define RDMA_MSG_CLIENT_BIND_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 72)
+
+#define RDMA_MSG_RDMA_RESOLVE_ADDR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 73)
+
+#define RDMA_MSG_NW_ADDR_UNKNOWN (GLFS_RPC_TRANS_RDMA_BASE + 74)
+
+#define RDMA_MSG_RDMA_BIND_ADDR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 75)
+
+#define RDMA_MSG_SEND_CLIENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 76)
+
+#define RDMA_MSG_UNRECG_MTU_VALUE (GLFS_RPC_TRANS_RDMA_BASE + 77)
+
+
+
+/*------------*/
+
+#define glfs_msg_end_rdma GLFS_RPC_TRANS_RDMA_MSGID_END, "Invalid: End of messages"
+
+#endif /* !_RPC_TRANS_RDMA_MESSAGES_H_ */
+
--
1.7.1