From f1b179a288c872575e64095ddc3a391d5210e057 Mon Sep 17 00:00:00 2001
From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Date: Wed, 4 May 2016 13:25:06 +0530
Subject: [PATCH 125/139] rpc: define client port range
Problem:
when bind-insecure is 'off', all the clients bind to secure ports,
if incase all the secure ports exhaust the client will no more bind
to secure ports and tries gets a random port which is obviously insecure.
we have seen the client obtaining a port number in the range 49152-65535
which are actually reserved as part of glusterd's pmap_registry for bricks,
hence this will lead to port clashes between client and brick processes.
Solution:
If we can define different port ranges for clients incase where secure ports
exhaust, we can avoid the maximum port clashes with in gluster processes.
Still we are prone to have clashes with other non-gluster processes, but
the chances being very low, but that's a different story on its own, which
will be handled in upcoming patches.
Backportof:
> Backportof:
>> Change-Id: Ib5ce05991aa1290ccb17f6f04ffd65caf411feaf
>> BUG: 1322805
>> Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
>> Reviewed-on: http://review.gluster.org/13998
>> Smoke: Gluster Build System <jenkins@build.gluster.com>
>> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>> CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
>> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
>> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
>> Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
> Change-Id: I712676d3e79145d78a17f2c361525e6ef82a4732
> BUG: 1323564
> Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
> Reviewed-on: http://review.gluster.org/14205
> Tested-by: Prasanna Kumar Kalever <pkalever@redhat.com>
> Smoke: Gluster Build System <jenkins@build.gluster.com>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Change-Id: I3ea5d194c9d0024090bf329eed7b685dd0ac1066
BUG: 1322306
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/73676
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
libglusterfs/src/common-utils.h | 3 +++
rpc/rpc-transport/rdma/src/name.c | 19 +++++++++++++++++++
rpc/rpc-transport/socket/src/name.c | 22 ++++++++++++++++++++--
xlators/mgmt/glusterd/src/glusterd-pmap.h | 2 --
4 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 3ba0f04..388b695 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -87,7 +87,10 @@ void trap (void);
* nfs port in volume status.
*/
#define GF_NFS3_PORT 2049
+
#define GF_CLIENT_PORT_CEILING 1024
+#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
+#define GF_CLNT_INSECURE_PORT_CEILING (GF_IANA_PRIV_PORTS_START - 1)
#define GF_PORT_MAX 65535
#define GF_MINUTE_IN_SECONDS 60
diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c
index d4502e7..d5de6f8 100644
--- a/rpc/rpc-transport/rdma/src/name.c
+++ b/rpc/rpc-transport/rdma/src/name.c
@@ -57,10 +57,17 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
int32_t ret = -1;
uint16_t port = ceiling - 1;
gf_boolean_t ports[GF_PORT_MAX];
+ int i = 0;
+loop:
ret = gf_process_reserved_ports (ports, ceiling);
while (port) {
+ if (port == GF_CLIENT_PORT_CEILING) {
+ ret = -1;
+ break;
+ }
+
/* ignore the reserved ports */
if (ports[port] == _gf_true) {
port--;
@@ -80,6 +87,18 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
port--;
}
+ /* Incase if all the secure ports are exhausted, we are no more
+ * binding to secure ports, hence instead of getting a random
+ * port, lets define the range to restrict it from getting from
+ * ports reserved for bricks i.e from range of 49152 - 65535
+ * which further may lead to port clash */
+ if (!port) {
+ ceiling = port = GF_CLNT_INSECURE_PORT_CEILING;
+ for (i = 0; i <= ceiling; i++)
+ ports[i] = _gf_false;
+ goto loop;
+ }
+
return ret;
}
diff --git a/rpc/rpc-transport/socket/src/name.c b/rpc/rpc-transport/socket/src/name.c
index aa43006..3f85b9f 100644
--- a/rpc/rpc-transport/socket/src/name.c
+++ b/rpc/rpc-transport/socket/src/name.c
@@ -45,11 +45,17 @@ af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
int32_t ret = -1;
uint16_t port = ceiling - 1;
gf_boolean_t ports[GF_PORT_MAX];
+ int i = 0;
+loop:
ret = gf_process_reserved_ports (ports, ceiling);
- while (port)
- {
+ while (port) {
+ if (port == GF_CLIENT_PORT_CEILING) {
+ ret = -1;
+ break;
+ }
+
/* ignore the reserved ports */
if (ports[port] == _gf_true) {
port--;
@@ -69,6 +75,18 @@ af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
port--;
}
+ /* Incase if all the secure ports are exhausted, we are no more
+ * binding to secure ports, hence instead of getting a random
+ * port, lets define the range to restrict it from getting from
+ * ports reserved for bricks i.e from range of 49152 - 65535
+ * which further may lead to port clash */
+ if (!port) {
+ ceiling = port = GF_CLNT_INSECURE_PORT_CEILING;
+ for (i = 0; i <= ceiling; i++)
+ ports[i] = _gf_false;
+ goto loop;
+ }
+
return ret;
}
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index 4743439..ea1e0de 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -28,8 +28,6 @@
#include "rpcsvc.h"
-#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
-
struct pmap_port_status {
gf_pmap_port_type_t type;
char *brickname;
--
1.7.1