From 1183db5268c3957975cdc6ad882113bfff7a8f32 Mon Sep 17 00:00:00 2001 From: Milind Changire Date: Mon, 21 May 2018 12:17:06 +0530 Subject: [PATCH 280/282] Revert "rpcsvc: scale rpcsvc_request_handler threads" This reverts commit 8503ed9b94777d47352f19ebfa844e151352b87f. > Reviewed-on: https://code.engineering.redhat.com/gerrit/131596 Change-Id: I537a40d29b027a7e06babb775b39bb111f0d0e3f BUG: 1558989 Signed-off-by: Milind Changire Reviewed-on: https://code.engineering.redhat.com/gerrit/139308 Tested-by: RHGS Build Bot Reviewed-by: Raghavendra Gowdappa Reviewed-by: Sunil Kumar Heggodu Gopala Acharya --- glusterfsd/src/Makefile.am | 1 - glusterfsd/src/glusterfsd-mgmt.c | 16 +---- glusterfsd/src/glusterfsd.h | 2 +- libglusterfs/src/event-poll.c | 7 -- rpc/rpc-lib/src/rpcsvc.c | 129 ++++------------------------------- rpc/rpc-lib/src/rpcsvc.h | 8 --- xlators/protocol/server/src/server.c | 11 +-- 7 files changed, 20 insertions(+), 154 deletions(-) diff --git a/glusterfsd/src/Makefile.am b/glusterfsd/src/Makefile.am index 8ab585c..0196204 100644 --- a/glusterfsd/src/Makefile.am +++ b/glusterfsd/src/Makefile.am @@ -22,7 +22,6 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) \ -I$(top_srcdir)/rpc/xdr/src \ -I$(top_builddir)/rpc/xdr/src \ -I$(top_srcdir)/xlators/nfs/server/src \ - -I$(top_srcdir)/xlators/protocol/server/src \ -I$(top_srcdir)/api/src AM_CFLAGS = -Wall $(GF_CFLAGS) diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index 3b9671c..665b62c 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -33,7 +33,6 @@ #include "syncop.h" #include "xlator.h" #include "syscall.h" -#include "server.h" static gf_boolean_t is_mgmt_rpc_reconnect = _gf_false; int need_emancipate = 0; @@ -186,15 +185,12 @@ glusterfs_terminate_response_send (rpcsvc_request_t *req, int op_ret) } void -glusterfs_autoscale_threads (glusterfs_ctx_t *ctx, int incr, xlator_t *this) +glusterfs_autoscale_threads (glusterfs_ctx_t *ctx, int incr) { struct event_pool *pool = ctx->event_pool; - server_conf_t *conf = this->private; - int thread_count = pool->eventthreadcount; pool->auto_thread_count += incr; - (void) event_reconfigure_threads (pool, thread_count+incr); - rpcsvc_ownthread_reconf (conf->rpc, pool->eventthreadcount); + (void) event_reconfigure_threads (pool, pool->eventthreadcount+incr); } static int @@ -964,7 +960,6 @@ glusterfs_handle_attach (rpcsvc_request_t *req) xlator_t *nextchild = NULL; glusterfs_graph_t *newgraph = NULL; glusterfs_ctx_t *ctx = NULL; - xlator_t *protocol_server = NULL; GF_ASSERT (req); this = THIS; @@ -1002,12 +997,7 @@ glusterfs_handle_attach (rpcsvc_request_t *req) nextchild->name); goto out; } - /* we need a protocol/server xlator as - * nextchild - */ - protocol_server = this->ctx->active->first; - glusterfs_autoscale_threads (this->ctx, 1, - protocol_server); + glusterfs_autoscale_threads (this->ctx, 1); } } else { gf_log (this->name, GF_LOG_WARNING, diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h index a72acc8..f66947b 100644 --- a/glusterfsd/src/glusterfsd.h +++ b/glusterfsd/src/glusterfsd.h @@ -124,7 +124,7 @@ int glusterfs_volume_top_read_perf (uint32_t blk_size, uint32_t blk_count, char *brick_path, double *throughput, double *time); void -glusterfs_autoscale_threads (glusterfs_ctx_t *ctx, int incr, xlator_t *this); +glusterfs_autoscale_threads (glusterfs_ctx_t *ctx, int incr); void xlator_mem_cleanup (xlator_t *this); diff --git a/libglusterfs/src/event-poll.c b/libglusterfs/src/event-poll.c index b1aca82..3bffc47 100644 --- a/libglusterfs/src/event-poll.c +++ b/libglusterfs/src/event-poll.c @@ -173,13 +173,6 @@ event_pool_new_poll (int count, int eventthreadcount) "thread count (%d) ignored", eventthreadcount); } - /* although, eventhreadcount for poll implementaiton is always - * going to be 1, eventthreadcount needs to be set to 1 so that - * rpcsvc_request_handler() thread scaling works flawlessly in - * both epoll and poll models - */ - event_pool->eventthreadcount = 1; - return event_pool; } diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index 31b5eb5..68e27ab 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -1877,105 +1877,39 @@ rpcsvc_request_handler (void *arg) goto unlock; } - while (list_empty (&program->request_queue) && - (program->threadcount <= - program->eventthreadcount)) { + while (list_empty (&program->request_queue)) pthread_cond_wait (&program->queue_cond, &program->queue_lock); - } - if (program->threadcount > program->eventthreadcount) { - done = 1; - program->threadcount--; - - gf_log (GF_RPCSVC, GF_LOG_INFO, - "program '%s' thread terminated; " - "total count:%d", - program->progname, - program->threadcount); - } else if (!list_empty (&program->request_queue)) { - req = list_entry (program->request_queue.next, - typeof (*req), request_list); - - list_del_init (&req->request_list); - } + req = list_entry (program->request_queue.next, + typeof (*req), request_list); + + list_del_init (&req->request_list); } unlock: pthread_mutex_unlock (&program->queue_lock); - if (req) { - THIS = req->svc->xl; - actor = rpcsvc_program_actor (req); - ret = actor->actor (req); - - if (ret != 0) { - rpcsvc_check_and_reply_error (ret, NULL, req); - } - req = NULL; - } - if (done) break; - } - return NULL; -} + THIS = req->svc->xl; -int -rpcsvc_spawn_threads (rpcsvc_t *svc, rpcsvc_program_t *program) -{ - int ret = 0, delta = 0, creates = 0; + actor = rpcsvc_program_actor (req); - if (!program || !svc) - goto out; - - pthread_mutex_lock (&program->queue_lock); - { - delta = program->eventthreadcount - program->threadcount; - - if (delta >= 0) { - while (delta--) { - ret = gf_thread_create (&program->thread, NULL, - rpcsvc_request_handler, - program, "rpcrqhnd"); - if (!ret) { - program->threadcount++; - creates++; - } - } + ret = actor->actor (req); - if (creates) { - gf_log (GF_RPCSVC, GF_LOG_INFO, - "spawned %d threads for program '%s'; " - "total count:%d", - creates, - program->progname, - program->threadcount); - } - } else { - gf_log (GF_RPCSVC, GF_LOG_INFO, - "terminating %d threads for program '%s'", - -delta, program->progname); - - /* this signal is to just wake up the threads so they - * test for the change in eventthreadcount and kill - * themselves until the program thread count becomes - * equal to the event thread count - */ - pthread_cond_broadcast (&program->queue_cond); + if (ret != 0) { + rpcsvc_check_and_reply_error (ret, NULL, req); } } - pthread_mutex_unlock (&program->queue_lock); -out: - return creates; + return NULL; } int rpcsvc_program_register (rpcsvc_t *svc, rpcsvc_program_t *program) { int ret = -1; - int creates = -1; rpcsvc_program_t *newprog = NULL; char already_registered = 0; @@ -2023,12 +1957,9 @@ rpcsvc_program_register (rpcsvc_t *svc, rpcsvc_program_t *program) newprog->ownthread = _gf_false; if (newprog->ownthread) { - newprog->eventthreadcount = 1; - creates = rpcsvc_spawn_threads (svc, newprog); - - if (creates < 1) { - goto out; - } + gf_thread_create (&newprog->thread, NULL, + rpcsvc_request_handler, + newprog, "rpcsvcrh"); } pthread_mutex_lock (&svc->rpclock); @@ -2885,38 +2816,6 @@ out: return ret; } -/* During reconfigure, Make sure to call this function after event-threads are - * reconfigured as programs' threadcount will be made equal to event threads. - */ - -int -rpcsvc_ownthread_reconf (rpcsvc_t *svc, int new_eventthreadcount) -{ - int ret = -1; - rpcsvc_program_t *program = NULL; - - if (!svc) { - ret = 0; - goto out; - } - - pthread_rwlock_wrlock (&svc->rpclock); - { - list_for_each_entry (program, &svc->programs, program) { - if (program->ownthread) { - program->eventthreadcount = - new_eventthreadcount; - rpcsvc_spawn_threads (svc, program); - } - } - } - pthread_rwlock_unlock (&svc->rpclock); - - ret = 0; -out: - return ret; -} - rpcsvc_actor_t gluster_dump_actors[GF_DUMP_MAXVALUE] = { [GF_DUMP_NULL] = {"NULL", GF_DUMP_NULL, NULL, NULL, 0, DRC_NA}, diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h index 4ae2350..73507b6 100644 --- a/rpc/rpc-lib/src/rpcsvc.h +++ b/rpc/rpc-lib/src/rpcsvc.h @@ -412,12 +412,6 @@ struct rpcsvc_program { pthread_mutex_t queue_lock; pthread_cond_t queue_cond; pthread_t thread; - int threadcount; - /* eventthreadcount is just a readonly copy of the actual value - * owned by the event sub-system - * It is used to control the scaling of rpcsvc_request_handler threads - */ - int eventthreadcount; }; typedef struct rpcsvc_cbk_program { @@ -629,6 +623,4 @@ rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen); rpcsvc_vector_sizer rpcsvc_get_program_vector_sizer (rpcsvc_t *svc, uint32_t prognum, uint32_t progver, int procnum); -extern int -rpcsvc_ownthread_reconf (rpcsvc_t *svc, int new_eventthreadcount); #endif diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 79f68e8..bc87a80 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -685,7 +685,7 @@ unref_transport: UNLOCK (&ctx->volfile_lock); if (victim_found) { xlator_mem_cleanup (travxl); - glusterfs_autoscale_threads (ctx, -1, this); + glusterfs_autoscale_threads (ctx, -1); } } GF_FREE (xlator_name); @@ -1054,12 +1054,6 @@ do_rpc: ret = server_init_grace_timer (this, options, conf); - /* rpcsvc thread reconfigure should be after events thread - * reconfigure - */ - new_nthread = - ((struct event_pool *)(this->ctx->event_pool))->eventthreadcount; - ret = rpcsvc_ownthread_reconf (rpc_conf, new_nthread); out: THIS = oldTHIS; gf_msg_debug ("", 0, "returning %d", ret); @@ -1672,10 +1666,9 @@ notify (xlator_t *this, int32_t event, void *data, ...) glusterfs_mgmt_pmap_signout (ctx, victim->name); - if (!xprt_found && victim_found) { xlator_mem_cleanup (victim); - glusterfs_autoscale_threads (ctx, -1, this); + glusterfs_autoscale_threads (ctx, -1); } } break; -- 1.8.3.1