diff -up nfs-utils-2.3.3/nfs.conf.orig nfs-utils-2.3.3/nfs.conf --- nfs-utils-2.3.3/nfs.conf.orig 2021-07-19 09:45:40.441448059 -0400 +++ nfs-utils-2.3.3/nfs.conf 2021-07-19 12:08:55.314182838 -0400 @@ -22,6 +22,8 @@ use-gss-proxy=1 # cred-cache-directory= # preferred-realm= # set-home=1 +# upcall-timeout=30 +# cancel-timed-out-upcalls=0 # [lockd] # port=0 diff -up nfs-utils-2.3.3/utils/gssd/gssd.c.orig nfs-utils-2.3.3/utils/gssd/gssd.c --- nfs-utils-2.3.3/utils/gssd/gssd.c.orig 2021-07-19 09:45:40.448448246 -0400 +++ nfs-utils-2.3.3/utils/gssd/gssd.c 2021-07-19 12:08:55.315182865 -0400 @@ -96,8 +96,29 @@ pthread_mutex_t clp_lock = PTHREAD_MUTEX static bool signal_received = false; static struct event_base *evbase = NULL; +int upcall_timeout = DEF_UPCALL_TIMEOUT; +static bool cancel_timed_out_upcalls = false; + TAILQ_HEAD(topdir_list_head, topdir) topdir_list; +/* + * active_thread_list: + * + * used to track upcalls for timeout purposes. + * + * protected by the active_thread_list_lock mutex. + * + * upcall_thread_info structures are added to the tail of the list + * by start_upcall_thread(), so entries closer to the head of the list + * will be closer to hitting the upcall timeout. + * + * upcall_thread_info structures are removed from the list upon a + * sucessful join of the upcall thread by the watchdog thread (via + * scan_active_thread_list(). + */ +TAILQ_HEAD(active_thread_list_head, upcall_thread_info) active_thread_list; +pthread_mutex_t active_thread_list_lock = PTHREAD_MUTEX_INITIALIZER; + struct topdir { TAILQ_ENTRY(topdir) list; TAILQ_HEAD(clnt_list_head, clnt_info) clnt_list; @@ -436,6 +457,138 @@ gssd_clnt_krb5_cb(int UNUSED(fd), short handle_krb5_upcall(clp); } +/* + * scan_active_thread_list: + * + * Walks the active_thread_list, trying to join as many upcall threads as + * possible. For threads that have terminated, the corresponding + * upcall_thread_info will be removed from the list and freed. Threads that + * are still busy and have exceeded the upcall_timeout will cause an error to + * be logged and may be canceled (depending on the value of + * cancel_timed_out_upcalls). + * + * Returns the number of seconds that the watchdog thread should wait before + * calling scan_active_thread_list() again. + */ +static int +scan_active_thread_list(void) +{ + struct upcall_thread_info *info; + struct timespec now; + unsigned int sleeptime; + bool sleeptime_set = false; + int err; + void *tret, *saveprev; + + sleeptime = upcall_timeout; + pthread_mutex_lock(&active_thread_list_lock); + clock_gettime(CLOCK_MONOTONIC, &now); + TAILQ_FOREACH(info, &active_thread_list, list) { + err = pthread_tryjoin_np(info->tid, &tret); + switch (err) { + case 0: + /* + * The upcall thread has either completed successfully, or + * has been canceled _and_ has acted on the cancellation request + * (i.e. has hit a cancellation point). We can now remove the + * upcall_thread_info from the list and free it. + */ + if (tret == PTHREAD_CANCELED) + printerr(3, "watchdog: thread id 0x%lx cancelled successfully\n", + info->tid); + saveprev = info->list.tqe_prev; + TAILQ_REMOVE(&active_thread_list, info, list); + free(info); + info = saveprev; + break; + case EBUSY: + /* + * The upcall thread is still running. If the timeout has expired + * then we either cancel the thread, log an error, and do an error + * downcall to the kernel (cancel_timed_out_upcalls=true) or simply + * log an error (cancel_timed_out_upcalls=false). In either case, + * the error is logged only once. + */ + if (now.tv_sec >= info->timeout.tv_sec) { + if (cancel_timed_out_upcalls && !(info->flags & UPCALL_THREAD_CANCELED)) { + printerr(0, "watchdog: thread id 0x%lx timed out\n", + info->tid); + pthread_cancel(info->tid); + info->flags |= (UPCALL_THREAD_CANCELED|UPCALL_THREAD_WARNED); + do_error_downcall(info->fd, info->uid, -ETIMEDOUT); + } else { + if (!(info->flags & UPCALL_THREAD_WARNED)) { + printerr(0, "watchdog: thread id 0x%lx running for %ld seconds\n", + info->tid, + now.tv_sec - info->timeout.tv_sec + upcall_timeout); + info->flags |= UPCALL_THREAD_WARNED; + } + } + } else if (!sleeptime_set) { + /* + * The upcall thread is still running, but the timeout has not yet + * expired. Calculate the time remaining until the timeout will + * expire. This is the amount of time the watchdog thread will + * wait before running again. We only need to do this for the busy + * thread closest to the head of the list - entries appearing later + * in the list will time out later. + */ + sleeptime = info->timeout.tv_sec - now.tv_sec; + sleeptime_set = true; + } + break; + default: + /* EDEADLK, EINVAL, and ESRCH... none of which should happen! */ + printerr(0, "watchdog: attempt to join thread id 0x%lx returned %d (%s)!\n", + info->tid, err, strerror(err)); + break; + } + } + pthread_mutex_unlock(&active_thread_list_lock); + + return sleeptime; +} + +static void * +watchdog_thread_fn(void *UNUSED(arg)) +{ + unsigned int sleeptime; + + for (;;) { + sleeptime = scan_active_thread_list(); + printerr(4, "watchdog: sleeping %u secs\n", sleeptime); + sleep(sleeptime); + } + return (void *)0; +} + +static int +start_watchdog_thread(void) +{ + pthread_attr_t attr; + pthread_t th; + int ret; + + ret = pthread_attr_init(&attr); + if (ret != 0) { + printerr(0, "ERROR: failed to init pthread attr: ret %d: %s\n", + ret, strerror(errno)); + return ret; + } + ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + if (ret != 0) { + printerr(0, "ERROR: failed to create pthread attr: ret %d: %s\n", + ret, strerror(errno)); + return ret; + } + ret = pthread_create(&th, &attr, watchdog_thread_fn, NULL); + if (ret != 0) { + printerr(0, "ERROR: pthread_create failed: ret %d: %s\n", + ret, strerror(errno)); + } + return ret; +} + static struct clnt_info * gssd_get_clnt(struct topdir *tdi, const char *name) { @@ -810,7 +963,7 @@ sig_die(int signal) static void usage(char *progname) { - fprintf(stderr, "usage: %s [-f] [-l] [-M] [-n] [-v] [-r] [-p pipefsdir] [-k keytab] [-d ccachedir] [-t timeout] [-R preferred realm] [-D] [-H]\n", + fprintf(stderr, "usage: %s [-f] [-l] [-M] [-n] [-v] [-r] [-p pipefsdir] [-k keytab] [-d ccachedir] [-t timeout] [-R preferred realm] [-D] [-H] [-U upcall timeout] [-C]\n", progname); exit(1); } @@ -831,6 +984,9 @@ read_gss_conf(void) #endif context_timeout = conf_get_num("gssd", "context-timeout", context_timeout); rpc_timeout = conf_get_num("gssd", "rpc-timeout", rpc_timeout); + upcall_timeout = conf_get_num("gssd", "upcall-timeout", upcall_timeout); + cancel_timed_out_upcalls = conf_get_bool("gssd", "cancel-timed-out-upcalls", + cancel_timed_out_upcalls); s = conf_get_str("gssd", "pipefs-directory"); if (!s) s = conf_get_str("general", "pipefs-directory"); @@ -872,7 +1028,7 @@ main(int argc, char *argv[]) verbosity = conf_get_num("gssd", "verbosity", verbosity); rpc_verbosity = conf_get_num("gssd", "rpc-verbosity", rpc_verbosity); - while ((opt = getopt(argc, argv, "HDfvrlmnMp:k:d:t:T:R:")) != -1) { + while ((opt = getopt(argc, argv, "HDfvrlmnMp:k:d:t:T:R:U:C")) != -1) { switch (opt) { case 'f': fg = 1; @@ -923,6 +1079,12 @@ main(int argc, char *argv[]) case 'H': set_home = false; break; + case 'U': + upcall_timeout = atoi(optarg); + break; + case 'C': + cancel_timed_out_upcalls = true; + break; default: usage(argv[0]); break; @@ -995,6 +1157,11 @@ main(int argc, char *argv[]) else progname = argv[0]; + if (upcall_timeout > MAX_UPCALL_TIMEOUT) + upcall_timeout = MAX_UPCALL_TIMEOUT; + else if (upcall_timeout < MIN_UPCALL_TIMEOUT) + upcall_timeout = MIN_UPCALL_TIMEOUT; + initerr(progname, verbosity, fg); #ifdef HAVE_LIBTIRPC_SET_DEBUG /* @@ -1045,6 +1212,14 @@ main(int argc, char *argv[]) gssd_inotify_cb, NULL); event_add(inotify_ev, NULL); + TAILQ_INIT(&active_thread_list); + + rc = start_watchdog_thread(); + if (rc != 0) { + printerr(0, "ERROR: failed to start watchdog thread: %d\n", rc); + exit(EXIT_FAILURE); + } + TAILQ_INIT(&topdir_list); gssd_scan(); daemon_ready(); diff -up nfs-utils-2.3.3/utils/gssd/gssd.h.orig nfs-utils-2.3.3/utils/gssd/gssd.h --- nfs-utils-2.3.3/utils/gssd/gssd.h.orig 2021-07-19 09:45:40.449448272 -0400 +++ nfs-utils-2.3.3/utils/gssd/gssd.h 2021-07-19 12:08:55.315182865 -0400 @@ -50,6 +50,12 @@ #define GSSD_DEFAULT_KEYTAB_FILE "/etc/krb5.keytab" #define GSSD_SERVICE_NAME "nfs" #define RPC_CHAN_BUF_SIZE 32768 + +/* timeouts are in seconds */ +#define MIN_UPCALL_TIMEOUT 5 +#define DEF_UPCALL_TIMEOUT 30 +#define MAX_UPCALL_TIMEOUT 600 + /* * The gss mechanisms that we can handle */ @@ -91,10 +97,22 @@ struct clnt_upcall_info { char *service; }; +struct upcall_thread_info { + TAILQ_ENTRY(upcall_thread_info) list; + pthread_t tid; + struct timespec timeout; + uid_t uid; + int fd; + unsigned short flags; +#define UPCALL_THREAD_CANCELED 0x0001 +#define UPCALL_THREAD_WARNED 0x0002 +}; + void handle_krb5_upcall(struct clnt_info *clp); void handle_gssd_upcall(struct clnt_info *clp); void free_upcall_info(struct clnt_upcall_info *info); void gssd_free_client(struct clnt_info *clp); +int do_error_downcall(int k5_fd, uid_t uid, int err); #endif /* _RPC_GSSD_H_ */ diff -up nfs-utils-2.3.3/utils/gssd/gssd.man.orig nfs-utils-2.3.3/utils/gssd/gssd.man --- nfs-utils-2.3.3/utils/gssd/gssd.man.orig 2021-07-19 09:45:40.443448112 -0400 +++ nfs-utils-2.3.3/utils/gssd/gssd.man 2021-07-19 12:08:55.315182865 -0400 @@ -8,7 +8,7 @@ rpc.gssd \- RPCSEC_GSS daemon .SH SYNOPSIS .B rpc.gssd -.RB [ \-DfMnlvrH ] +.RB [ \-DfMnlvrHC ] .RB [ \-k .IR keytab ] .RB [ \-p @@ -17,6 +17,10 @@ rpc.gssd \- RPCSEC_GSS daemon .IR ccachedir ] .RB [ \-t .IR timeout ] +.RB [ \-T +.IR timeout ] +.RB [ \-U +.IR timeout ] .RB [ \-R .IR realm ] .SH INTRODUCTION @@ -290,7 +294,7 @@ seconds, which allows changing Kerberos The default is no explicit timeout, which means the kernel context will live the lifetime of the Kerberos service ticket used in its creation. .TP -.B -T timeout +.BI "-T " timeout Timeout, in seconds, to create an RPC connection with a server while establishing an authenticated gss context for a user. The default timeout is set to 5 seconds. @@ -298,6 +302,18 @@ If you get messages like "WARNING: can't %servername% for user with uid %uid%: RPC: Remote system error - Connection timed out", you should consider an increase of this timeout. .TP +.BI "-U " timeout +Timeout, in seconds, for upcall threads. Threads executing longer than +.I timeout +seconds will cause an error message to be logged. The default +.I timeout +is 30 seconds. The minimum is 5 seconds. The maximum is 600 seconds. +.TP +.B -C +In addition to logging an error message for threads that have timed out, +the thread will be canceled and an error of -ETIMEDOUT will be reported +to the kernel. +.TP .B -H Avoids setting $HOME to "/". This allows rpc.gssd to read per user k5identity files versus trying to read /.k5identity for each user. @@ -365,6 +381,17 @@ Equivalent to Equivalent to .BR -R . .TP +.B upcall-timeout +Equivalent to +.BR -U . +.TP +.B cancel-timed-out-upcalls +Setting to +.B true +is equivalent to providing the +.B -C +flag. +.TP .B set-home Setting to .B false diff -up nfs-utils-2.3.3/utils/gssd/gssd_proc.c.orig nfs-utils-2.3.3/utils/gssd/gssd_proc.c --- nfs-utils-2.3.3/utils/gssd/gssd_proc.c.orig 2021-07-19 09:45:40.449448272 -0400 +++ nfs-utils-2.3.3/utils/gssd/gssd_proc.c 2021-07-19 12:08:55.316182891 -0400 @@ -81,11 +81,24 @@ #include "gss_names.h" extern pthread_mutex_t clp_lock; +extern pthread_mutex_t active_thread_list_lock; +extern int upcall_timeout; +extern TAILQ_HEAD(active_thread_list_head, upcall_thread_info) active_thread_list; /* Encryption types supported by the kernel rpcsec_gss code */ int num_krb5_enctypes = 0; krb5_enctype *krb5_enctypes = NULL; +/* Args for the cleanup_handler() */ +struct cleanup_args { + OM_uint32 *min_stat; + gss_buffer_t acceptor; + gss_buffer_t token; + struct authgss_private_data *pd; + AUTH **auth; + CLIENT **rpc_clnt; +}; + /* * Parse the supported encryption type information */ @@ -184,7 +197,7 @@ out_err: return; } -static int +int do_error_downcall(int k5_fd, uid_t uid, int err) { char buf[1024]; @@ -604,27 +617,66 @@ out: } /* + * cleanup_handler: + * + * Free any resources allocated by process_krb5_upcall(). + * + * Runs upon normal termination of process_krb5_upcall as well as if the + * thread is canceled. + */ +static void +cleanup_handler(void *arg) +{ + struct cleanup_args *args = (struct cleanup_args *)arg; + + gss_release_buffer(args->min_stat, args->acceptor); + if (args->token->value) + free(args->token->value); +#ifdef HAVE_AUTHGSS_FREE_PRIVATE_DATA + if (args->pd->pd_ctx_hndl.length != 0 || args->pd->pd_ctx != 0) + authgss_free_private_data(args->pd); +#endif + if (*args->auth) + AUTH_DESTROY(*args->auth); + if (*args->rpc_clnt) + clnt_destroy(*args->rpc_clnt); +} + +/* + * process_krb5_upcall: + * * this code uses the userland rpcsec gss library to create a krb5 * context on behalf of the kernel + * + * This is the meat of the upcall thread. Note that cancelability is disabled + * and enabled at various points to ensure that any resources reserved by the + * lower level libraries are released safely. */ static void -process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *srchost, - char *tgtname, char *service) +process_krb5_upcall(struct clnt_upcall_info *info) { + struct clnt_info *clp = info->clp; + uid_t uid = info->uid; + int fd = info->fd; + char *srchost = info->srchost; + char *tgtname = info->target; + char *service = info->service; CLIENT *rpc_clnt = NULL; AUTH *auth = NULL; struct authgss_private_data pd; gss_buffer_desc token; - int err, downcall_err = -EACCES; + int err, downcall_err; OM_uint32 maj_stat, min_stat, lifetime_rec; gss_name_t gacceptor = GSS_C_NO_NAME; gss_OID mech; gss_buffer_desc acceptor = {0}; + struct cleanup_args cleanup_args = {&min_stat, &acceptor, &token, &pd, &auth, &rpc_clnt}; token.length = 0; token.value = NULL; memset(&pd, 0, sizeof(struct authgss_private_data)); + pthread_cleanup_push(cleanup_handler, &cleanup_args); /* * If "service" is specified, then the kernel is indicating that * we must use machine credentials for this request. (Regardless @@ -646,6 +698,8 @@ process_krb5_upcall(struct clnt_info *cl * used for this case is not important. * */ + downcall_err = -EACCES; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); if (uid != 0 || (uid == 0 && root_uses_machine_creds == 0 && service == NULL)) { @@ -666,15 +720,21 @@ process_krb5_upcall(struct clnt_info *cl goto out_return_error; } } + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + pthread_testcancel(); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); if (!authgss_get_private_data(auth, &pd)) { printerr(1, "WARNING: Failed to obtain authentication " "data for user with uid %d for server %s\n", uid, clp->servername); goto out_return_error; } + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + pthread_testcancel(); /* Grab the context lifetime and acceptor name out of the ctx. */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); maj_stat = gss_inquire_context(&min_stat, pd.pd_ctx, NULL, &gacceptor, &lifetime_rec, &mech, NULL, NULL, NULL); @@ -686,37 +746,35 @@ process_krb5_upcall(struct clnt_info *cl get_hostbased_client_buffer(gacceptor, mech, &acceptor); gss_release_name(&min_stat, &gacceptor); } + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + pthread_testcancel(); /* * The serialization can mean turning pd.pd_ctx into a lucid context. If * that happens then the pd.pd_ctx will be unusable, so we must never * try to use it after this point. */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); if (serialize_context_for_kernel(&pd.pd_ctx, &token, &krb5oid, NULL)) { printerr(1, "WARNING: Failed to serialize krb5 context for " "user with uid %d for server %s\n", uid, clp->servername); goto out_return_error; } + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + pthread_testcancel(); do_downcall(fd, uid, &pd, &token, lifetime_rec, &acceptor); out: - gss_release_buffer(&min_stat, &acceptor); - if (token.value) - free(token.value); -#ifdef HAVE_AUTHGSS_FREE_PRIVATE_DATA - if (pd.pd_ctx_hndl.length != 0 || pd.pd_ctx != 0) - authgss_free_private_data(&pd); -#endif - if (auth) - AUTH_DESTROY(auth); - if (rpc_clnt) - clnt_destroy(rpc_clnt); + pthread_cleanup_pop(1); return; out_return_error: + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + pthread_testcancel(); + do_error_downcall(fd, uid, downcall_err); goto out; } @@ -782,36 +840,69 @@ void free_upcall_info(struct clnt_upcall } static void -gssd_work_thread_fn(struct clnt_upcall_info *info) +cleanup_clnt_upcall_info(void *arg) { - process_krb5_upcall(info->clp, info->uid, info->fd, info->srchost, info->target, info->service); + struct clnt_upcall_info *info = (struct clnt_upcall_info *)arg; + free_upcall_info(info); } +static void +gssd_work_thread_fn(struct clnt_upcall_info *info) +{ + pthread_cleanup_push(cleanup_clnt_upcall_info, info); + process_krb5_upcall(info); + pthread_cleanup_pop(1); +} + +static struct upcall_thread_info * +alloc_upcall_thread_info(void) +{ + struct upcall_thread_info *info; + + info = malloc(sizeof(struct upcall_thread_info)); + if (info == NULL) + return NULL; + memset(info, 0, sizeof(*info)); + return info; +} + static int -start_upcall_thread(void (*func)(struct clnt_upcall_info *), void *info) +start_upcall_thread(void (*func)(struct clnt_upcall_info *), struct clnt_upcall_info *info) { pthread_attr_t attr; pthread_t th; + struct upcall_thread_info *tinfo; int ret; + tinfo = alloc_upcall_thread_info(); + if (!tinfo) + return -ENOMEM; + tinfo->fd = info->fd; + tinfo->uid = info->uid; + ret = pthread_attr_init(&attr); if (ret != 0) { printerr(0, "ERROR: failed to init pthread attr: ret %d: %s\n", ret, strerror(errno)); - return ret; - } - ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - if (ret != 0) { - printerr(0, "ERROR: failed to create pthread attr: ret %d: " - "%s\n", ret, strerror(errno)); + free(tinfo); return ret; } ret = pthread_create(&th, &attr, (void *)func, (void *)info); - if (ret != 0) + if (ret != 0) { printerr(0, "ERROR: pthread_create failed: ret %d: %s\n", ret, strerror(errno)); + free(tinfo); + return ret; + } + tinfo->tid = th; + pthread_mutex_lock(&active_thread_list_lock); + clock_gettime(CLOCK_MONOTONIC, &tinfo->timeout); + tinfo->timeout.tv_sec += upcall_timeout; + TAILQ_INSERT_TAIL(&active_thread_list, tinfo, list); + pthread_mutex_unlock(&active_thread_list_lock); + return ret; }