From b7c66c030242c9271f8f0629b631e5a84a97cbe6 Mon Sep 17 00:00:00 2001

From: Stepan Broz <sbroz@redhat.com>

Date: Tue, 16 Apr 2024 18:48:51 +0200

Subject: [PATCH] Fail the DNSSEC validation on the first failure

Be more strict when encountering DNSSEC validation failures - fail on

the first failure.  This will break domains that have DNSSEC signing

keys with duplicate key ids, but this is something that's much easier

to fix on the authoritative side, so we are just going to be strict

on the resolver side where it is causing performance problems.

(cherry picked from commit 8b7ecba9885e163c07c2dd3e1ceab79b2ba89e34)

Add normal and slow task queues

Split the task manager queues into normal and slow task queues, so we

can move the tasks that blocks processing for a long time (like DNSSEC

validation) into the slow queue which doesn't block fast

operations (like responding from the cache).  This mitigates the whole

class of KeyTrap-like issues.

(cherry picked from commit db083a21726300916fa0b9fd8a433a796fedf636)

Improve the selecting of the new signing key by remembering where

we stopped the iteration and just continue from that place instead

of iterating from the start over and over again each time.

(cherry picked from commit 75faeefcab47e4f1e12b358525190b4be90f97de)

Optimize selecting the signing key

Don't parse the crypto data before parsing and matching the id and the

algorithm.

(cherry picked from commit b38552cca7200a72658e482f8407f57516efc5db)

6322.   [security]      Specific DNS answers could cause a denial-of-service

                        condition due to DNS validation taking a long time.

                        (CVE-2023-50387) [GL #4424]

                        The same code change also addresses another problem:

                        preparing NSEC3 closest encloser proofs could exhaust

                        available CPU resources. (CVE-2023-50868) [GL #4459]

Related to [GL #4424] and [GL #4459]:

Add normal task queue also to non-thread version

Non-thread builds are used by us for dhcp package. Make it working

again.

---

 lib/dns/dst_api.c               |  25 +++--

 lib/dns/include/dns/validator.h |   1 +

 lib/dns/include/dst/dst.h       |   4 +

 lib/dns/resolver.c              |   2 +-

 lib/dns/validator.c             |  96 ++++++++----------

 lib/dns/win32/libdns.def.in     |   1 +

 lib/isc/include/isc/task.h      |  11 ++-

 lib/isc/task.c                  | 167 ++++++++++++++++++++++----------

 8 files changed, 193 insertions(+), 114 deletions(-)

diff --git a/lib/dns/dst_api.c b/lib/dns/dst_api.c

index dbece0a..3ed8f7f 100644

--- a/lib/dns/dst_api.c

+++ b/lib/dns/dst_api.c

@@ -103,6 +103,7 @@ static isc_result_t	frombuffer(dns_name_t *name,

 				   dns_rdataclass_t rdclass,

 				   isc_buffer_t *source,

 				   isc_mem_t *mctx,

+				   isc_boolean_t no_rdata,

 				   dst_key_t **keyp);

 static isc_result_t	algorithm_status(unsigned int alg);

@@ -751,6 +752,13 @@ isc_result_t

 dst_key_fromdns(dns_name_t *name, dns_rdataclass_t rdclass,

 		isc_buffer_t *source, isc_mem_t *mctx, dst_key_t **keyp)

 {

+	return (dst_key_fromdns_ex(name, rdclass, source, mctx, ISC_FALSE, keyp));

+}

+

+isc_result_t

+dst_key_fromdns_ex(dns_name_t *name, dns_rdataclass_t rdclass,

+		   isc_buffer_t *source, isc_mem_t *mctx, isc_boolean_t no_rdata,

+		   dst_key_t **keyp) {

 	isc_uint8_t alg, proto;

 	isc_uint32_t flags, extflags;

 	dst_key_t *key = NULL;

@@ -779,7 +787,7 @@ dst_key_fromdns(dns_name_t *name, dns_rdataclass_t rdclass,

 	}

 	result = frombuffer(name, alg, flags, proto, rdclass, source,

-			    mctx, &key);

+			    mctx, no_rdata, &key);

 	if (result != ISC_R_SUCCESS)

 		return (result);

 	key->key_id = id;

@@ -801,7 +809,7 @@ dst_key_frombuffer(dns_name_t *name, unsigned int alg,

 	REQUIRE(dst_initialized);

 	result = frombuffer(name, alg, flags, protocol, rdclass, source,

-			    mctx, &key);

+			    mctx, ISC_FALSE, &key);

 	if (result != ISC_R_SUCCESS)

 		return (result);

@@ -1899,7 +1907,8 @@ computeid(dst_key_t *key) {

 static isc_result_t

 frombuffer(dns_name_t *name, unsigned int alg, unsigned int flags,

 	   unsigned int protocol, dns_rdataclass_t rdclass,

-	   isc_buffer_t *source, isc_mem_t *mctx, dst_key_t **keyp)

+	   isc_buffer_t *source, isc_mem_t *mctx, isc_boolean_t no_rdata,

+	   dst_key_t **keyp)

 {

 	dst_key_t *key;

 	isc_result_t ret;

@@ -1924,10 +1933,12 @@ frombuffer(dns_name_t *name, unsigned int alg, unsigned int flags,

 			return (DST_R_UNSUPPORTEDALG);

 		}

-		ret = key->func->fromdns(key, source);

-		if (ret != ISC_R_SUCCESS) {

-			dst_key_free(&key);

-			return (ret);

+		if (!no_rdata) {

+			ret = key->func->fromdns(key, source);

+			if (ret != ISC_R_SUCCESS) {

+				dst_key_free(&key);

+				return (ret);

+			}

 		}

 	}

diff --git a/lib/dns/include/dns/validator.h b/lib/dns/include/dns/validator.h

index 100eab7..a4d15b4 100644

--- a/lib/dns/include/dns/validator.h

+++ b/lib/dns/include/dns/validator.h

@@ -158,6 +158,7 @@ struct dns_validator {

 	unsigned int			depth;

 	unsigned int			authcount;

 	unsigned int			authfail;

+	isc_boolean_t			failed;

 	isc_stdtime_t			start;

 };

diff --git a/lib/dns/include/dst/dst.h b/lib/dns/include/dst/dst.h

index 42c67d4..6018bc2 100644

--- a/lib/dns/include/dst/dst.h

+++ b/lib/dns/include/dst/dst.h

@@ -414,6 +414,10 @@ dst_key_tofile(const dst_key_t *key, int type, const char *directory);

  */

 isc_result_t

+dst_key_fromdns_ex(dns_name_t *name, dns_rdataclass_t rdclass,

+		   isc_buffer_t *source, isc_mem_t *mctx, isc_boolean_t no_rdata,

+		   dst_key_t **keyp);

+isc_result_t

 dst_key_fromdns(dns_name_t *name, dns_rdataclass_t rdclass,

 		isc_buffer_t *source, isc_mem_t *mctx, dst_key_t **keyp);

 /*%<

diff --git a/lib/dns/resolver.c b/lib/dns/resolver.c

index a55b5fd..b2de13a 100644

--- a/lib/dns/resolver.c

+++ b/lib/dns/resolver.c

@@ -9089,7 +9089,7 @@ dns_resolver_create(dns_view_t *view,

 		if (result != ISC_R_SUCCESS)

 			goto cleanup_buckets;

 		res->buckets[i].task = NULL;

-		result = isc_task_create(taskmgr, 0, &res->buckets[i].task);

+		result = isc_task_create(taskmgr, ISC_TASK_QUANTUM_SLOW, &res->buckets[i].task);

 		if (result != ISC_R_SUCCESS) {

 			DESTROYLOCK(&res->buckets[i].lock);

 			goto cleanup_buckets;

diff --git a/lib/dns/validator.c b/lib/dns/validator.c

index 289b505..695a5c2 100644

--- a/lib/dns/validator.c

+++ b/lib/dns/validator.c

@@ -1197,6 +1197,12 @@ create_validator(dns_validator_t *val, dns_name_t *name, dns_rdatatype_t type,

  * val->key at it.

  *

  * If val->key is non-NULL, this returns the next matching key.

+ * If val->key is already non-NULL, start searching from the next position in

+ * 'rdataset' to find the *next* key that could have signed 'siginfo', then

+ * set val->key to that.

+ *

+ * Returns ISC_R_SUCCESS if a possible matching key has been found,

+ * ISC_R_NOTFOUND if not. Any other value indicates error.

  */

 static isc_result_t

 get_dst_key(dns_validator_t *val, dns_rdata_rrsig_t *siginfo,

@@ -1206,54 +1212,58 @@ get_dst_key(dns_validator_t *val, dns_rdata_rrsig_t *siginfo,

 	isc_buffer_t b;

 	dns_rdata_t rdata = DNS_RDATA_INIT;

 	dst_key_t *oldkey = val->key;

-	isc_boolean_t foundold;

+	isc_boolean_t no_rdata = ISC_FALSE;

-	if (oldkey == NULL)

-		foundold = ISC_TRUE;

-	else {

-		foundold = ISC_FALSE;

+	if (oldkey == NULL) {

+		result = dns_rdataset_first(rdataset);

+	} else {

+		dst_key_free(&oldkey);

 		val->key = NULL;

+		result = dns_rdataset_next(rdataset);

+	}

+

+	if (result != ISC_R_SUCCESS) {

+		goto done;

 	}

-	result = dns_rdataset_first(rdataset);

-	if (result != ISC_R_SUCCESS)

-		goto failure;

 	do {

 		dns_rdataset_current(rdataset, &rdata);

 		isc_buffer_init(&b, rdata.data, rdata.length);

 		isc_buffer_add(&b, rdata.length);

 		INSIST(val->key == NULL);

-		result = dst_key_fromdns(&siginfo->signer, rdata.rdclass, &b,

-					 val->view->mctx, &val->key);

+		result = dst_key_fromdns_ex(&siginfo->signer, rdata.rdclass, &b,

+					    val->view->mctx, no_rdata,

+					    &val->key);

 		if (result == ISC_R_SUCCESS) {

 			if (siginfo->algorithm ==

 				    (dns_secalg_t)dst_key_alg(val->key) &&

 			    siginfo->keyid ==

 				    (dns_keytag_t)dst_key_id(val->key) &&

+			    (dst_key_flags(val->key) & DNS_KEYFLAG_REVOKE) ==

+				    0 &&

 			    dst_key_iszonekey(val->key))

 			{

-				if (foundold) {

-					/*

-					 * This is the key we're looking for.

-					 */

-					return (ISC_R_SUCCESS);

-				} else if (dst_key_compare(oldkey, val->key)) {

-					foundold = ISC_TRUE;

-					dst_key_free(&oldkey);

+				if (no_rdata) {

+					/* Retry with full key */

+					dns_rdata_reset(&rdata);

+					dst_key_free(&val->key);

+					no_rdata = ISC_FALSE;

+					continue;

 				}

+				/* This is the key we're looking for. */

+				goto done;

 			}

 			dst_key_free(&val->key);

 		}

 		dns_rdata_reset(&rdata);

 		result = dns_rdataset_next(rdataset);

+		no_rdata = ISC_TRUE;

 	} while (result == ISC_R_SUCCESS);

-	if (result == ISC_R_NOMORE)

+done:

+	if (result == ISC_R_NOMORE) {

 		result = ISC_R_NOTFOUND;

-

- failure:

-	if (oldkey != NULL)

-		dst_key_free(&oldkey);

+	}

 	return (result);

 }

@@ -1622,37 +1632,13 @@ validate(dns_validator_t *val, isc_boolean_t resume) {

 			continue;

 		}

-		do {

-			vresult = verify(val, val->key, &rdata,

-					val->siginfo->keyid);

-			if (vresult == ISC_R_SUCCESS)

-				break;

-			if (val->keynode != NULL) {

-				dns_keynode_t *nextnode = NULL;

-				result = dns_keytable_findnextkeynode(

-							val->keytable,

-							val->keynode,

-							&nextnode);

-				dns_keytable_detachkeynode(val->keytable,

-							   &val->keynode);

-				val->keynode = nextnode;

-				if (result != ISC_R_SUCCESS) {

-					val->key = NULL;

-					break;

-				}

-				val->key = dns_keynode_key(val->keynode);

-				if (val->key == NULL)

-					break;

-			} else {

-				if (get_dst_key(val, val->siginfo, val->keyset)

-				    != ISC_R_SUCCESS)

-					break;

-			}

-		} while (1);

-		if (vresult != ISC_R_SUCCESS)

+		vresult = verify(val, val->key, &rdata,

+				val->siginfo->keyid);

+		if (vresult != ISC_R_SUCCESS) {

+			val->failed = ISC_TRUE;

 			validator_log(val, ISC_LOG_DEBUG(3),

 				      "failed to verify rdataset");

-		else {

+		} else {

 			dns_rdataset_trimttl(event->rdataset,

 					     event->sigrdataset,

 					     val->siginfo, val->start,

@@ -1689,9 +1675,13 @@ validate(dns_validator_t *val, isc_boolean_t resume) {

 		} else {

 			validator_log(val, ISC_LOG_DEBUG(3),

 				      "verify failure: %s",

-				      isc_result_totext(result));

+				      isc_result_totext(vresult));

 			resume = ISC_FALSE;

 		}

+		if (val->failed) {

+			result = ISC_R_NOMORE;

+			break;

+		}

 	}

 	if (result != ISC_R_NOMORE) {

 		validator_log(val, ISC_LOG_DEBUG(3),

diff --git a/lib/dns/win32/libdns.def.in b/lib/dns/win32/libdns.def.in

index d48eeb2..54e11a3 100644

--- a/lib/dns/win32/libdns.def.in

+++ b/lib/dns/win32/libdns.def.in

@@ -1432,6 +1432,7 @@ dst_key_format

 dst_key_free

 dst_key_frombuffer

 dst_key_fromdns

+dst_key_fromdns_ex

 dst_key_fromfile

 dst_key_fromgssapi

 dst_key_fromlabel

diff --git a/lib/isc/include/isc/task.h b/lib/isc/include/isc/task.h

index 1273214..9b6f58a 100644

--- a/lib/isc/include/isc/task.h

+++ b/lib/isc/include/isc/task.h

@@ -96,8 +96,15 @@ ISC_LANG_BEGINDECLS

  ***/

 typedef enum {

-		isc_taskmgrmode_normal = 0,

-		isc_taskmgrmode_privileged

+	isc_taskqueue_normal = 0,

+	isc_taskqueue_slow = 1,

+} isc_taskqueue_t;

+

+#define ISC_TASK_QUANTUM_SLOW 1024

+

+typedef enum {

+	isc_taskmgrmode_normal = 0,

+	isc_taskmgrmode_privileged

 } isc_taskmgrmode_t;

 /*% Task and task manager methods */

diff --git a/lib/isc/task.c b/lib/isc/task.c

index 139ff22..3fe442d 100644

--- a/lib/isc/task.c

+++ b/lib/isc/task.c

@@ -105,6 +105,7 @@ struct isc__task {

 	isc_eventlist_t			on_shutdown;

 	unsigned int			nevents;

 	unsigned int			quantum;

+	unsigned int			qid;

 	unsigned int			flags;

 	isc_stdtime_t			now;

 	isc_time_t			tnow;

@@ -139,11 +140,11 @@ struct isc__taskmgr {

 	/* Locked by task manager lock. */

 	unsigned int			default_quantum;

 	LIST(isc__task_t)		tasks;

-	isc__tasklist_t			ready_tasks;

-	isc__tasklist_t			ready_priority_tasks;

+	isc__tasklist_t			ready_tasks[2];

+	isc__tasklist_t			ready_priority_tasks[2];

 	isc_taskmgrmode_t		mode;

 #ifdef ISC_PLATFORM_USETHREADS

-	isc_condition_t			work_available;

+	isc_condition_t			work_available[2];

 	isc_condition_t			exclusive_granted;

 	isc_condition_t			paused;

 #endif /* ISC_PLATFORM_USETHREADS */

@@ -245,13 +246,13 @@ isc_taskmgrmode_t

 isc__taskmgr_mode(isc_taskmgr_t *manager0);

 static inline isc_boolean_t

-empty_readyq(isc__taskmgr_t *manager);

+empty_readyq(isc__taskmgr_t *manager, isc_taskqueue_t qid);

 static inline isc__task_t *

-pop_readyq(isc__taskmgr_t *manager);

+pop_readyq(isc__taskmgr_t *manager, isc_taskqueue_t qid);

 static inline void

-push_readyq(isc__taskmgr_t *manager, isc__task_t *task);

+push_readyq(isc__taskmgr_t *manager, isc__task_t *task, isc_taskqueue_t qid);

 static struct isc__taskmethods {

 	isc_taskmethods_t methods;

@@ -322,7 +323,8 @@ task_finished(isc__task_t *task) {

 		 * any idle worker threads so they

 		 * can exit.

 		 */

-		BROADCAST(&manager->work_available);

+		BROADCAST(&manager->work_available[isc_taskqueue_normal]);

+		BROADCAST(&manager->work_available[isc_taskqueue_slow]);

 	}

 #endif /* USE_WORKER_THREADS */

 	UNLOCK(&manager->lock);

@@ -360,7 +362,13 @@ isc__task_create(isc_taskmgr_t *manager0, unsigned int quantum,

 	INIT_LIST(task->events);

 	INIT_LIST(task->on_shutdown);

 	task->nevents = 0;

-	task->quantum = quantum;

+	if (quantum >= ISC_TASK_QUANTUM_SLOW) {

+		task->qid = isc_taskqueue_slow;

+		task->quantum = quantum - ISC_TASK_QUANTUM_SLOW;

+	} else {

+		task->qid = isc_taskqueue_normal;

+		task->quantum = quantum;

+	}

 	task->flags = 0;

 	task->now = 0;

 	isc_time_settoepoch(&task->tnow);

@@ -471,10 +479,10 @@ task_ready(isc__task_t *task) {

 	XTRACE("task_ready");

 	LOCK(&manager->lock);

-	push_readyq(manager, task);

+	push_readyq(manager, task, task->qid);

 #ifdef USE_WORKER_THREADS

 	if (manager->mode == isc_taskmgrmode_normal || has_privilege)

-		SIGNAL(&manager->work_available);

+		SIGNAL(&manager->work_available[task->qid]);

 #endif /* USE_WORKER_THREADS */

 	UNLOCK(&manager->lock);

 }

@@ -945,13 +953,13 @@ isc__task_getcurrenttimex(isc_task_t *task0, isc_time_t *t) {

  * Caller must hold the task manager lock.

  */

 static inline isc_boolean_t

-empty_readyq(isc__taskmgr_t *manager) {

+empty_readyq(isc__taskmgr_t *manager, isc_taskqueue_t qid) {

 	isc__tasklist_t queue;

 	if (manager->mode == isc_taskmgrmode_normal)

-		queue = manager->ready_tasks;

+		queue = manager->ready_tasks[qid];

 	else

-		queue = manager->ready_priority_tasks;

+		queue = manager->ready_priority_tasks[qid];

 	return (ISC_TF(EMPTY(queue)));

 }

@@ -965,18 +973,18 @@ empty_readyq(isc__taskmgr_t *manager) {

  * Caller must hold the task manager lock.

  */

 static inline isc__task_t *

-pop_readyq(isc__taskmgr_t *manager) {

+pop_readyq(isc__taskmgr_t *manager, isc_taskqueue_t qid) {

 	isc__task_t *task;

 	if (manager->mode == isc_taskmgrmode_normal)

-		task = HEAD(manager->ready_tasks);

+		task = HEAD(manager->ready_tasks[qid]);

 	else

-		task = HEAD(manager->ready_priority_tasks);

+		task = HEAD(manager->ready_priority_tasks[qid]);

 	if (task != NULL) {

-		DEQUEUE(manager->ready_tasks, task, ready_link);

+		DEQUEUE(manager->ready_tasks[qid], task, ready_link);

 		if (ISC_LINK_LINKED(task, ready_priority_link))

-			DEQUEUE(manager->ready_priority_tasks, task,

+			DEQUEUE(manager->ready_priority_tasks[qid], task,

 				ready_priority_link);

 	}

@@ -990,16 +998,16 @@ pop_readyq(isc__taskmgr_t *manager) {

  * Caller must hold the task manager lock.

  */

 static inline void

-push_readyq(isc__taskmgr_t *manager, isc__task_t *task) {

-	ENQUEUE(manager->ready_tasks, task, ready_link);

+push_readyq(isc__taskmgr_t *manager, isc__task_t *task, isc_taskqueue_t qid) {

+	ENQUEUE(manager->ready_tasks[qid], task, ready_link);

 	if ((task->flags & TASK_F_PRIVILEGED) != 0)

-		ENQUEUE(manager->ready_priority_tasks, task,

+		ENQUEUE(manager->ready_priority_tasks[qid], task,

 			ready_priority_link);

 	manager->tasks_ready++;

 }

 static void

-dispatch(isc__taskmgr_t *manager) {

+dispatch(isc__taskmgr_t *manager, isc_taskqueue_t qid) {

 	isc__task_t *task;

 #ifndef USE_WORKER_THREADS

 	unsigned int total_dispatch_count = 0;

@@ -1078,26 +1086,26 @@ dispatch(isc__taskmgr_t *manager) {

 		 * If a pause has been requested, don't do any work

 		 * until it's been released.

 		 */

-		while ((empty_readyq(manager) || manager->pause_requested ||

+		while ((empty_readyq(manager, qid) || manager->pause_requested ||

 			manager->exclusive_requested) && !FINISHED(manager))

 		{

 			XTHREADTRACE(isc_msgcat_get(isc_msgcat,

 						    ISC_MSGSET_GENERAL,

 						    ISC_MSG_WAIT, "wait"));

-			WAIT(&manager->work_available, &manager->lock);

+			WAIT(&manager->work_available[qid], &manager->lock);

 			XTHREADTRACE(isc_msgcat_get(isc_msgcat,

 						    ISC_MSGSET_TASK,

 						    ISC_MSG_AWAKE, "awake"));

 		}

 #else /* USE_WORKER_THREADS */

 		if (total_dispatch_count >= DEFAULT_TASKMGR_QUANTUM ||

-		    empty_readyq(manager))

+		    empty_readyq(manager, qid))

 			break;

 #endif /* USE_WORKER_THREADS */

 		XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_TASK,

 					    ISC_MSG_WORKING, "working"));

-		task = pop_readyq(manager);

+		task = pop_readyq(manager, qid);

 		if (task != NULL) {

 			unsigned int dispatch_count = 0;

 			isc_boolean_t done = ISC_FALSE;

@@ -1261,7 +1269,7 @@ dispatch(isc__taskmgr_t *manager) {

 				 * might even hurt rather than help.

 				 */

 #ifdef USE_WORKER_THREADS

-				push_readyq(manager, task);

+				push_readyq(manager, task, qid);

 #else

 				ENQUEUE(new_ready_tasks, task, ready_link);

 				if ((task->flags & TASK_F_PRIVILEGED) != 0)

@@ -1279,20 +1287,24 @@ dispatch(isc__taskmgr_t *manager) {

 		 * we're stuck.  Automatically drop privileges at that

 		 * point and continue with the regular ready queue.

 		 */

-		if (manager->tasks_running == 0 && empty_readyq(manager)) {

+		if (manager->tasks_running == 0 && empty_readyq(manager, isc_taskqueue_normal) && empty_readyq(manager, isc_taskqueue_slow)) {

 			manager->mode = isc_taskmgrmode_normal;

-			if (!empty_readyq(manager))

-				BROADCAST(&manager->work_available);

+			if (!empty_readyq(manager, isc_taskqueue_normal)) {

+				BROADCAST(&manager->work_available[isc_taskqueue_normal]);

+			}

+			if (!empty_readyq(manager, isc_taskqueue_slow)) {

+				BROADCAST(&manager->work_available[isc_taskqueue_slow]);

+			}

 		}

 #endif

 	}

 #ifndef USE_WORKER_THREADS

-	ISC_LIST_APPENDLIST(manager->ready_tasks, new_ready_tasks, ready_link);

-	ISC_LIST_APPENDLIST(manager->ready_priority_tasks, new_priority_tasks,

+	ISC_LIST_APPENDLIST(manager->ready_tasks[qid], new_ready_tasks, ready_link);

+	ISC_LIST_APPENDLIST(manager->ready_priority_tasks[qid], new_priority_tasks,

 			    ready_priority_link);

 	manager->tasks_ready += tasks_ready;

-	if (empty_readyq(manager))

+	if (empty_readyq(manager, qid))

 		manager->mode = isc_taskmgrmode_normal;

 #endif

@@ -1304,13 +1316,37 @@ static isc_threadresult_t

 #ifdef _WIN32

 WINAPI

 #endif

-run(void *uap) {

+run_normal(void *uap) {

 	isc__taskmgr_t *manager = uap;

 	XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,

 				    ISC_MSG_STARTING, "starting"));

-	dispatch(manager);

+	dispatch(manager, isc_taskqueue_normal);

+

+	XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,

+				    ISC_MSG_EXITING, "exiting"));

+

+#ifdef OPENSSL_LEAKS

+	ERR_remove_state(0);

+#endif

+

+	return ((isc_threadresult_t)0);

+}

+#endif /* USE_WORKER_THREADS */

+

+#ifdef USE_WORKER_THREADS

+static isc_threadresult_t

+#ifdef _WIN32

+WINAPI

+#endif

+run_slow(void *uap) {

+	isc__taskmgr_t *manager = uap;

+

+	XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,

+				    ISC_MSG_STARTING, "starting"));

+

+	dispatch(manager, isc_taskqueue_slow);

 	XTHREADTRACE(isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,

 				    ISC_MSG_EXITING, "exiting"));

@@ -1329,7 +1365,8 @@ manager_free(isc__taskmgr_t *manager) {

 #ifdef USE_WORKER_THREADS

 	(void)isc_condition_destroy(&manager->exclusive_granted);

-	(void)isc_condition_destroy(&manager->work_available);

+	(void)isc_condition_destroy(&manager->work_available[isc_taskqueue_normal]);

+	(void)isc_condition_destroy(&manager->work_available[isc_taskqueue_slow]);

 	(void)isc_condition_destroy(&manager->paused);

 	isc_mem_free(manager->mctx, manager->threads);

 #endif /* USE_WORKER_THREADS */

@@ -1396,12 +1433,20 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,

 #ifdef USE_WORKER_THREADS

 	manager->workers = 0;

 	manager->threads = isc_mem_allocate(mctx,

-					    workers * sizeof(isc_thread_t));

+					    2 * workers * sizeof(isc_thread_t));

 	if (manager->threads == NULL) {

 		result = ISC_R_NOMEMORY;

 		goto cleanup_lock;

 	}

-	if (isc_condition_init(&manager->work_available) != ISC_R_SUCCESS) {

+	if (isc_condition_init(&manager->work_available[isc_taskqueue_normal]) != ISC_R_SUCCESS) {

+		UNEXPECTED_ERROR(__FILE__, __LINE__,

+				 "isc_condition_init() %s",

+				 isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,

+						ISC_MSG_FAILED, "failed"));

+		result = ISC_R_UNEXPECTED;

+		goto cleanup_threads;

+	}

+	if (isc_condition_init(&manager->work_available[isc_taskqueue_slow]) != ISC_R_SUCCESS) {

 		UNEXPECTED_ERROR(__FILE__, __LINE__,

 				 "isc_condition_init() %s",

 				 isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,

@@ -1430,8 +1475,10 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,

 		default_quantum = DEFAULT_DEFAULT_QUANTUM;

 	manager->default_quantum = default_quantum;

 	INIT_LIST(manager->tasks);

-	INIT_LIST(manager->ready_tasks);

-	INIT_LIST(manager->ready_priority_tasks);

+	INIT_LIST(manager->ready_tasks[isc_taskqueue_normal]);

+	INIT_LIST(manager->ready_tasks[isc_taskqueue_slow]);

+	INIT_LIST(manager->ready_priority_tasks[isc_taskqueue_normal]);

+	INIT_LIST(manager->ready_priority_tasks[isc_taskqueue_slow]);

 	manager->tasks_running = 0;

 	manager->tasks_ready = 0;

 	manager->exclusive_requested = ISC_FALSE;

@@ -1447,7 +1494,19 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,

 	 * Start workers.

 	 */

 	for (i = 0; i < workers; i++) {

-		if (isc_thread_create(run, manager,

+		if (isc_thread_create(run_normal, manager,

+				      &manager->threads[manager->workers]) ==

+		    ISC_R_SUCCESS) {

+			char name[21];	/* thread name limit on Linux */

+			snprintf(name, sizeof(name), "isc-worker%04u", i);

+			isc_thread_setname(manager->threads[manager->workers],

+					   name);

+			manager->workers++;

+			started++;

+		}

+	}

+	for (; i < workers * 2; i++) {

+		if (isc_thread_create(run_slow, manager,

 				      &manager->threads[manager->workers]) ==

 		    ISC_R_SUCCESS) {

 			char name[16];	/* thread name limit on Linux */

@@ -1464,7 +1523,7 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,

 		manager_free(manager);

 		return (ISC_R_NOTHREADS);

 	}

-	isc_thread_setconcurrency(workers);

+	isc_thread_setconcurrency(workers * 2);

 #endif /* USE_WORKER_THREADS */

 #ifdef USE_SHARED_MANAGER

 	manager->refs = 1;

@@ -1479,7 +1538,8 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,

  cleanup_exclusivegranted:

 	(void)isc_condition_destroy(&manager->exclusive_granted);

  cleanup_workavailable:

-	(void)isc_condition_destroy(&manager->work_available);

+	(void)isc_condition_destroy(&manager->work_available[isc_taskqueue_slow]);

+	(void)isc_condition_destroy(&manager->work_available[isc_taskqueue_normal]);

  cleanup_threads:

 	isc_mem_free(mctx, manager->threads);

  cleanup_lock:

@@ -1564,7 +1624,7 @@ isc__taskmgr_destroy(isc_taskmgr_t **managerp) {

 	     task = NEXT(task, link)) {

 		LOCK(&task->lock);

 		if (task_shutdown(task))

-			push_readyq(manager, task);

+			push_readyq(manager, task, task->qid);

 		UNLOCK(&task->lock);

 	}

 #ifdef USE_WORKER_THREADS

@@ -1573,7 +1633,8 @@ isc__taskmgr_destroy(isc_taskmgr_t **managerp) {

 	 * there's work left to do, and if there are already no tasks left

 	 * it will cause the workers to see manager->exiting.

 	 */

-	BROADCAST(&manager->work_available);

+	BROADCAST(&manager->work_available[isc_taskqueue_normal]);