Blame SOURCES/0009-Ticket-47504-idlistscanlimit-per-index-type-value.patch

ba46c7
From f0d8c0c43d9c6ebd82cb0a223ad162b1c2fbb8f5 Mon Sep 17 00:00:00 2001
ba46c7
From: Rich Megginson <rmeggins@redhat.com>
ba46c7
Date: Mon, 16 Sep 2013 09:49:14 -0600
ba46c7
Subject: [PATCH 09/28] Ticket #47504 idlistscanlimit per index/type/value
ba46c7
ba46c7
https://fedorahosted.org/389/ticket/47504
ba46c7
Reviewed by: nhosoi (Thanks!)
ba46c7
Branch: 389-ds-base-1.3.1
ba46c7
Fix Description: Added a new attribute nsIndexIDListScanLimit to nsIndex
ba46c7
 nsIndexIDListScanLimit: limit=NNN [type=eq[,sub,...]] [flags=ADD[,XXX..]] [values=val[,val...]]
ba46c7
* The limit is the idlistscanlimit to apply.  The value can be -1 (unlimited), 0
ba46c7
(do not use the index at all), or a number to use for the idlistscanlimit.
ba46c7
* The type is a comma-delimited list of index types (eq,sub,etc.) to which the
ba46c7
limit applies.  The index type must be one of the index types configured for the index (you can't configure an id list for type=sub if you do not already have
ba46c7
a substring index configured for the attribute).
ba46c7
* The flags are a comma-delimited list of additional criteria which must match.
ba46c7
** flags ADD - only apply the limit when the index is used in an AND (&) filter
ba46c7
The values are a comma-delimited list of values which must match in the search
ba46c7
filter in order for the limit to be applied.  Since the matches are done one
ba46c7
at a time (we evaluate one filter component at a time), the values will match
ba46c7
if any of the values match.
ba46c7
* The values must be used with only one type at a time.  If values are specified,
ba46c7
type must be specified, and type must be a type that deals with values, such
ba46c7
as eq or sub.  There must be only one type specified - you can't specify values
ba46c7
if you use type=eq,pres or otherwise specify more than one type.  The values
ba46c7
must correspond to the index type (eq, sub), and must correspond to the syntax
ba46c7
of the attribute to which the index is applied - that is, if you have
ba46c7
attribute uidNumber (integer) and it is indexed for eq, you can't specify
ba46c7
type=eq values=abc because "abc" is not integer syntax.
ba46c7
If the values contain spaces, commas, nulls, other values which require
ba46c7
escapes, the LDAP filter escape syntax should be used - backslash '\' followed
ba46c7
by the 2 hex digit code for the character.
ba46c7
The values are processed as if they were filter values - so for "sub" values,
ba46c7
values like "values=*sm*ith*" will be processed as if they were values in a
ba46c7
substring search filter like (sn=*sm*ith*)
ba46c7
* nsIndexIDListScanLimit is multi-valued.  If a search matches more than one
ba46c7
nsIndexIDListScanLimit, the rules are applied in priority order.
ba46c7
The priority is as follows, from highest to lowest:
ba46c7
 * * match type, flags, value
ba46c7
 * * match type, value
ba46c7
 * * match type, flags
ba46c7
 * * match type
ba46c7
 * * match flags
ba46c7
 * For example, if you have
ba46c7
 * dn: cn=objectclass,...
ba46c7
 * objectclass: nsIndex
ba46c7
 * nsIndexType: eq
ba46c7
 * nsIndexIDListScanLimit: limit=0 type=eq flags=AND value=inetOrgPerson
ba46c7
 * nsIndexIDListScanLimit: limit=1 type=eq value=inetOrgPerson
ba46c7
 * nsIndexIDListScanLimit: limit=2 type=eq flags=AND
ba46c7
 * nsIndexIDListScanLimit: limit=3 type=eq
ba46c7
 * nsIndexIDListScanLimit: limit=4 flags=AND
ba46c7
 * nsIndexIDListScanLimit: limit=5
ba46c7
 * If the search filter is (&(objectclass=inetOrgPerson)(uid=foo)) then the limit=0 because all
ba46c7
 *  3 of type, flags, and value match
ba46c7
 * If the search filter is (objectclass=inetOrgPerson) then the limit=1 because type and value match
ba46c7
 *  but flag does not
ba46c7
 * If the search filter is (&(objectclass=posixAccount)(uid=foo)) the the limit=2 because type and
ba46c7
 *  flags match
ba46c7
 * If the search filter is (objectclass=posixAccount) then the limit=3 because only the type matches
ba46c7
 * If the search filter is (&(objectclass=*account*)(objectclass=*)) then the limit=4 because only
ba46c7
 *  flags match but not the types (sub and pres)
ba46c7
 * If the search filter is (objectclass=*account*) then the limit=5 because only the attribute matches
ba46c7
 *  but none of flags, type, or value matches
ba46c7
To add in testing/debugging, the LDAP_DEBUG_BACKLDBM log level is used to
ba46c7
print information about searches which exceed the idlistscanlimit.
ba46c7
Platforms tested: RHEL6 x86_64
ba46c7
Flag Day: no
ba46c7
Doc impact: yes - document new attribute
ba46c7
(cherry picked from commit 824b3019beffa5bf2bc5ab2a2a3e579d50833577)
ba46c7
(cherry picked from commit b348886030318cd43855ae439ec3f30f898b8cd4)
ba46c7
---
ba46c7
 ldap/schema/01core389.ldif                     |    3 +-
ba46c7
 ldap/servers/slapd/back-ldbm/ancestorid.c      |    2 +-
ba46c7
 ldap/servers/slapd/back-ldbm/back-ldbm.h       |    9 +
ba46c7
 ldap/servers/slapd/back-ldbm/filterindex.c     |   14 +-
ba46c7
 ldap/servers/slapd/back-ldbm/idl_new.c         |   17 +-
ba46c7
 ldap/servers/slapd/back-ldbm/index.c           |  121 ++++++-
ba46c7
 ldap/servers/slapd/back-ldbm/ldbm_attr.c       |  494 ++++++++++++++++++++++++
ba46c7
 ldap/servers/slapd/back-ldbm/proto-back-ldbm.h |    2 +-
ba46c7
 ldap/servers/slapd/proto-slap.h                |    1 +
ba46c7
 9 files changed, 646 insertions(+), 17 deletions(-)
ba46c7
ba46c7
diff --git a/ldap/schema/01core389.ldif b/ldap/schema/01core389.ldif
ba46c7
index 8ef702d..b9baae7 100644
ba46c7
--- a/ldap/schema/01core389.ldif
ba46c7
+++ b/ldap/schema/01core389.ldif
ba46c7
@@ -66,6 +66,7 @@ attributeTypes: ( 2.16.840.1.113730.3.1.593 NAME 'nsSNMPName' DESC 'Netscape def
ba46c7
 attributeTypes: ( 2.16.840.1.113730.3.1.242 NAME 'nsSystemIndex' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 X-ORIGIN 'Netscape Directory Server' )
ba46c7
 attributeTypes: ( 2.16.840.1.113730.3.1.327 NAME 'nsIndexType' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 X-ORIGIN 'Netscape Directory Server' )
ba46c7
 attributeTypes: ( 2.16.840.1.113730.3.1.328 NAME 'nsMatchingRule' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 X-ORIGIN 'Netscape Directory Server' )
ba46c7
+attributeTypes: ( 2.16.840.1.113730.3.1.2161 NAME 'nsIndexIDListScanLimit' DESC 'fine grained idlistscanlimit - per index/type/value' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 X-ORIGIN 'Netscape Directory Server' )
ba46c7
 attributeTypes: ( 2.16.840.1.113730.3.1.542 NAME 'nsUniqueId' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-ORIGIN 'Netscape Directory Server' )
ba46c7
 attributeTypes: ( 2.16.840.1.113730.3.1.543 NAME 'nsState' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-ORIGIN 'Netscape Directory Server' )
ba46c7
 attributeTypes: ( 2.16.840.1.113730.3.1.544 NAME 'nsParentUniqueId' DESC 'Netscape defined attribute type' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN 'Netscape Directory Server' )
ba46c7
@@ -158,7 +159,7 @@ attributeTypes: ( 2.16.840.1.113730.3.1.2156 NAME 'nsslapd-sasl-max-buffer-size'
ba46c7
 #
ba46c7
 objectClasses: ( 2.16.840.1.113730.3.2.40 NAME 'directoryServerFeature' DESC 'Netscape defined objectclass' SUP top MAY ( oid $ cn $ multiLineDescription ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
 objectClasses: ( 2.16.840.1.113730.3.2.41 NAME 'nsslapdPlugin' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsslapd-pluginPath $ nsslapd-pluginInitFunc $ nsslapd-pluginType $ nsslapd-pluginId $ nsslapd-pluginVersion $ nsslapd-pluginVendor $ nsslapd-pluginDescription $ nsslapd-pluginEnabled ) MAY ( nsslapd-pluginConfigArea $ nsslapd-plugin-depends-on-type ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
-objectClasses: ( 2.16.840.1.113730.3.2.44 NAME 'nsIndex' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsSystemIndex ) MAY ( description $ nsIndexType $ nsMatchingRule ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
+objectClasses: ( 2.16.840.1.113730.3.2.44 NAME 'nsIndex' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsSystemIndex ) MAY ( description $ nsIndexType $ nsMatchingRule $ nsIndexIDListScanLimit ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
 objectClasses: ( 2.16.840.1.113730.3.2.109 NAME 'nsBackendInstance' DESC 'Netscape defined objectclass' SUP top  MUST ( CN ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
 objectClasses: ( 2.16.840.1.113730.3.2.110 NAME 'nsMappingTree' DESC 'Netscape defined objectclass' SUP top  MUST ( CN ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
 objectClasses: ( 2.16.840.1.113730.3.2.104 NAME 'nsContainer' DESC 'Netscape defined objectclass' SUP top  MUST ( CN ) X-ORIGIN 'Netscape Directory Server' )
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/ancestorid.c b/ldap/servers/slapd/back-ldbm/ancestorid.c
ba46c7
index 2f32f8f..8722b68 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/ancestorid.c
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/ancestorid.c
ba46c7
@@ -1008,7 +1008,7 @@ int ldbm_ancestorid_read_ext(
ba46c7
     bv.bv_val = keybuf;
ba46c7
     bv.bv_len = PR_snprintf(keybuf, sizeof(keybuf), "%lu", (u_long)id);
ba46c7
 
ba46c7
-    *idl = index_read_ext_allids(be, LDBM_ANCESTORID_STR, indextype_EQUALITY, &bv, txn, &ret, NULL, allidslimit);
ba46c7
+    *idl = index_read_ext_allids(NULL, be, LDBM_ANCESTORID_STR, indextype_EQUALITY, &bv, txn, &ret, NULL, allidslimit);
ba46c7
 
ba46c7
     return ret;
ba46c7
 }
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h
ba46c7
index c5179d3..970f3c2 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h
ba46c7
@@ -462,6 +462,14 @@ typedef int (*dup_compare_fn_type)(
ba46c7
 #endif
ba46c7
 				const DBT *,const DBT *);
ba46c7
 
ba46c7
+struct index_idlistsizeinfo {
ba46c7
+	int ai_idlistsizelimit; /* max id list size */
ba46c7
+	int ai_indextype; /* index type */
ba46c7
+	unsigned int ai_flags;
ba46c7
+#define INDEX_ALLIDS_FLAG_AND 0x01
ba46c7
+	Slapi_ValueSet *ai_values; /* index keys to apply the max id list size to */
ba46c7
+};
ba46c7
+
ba46c7
 /* for the cache of attribute information (which are indexed, etc.) */
ba46c7
 struct attrinfo {
ba46c7
 	char	*ai_type;	  /* type name (cn, sn, ...)	*/
ba46c7
@@ -510,6 +518,7 @@ struct attrinfo {
ba46c7
 							 * the default length triplet is 2, 3, 2.
ba46c7
                              */
ba46c7
 	Slapi_Attr ai_sattr;	/* interface to syntax and matching rule plugins */
ba46c7
+	DataList *ai_idlistinfo; /* fine grained id list */
ba46c7
 };
ba46c7
 
ba46c7
 #define MAXDBCACHE	20
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/filterindex.c b/ldap/servers/slapd/back-ldbm/filterindex.c
ba46c7
index 489e85e..d1fddf9 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/filterindex.c
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/filterindex.c
ba46c7
@@ -67,6 +67,7 @@ static IDList * range_candidates(
ba46c7
 );
ba46c7
 static IDList *
ba46c7
 keys2idl(
ba46c7
+    Slapi_PBlock *pb,
ba46c7
     backend     *be,
ba46c7
     char        *type,
ba46c7
     const char  *indextype,
ba46c7
@@ -313,7 +314,7 @@ ava_candidates(
ba46c7
         ivals=ptr;
ba46c7
 
ba46c7
         slapi_attr_assertion2keys_ava_sv( &sattr, &tmp, (Slapi_Value ***)&ivals, LDAP_FILTER_EQUALITY_FAST);
ba46c7
-        idl = keys2idl( be, type, indextype, ivals, err, &unindexed, &txn, allidslimit );
ba46c7
+        idl = keys2idl( pb, be, type, indextype, ivals, err, &unindexed, &txn, allidslimit );
ba46c7
         if ( unindexed ) {
ba46c7
             unsigned int opnote = SLAPI_OP_NOTE_UNINDEXED;
ba46c7
             slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote );
ba46c7
@@ -345,7 +346,7 @@ ava_candidates(
ba46c7
             idl = idl_allids( be );
ba46c7
             goto done;
ba46c7
         }
ba46c7
-        idl = keys2idl( be, type, indextype, ivals, err, &unindexed, &txn, allidslimit );
ba46c7
+        idl = keys2idl( pb, be, type, indextype, ivals, err, &unindexed, &txn, allidslimit );
ba46c7
         if ( unindexed ) {
ba46c7
             unsigned int opnote = SLAPI_OP_NOTE_UNINDEXED;
ba46c7
             slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote );
ba46c7
@@ -382,7 +383,7 @@ presence_candidates(
ba46c7
         return( NULL );
ba46c7
     }
ba46c7
     slapi_pblock_get(pb, SLAPI_TXN, &txn.back_txn_txn);
ba46c7
-    idl = index_read_ext_allids( be, type, indextype_PRESENCE,
ba46c7
+    idl = index_read_ext_allids( pb, be, type, indextype_PRESENCE,
ba46c7
                                  NULL, &txn, err, &unindexed, allidslimit );
ba46c7
 
ba46c7
     if ( unindexed ) {
ba46c7
@@ -491,7 +492,7 @@ extensible_candidates(
ba46c7
                         {
ba46c7
                             int unindexed = 0;
ba46c7
                             IDList* idl3 = (mrOP == SLAPI_OP_EQUAL) ?
ba46c7
-                                index_read_ext_allids(be, mrTYPE, mrOID, *key, &txn,
ba46c7
+                                index_read_ext_allids(pb, be, mrTYPE, mrOID, *key, &txn,
ba46c7
                                                       err, &unindexed, allidslimit) :
ba46c7
                                 index_range_read_ext(pb, be, mrTYPE, mrOID, mrOP,
ba46c7
                                                      *key, NULL, 0, &txn, err, allidslimit);
ba46c7
@@ -928,7 +929,7 @@ substring_candidates(
ba46c7
      * IDLists together.
ba46c7
      */
ba46c7
     slapi_pblock_get(pb, SLAPI_TXN, &txn.back_txn_txn);
ba46c7
-    idl = keys2idl( be, type, indextype_SUB, ivals, err, &unindexed, &txn, allidslimit );
ba46c7
+    idl = keys2idl( pb, be, type, indextype_SUB, ivals, err, &unindexed, &txn, allidslimit );
ba46c7
     if ( unindexed ) {
ba46c7
         slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote );
ba46c7
         pagedresults_set_unindexed( pb->pb_conn, pb->pb_op, pr_idx );
ba46c7
@@ -942,6 +943,7 @@ substring_candidates(
ba46c7
 
ba46c7
 static IDList *
ba46c7
 keys2idl(
ba46c7
+    Slapi_PBlock *pb,
ba46c7
     backend     *be,
ba46c7
     char        *type,
ba46c7
     const char  *indextype,
ba46c7
@@ -961,7 +963,7 @@ keys2idl(
ba46c7
     for ( i = 0; ivals[i] != NULL; i++ ) {
ba46c7
         IDList    *idl2;
ba46c7
 
ba46c7
-        idl2 = index_read_ext_allids( be, type, indextype, slapi_value_get_berval(ivals[i]), txn, err, unindexed, allidslimit );
ba46c7
+        idl2 = index_read_ext_allids( pb, be, type, indextype, slapi_value_get_berval(ivals[i]), txn, err, unindexed, allidslimit );
ba46c7
 
ba46c7
 #ifdef LDAP_DEBUG
ba46c7
         /* XXX if ( slapd_ldap_debug & LDAP_DEBUG_TRACE ) { XXX */
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/idl_new.c b/ldap/servers/slapd/back-ldbm/idl_new.c
ba46c7
index 2b52f33..50ad5cb 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/idl_new.c
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/idl_new.c
ba46c7
@@ -274,7 +274,7 @@ idl_new_fetch(
ba46c7
             }
ba46c7
             memcpy(&id, dataret.data, sizeof(ID));
ba46c7
             if (id == lastid) { /* dup */
ba46c7
-                LDAPDebug1Arg(LDAP_DEBUG_TRACE, "Detedted duplicate id "
ba46c7
+                LDAPDebug1Arg(LDAP_DEBUG_TRACE, "Detected duplicate id "
ba46c7
                               "%d due to DB_MULTIPLE error - skipping\n",
ba46c7
                               id);
ba46c7
                 continue; /* get next one */
ba46c7
@@ -293,14 +293,17 @@ idl_new_fetch(
ba46c7
         }
ba46c7
 
ba46c7
         LDAPDebug(LDAP_DEBUG_TRACE, "bulk fetch buffer nids=%d\n", count, 0, 0); 
ba46c7
-#if defined(DB_ALLIDS_ON_READ)
ba46c7
+#if defined(DB_ALLIDS_ON_READ)	
ba46c7
         /* enforce the allids read limit */
ba46c7
         if ((NEW_IDL_NO_ALLID != *flag_err) && (NULL != a) &&
ba46c7
-             (idl != NULL) && idl_new_exceeds_allidslimit(count, a, allidslimit)) {
ba46c7
-            idl->b_nids = 1;
ba46c7
-            idl->b_ids[0] = ALLID;
ba46c7
-            ret = DB_NOTFOUND; /* fool the code below into thinking that we finished the dups */
ba46c7
-            break;
ba46c7
+            (idl != NULL) && idl_new_exceeds_allidslimit(count, a, allidslimit)) {
ba46c7
+        	idl->b_nids = 1;
ba46c7
+        	idl->b_ids[0] = ALLID;
ba46c7
+        	ret = DB_NOTFOUND; /* fool the code below into thinking that we finished the dups */
ba46c7
+        	LDAPDebug(LDAP_DEBUG_BACKLDBM, "search for key for attribute index %s "
ba46c7
+        		  "exceeded allidslimit %d - count is %d\n",
ba46c7
+        		  a->ai_type, allidslimit, count);
ba46c7
+        	break;
ba46c7
         }
ba46c7
 #endif
ba46c7
         ret = cursor->c_get(cursor,&key,&data,DB_NEXT_DUP|DB_MULTIPLE);
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c
ba46c7
index bb69c57..d5ca16a 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/index.c
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/index.c
ba46c7
@@ -52,6 +52,8 @@
ba46c7
 static const char *errmsg = "database index operation failed";
ba46c7
 
ba46c7
 static int   is_indexed (const char* indextype, int indexmask, char** index_rules);
ba46c7
+static int index_get_allids( int default_allids, const char *indextype, struct attrinfo *ai, const struct berval *val, unsigned int flags );
ba46c7
+
ba46c7
 static Slapi_Value **
ba46c7
 valuearray_minus_valuearray(
ba46c7
     const Slapi_Attr *sattr, 
ba46c7
@@ -888,6 +890,7 @@ index_read(
ba46c7
  */
ba46c7
 IDList *
ba46c7
 index_read_ext_allids(
ba46c7
+    Slapi_PBlock *pb,
ba46c7
     backend *be,
ba46c7
     char		*type,
ba46c7
     const char		*indextype,
ba46c7
@@ -910,6 +913,8 @@ index_read_ext_allids(
ba46c7
 	char		*basetmp, *basetype;
ba46c7
 	int retry_count  = 0;
ba46c7
 	struct berval	*encrypted_val = NULL;
ba46c7
+	int is_and = 0;
ba46c7
+	unsigned int ai_flags = 0;
ba46c7
 
ba46c7
 	*err = 0;
ba46c7
 
ba46c7
@@ -976,6 +981,23 @@ index_read_ext_allids(
ba46c7
 		slapi_ch_free_string( &basetmp );
ba46c7
 		return( idl );
ba46c7
 	}
ba46c7
+	if (pb) {
ba46c7
+		slapi_pblock_get(pb, SLAPI_SEARCH_IS_AND, &is_and);
ba46c7
+	}
ba46c7
+	ai_flags = is_and ? INDEX_ALLIDS_FLAG_AND : 0;
ba46c7
+	allidslimit = index_get_allids( allidslimit, indextype, ai, val, ai_flags );
ba46c7
+	if (allidslimit == 0) {
ba46c7
+		idl = idl_allids( be );
ba46c7
+		if (unindexed != NULL) *unindexed = 1;
ba46c7
+		LDAPDebug1Arg( LDAP_DEBUG_BACKLDBM, "<= index_read %lu candidates "
ba46c7
+		    "(do not use index)\n", (u_long)IDL_NIDS(idl) );
ba46c7
+		LDAPDebug( LDAP_DEBUG_BACKLDBM, "<= index_read index attr %s type %s "
ba46c7
+		    "for value %s does not use index\n", basetype, indextype,
ba46c7
+		    (val && val->bv_val) ? val->bv_val : "ALL" );
ba46c7
+		index_free_prefix( prefix );
ba46c7
+		slapi_ch_free_string( &basetmp );
ba46c7
+		return( idl );
ba46c7
+	}
ba46c7
 	if ( (*err = dblayer_get_index_file( be, ai, &db, DBOPEN_CREATE )) != 0 ) {
ba46c7
 		LDAPDebug( LDAP_DEBUG_TRACE,
ba46c7
 		    "<= index_read NULL (index file open for attr %s)\n",
ba46c7
@@ -1063,7 +1085,7 @@ index_read_ext(
ba46c7
     int			*unindexed
ba46c7
 )
ba46c7
 {
ba46c7
-    return index_read_ext_allids(be, type, indextype, val, txn, err, unindexed, 0);
ba46c7
+    return index_read_ext_allids(NULL, be, type, indextype, val, txn, err, unindexed, 0);
ba46c7
 }
ba46c7
 
ba46c7
 /* This function compares two index keys.  It is assumed
ba46c7
@@ -2341,3 +2363,100 @@ valuearray_minus_valuearray(
ba46c7
 
ba46c7
     return c;
ba46c7
 }
ba46c7
+
ba46c7
+/*
ba46c7
+ * Find the most specific match for the given index type, flags, and value, and return the allids value
ba46c7
+ * for that match.  The priority is as follows, from highest to lowest:
ba46c7
+ * * match type, flags, value
ba46c7
+ * * match type, value
ba46c7
+ * * match type, flags
ba46c7
+ * * match type
ba46c7
+ * * match flags
ba46c7
+ * Note that for value to match, the type must be one that supports values e.g. eq or sub, so that
ba46c7
+ * in order for value to match, there must be a type
ba46c7
+ * For example, if you have
ba46c7
+ * dn: cn=objectclass,...
ba46c7
+ * objectclass: nsIndex
ba46c7
+ * nsIndexType: eq
ba46c7
+ * nsIndexIDListScanLimit: limit=0 type=eq flags=AND value=inetOrgPerson
ba46c7
+ * nsIndexIDListScanLimit: limit=1 type=eq value=inetOrgPerson
ba46c7
+ * nsIndexIDListScanLimit: limit=2 type=eq flags=AND
ba46c7
+ * nsIndexIDListScanLimit: limit=3 type=eq
ba46c7
+ * nsIndexIDListScanLimit: limit=4 flags=AND
ba46c7
+ * nsIndexIDListScanLimit: limit=5
ba46c7
+ * If the search filter is (&(objectclass=inetOrgPerson)(uid=foo)) then the limit=0 because all
ba46c7
+ *  3 of type, flags, and value match
ba46c7
+ * If the search filter is (objectclass=inetOrgPerson) then the limit=1 because type and value match
ba46c7
+ *  but flag does not
ba46c7
+ * If the search filter is (&(objectclass=posixAccount)(uid=foo)) the the limit=2 because type and
ba46c7
+ *  flags match
ba46c7
+ * If the search filter is (objectclass=posixAccount) then the limit=3 because only the type matches
ba46c7
+ * If the search filter is (&(objectclass=*account*)(objectclass=*)) then the limit=4 because only
ba46c7
+ *  flags match but not the types (sub and pres)
ba46c7
+ * If the search filter is (objectclass=*account*) then the limit=5 because only the attribute matches
ba46c7
+ *  but none of flags, type, or value matches
ba46c7
+ */
ba46c7
+#define AI_HAS_VAL 0x04
ba46c7
+#define AI_HAS_TYPE 0x02
ba46c7
+#define AI_HAS_FLAG 0x01
ba46c7
+static int
ba46c7
+index_get_allids( int default_allids, const char *indextype, struct attrinfo *ai, const struct berval *val, unsigned int flags )
ba46c7
+{
ba46c7
+    int allids = default_allids;
ba46c7
+    Slapi_Value sval;
ba46c7
+    struct index_idlistsizeinfo *iter; /* iterator */
ba46c7
+    int cookie = 0;
ba46c7
+    int best_score = 0;
ba46c7
+    struct index_idlistsizeinfo *best_match = NULL;
ba46c7
+
ba46c7
+    if (!ai->ai_idlistinfo) {
ba46c7
+        return allids;
ba46c7
+    }
ba46c7
+
ba46c7
+    if (val) { /* val should already be a Slapi_Value, but some paths do not use Slapi_Value */
ba46c7
+        sval.bv.bv_val = val->bv_val;
ba46c7
+        sval.bv.bv_len = val->bv_len;
ba46c7
+        sval.v_csnset = NULL;
ba46c7
+        sval.v_flags = SLAPI_ATTR_FLAG_NORMALIZED; /* the value must be a normalized key */
ba46c7
+    }
ba46c7
+
ba46c7
+    /* loop through all of the idlistinfo objects to find the best match */
ba46c7
+    for (iter = (struct index_idlistsizeinfo *)dl_get_first(ai->ai_idlistinfo, &cookie); iter;
ba46c7
+         iter = (struct index_idlistsizeinfo *)dl_get_next(ai->ai_idlistinfo, &cookie)) {
ba46c7
+        int iter_score = 0;
ba46c7
+
ba46c7
+        if (iter->ai_indextype != 0) { /* info defines a type which must match */
ba46c7
+            if (is_indexed(indextype, iter->ai_indextype, ai->ai_index_rules)) {
ba46c7
+                iter_score |= AI_HAS_TYPE;
ba46c7
+            } else {
ba46c7
+                continue; /* does not match, go to next one */
ba46c7
+            }
ba46c7
+        }
ba46c7
+        if (iter->ai_flags != 0) {
ba46c7
+            if (flags & iter->ai_flags) {
ba46c7
+                iter_score |= AI_HAS_FLAG;
ba46c7
+            } else {
ba46c7
+                continue; /* does not match, go to next one */
ba46c7
+            }
ba46c7
+        }
ba46c7
+        if (iter->ai_values != NULL) {
ba46c7
+            if ((val != NULL) && slapi_valueset_find(&ai->ai_sattr, iter->ai_values, &sval)) {
ba46c7
+                iter_score |= AI_HAS_VAL;
ba46c7
+            } else {
ba46c7
+                continue; /* does not match, go to next one */
ba46c7
+            }
ba46c7
+        }
ba46c7
+
ba46c7
+        if (iter_score >= best_score) {
ba46c7
+            best_score = iter_score;
ba46c7
+            best_match = iter;
ba46c7
+        }
ba46c7
+    }
ba46c7
+
ba46c7
+    if (best_match) {
ba46c7
+        allids = best_match->ai_idlistsizelimit;
ba46c7
+    }
ba46c7
+
ba46c7
+    return allids;
ba46c7
+}
ba46c7
+
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_attr.c b/ldap/servers/slapd/back-ldbm/ldbm_attr.c
ba46c7
index fd43ce9..16ffa42 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/ldbm_attr.c
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/ldbm_attr.c
ba46c7
@@ -44,6 +44,22 @@
ba46c7
 
ba46c7
 #include "back-ldbm.h"
ba46c7
 
ba46c7
+static void
ba46c7
+attr_index_idlistsize_done(struct index_idlistsizeinfo *idlinfo)
ba46c7
+{
ba46c7
+    if (idlinfo) {
ba46c7
+        slapi_valueset_free(idlinfo->ai_values);
ba46c7
+        idlinfo->ai_values = NULL;
ba46c7
+    }
ba46c7
+}
ba46c7
+
ba46c7
+static void
ba46c7
+attr_index_idlistsize_free(struct index_idlistsizeinfo **idlinfo)
ba46c7
+{
ba46c7
+    attr_index_idlistsize_done(*idlinfo);
ba46c7
+    slapi_ch_free((void **)idlinfo);
ba46c7
+}
ba46c7
+
ba46c7
 struct attrinfo *
ba46c7
 attrinfo_new()
ba46c7
 {
ba46c7
@@ -52,6 +68,15 @@ attrinfo_new()
ba46c7
 }
ba46c7
 
ba46c7
 void
ba46c7
+attrinfo_delete_idlistinfo(DataList **idlinfo_dl)
ba46c7
+{
ba46c7
+    if (idlinfo_dl && *idlinfo_dl) {
ba46c7
+        dl_cleanup(*idlinfo_dl, (FREEFN)attr_index_idlistsize_free);
ba46c7
+        dl_free(idlinfo_dl);
ba46c7
+    }
ba46c7
+}
ba46c7
+
ba46c7
+void
ba46c7
 attrinfo_delete(struct attrinfo **pp)
ba46c7
 {
ba46c7
     if(pp!=NULL && *pp!=NULL)
ba46c7
@@ -62,6 +87,7 @@ attrinfo_delete(struct attrinfo **pp)
ba46c7
         slapi_ch_free((void**)(*pp)->ai_index_rules);
ba46c7
         slapi_ch_free((void**)&((*pp)->ai_attrcrypt));
ba46c7
         attr_done(&((*pp)->ai_sattr));
ba46c7
+        attrinfo_delete_idlistinfo(&(*pp)->ai_idlistinfo);
ba46c7
         slapi_ch_free((void**)pp);
ba46c7
         *pp= NULL;
ba46c7
     }
ba46c7
@@ -126,6 +152,10 @@ ainfo_dup(
ba46c7
   if ( b->ai_indexmask & INDEX_RULES ) {
ba46c7
     charray_merge( &a->ai_index_rules, b->ai_index_rules, 1 );
ba46c7
   }
ba46c7
+  /* free the old idlistinfo from a - transfer the list from b to a */
ba46c7
+  attrinfo_delete_idlistinfo(&a->ai_idlistinfo);
ba46c7
+  a->ai_idlistinfo = b->ai_idlistinfo;
ba46c7
+  b->ai_idlistinfo = NULL;
ba46c7
   
ba46c7
   return( 1 );
ba46c7
 }
ba46c7
@@ -166,6 +196,464 @@ _set_attr_substrlen(int index, char *str, int **substrlens)
ba46c7
 	}
ba46c7
 }
ba46c7
 
ba46c7
+#define NS_INDEX_IDLISTSCANLIMIT "nsIndexIDListScanLimit"
ba46c7
+#define LIMIT_KW "limit="
ba46c7
+#define LIMIT_LEN sizeof(LIMIT_KW)-1
ba46c7
+#define TYPE_KW "type="
ba46c7
+#define TYPE_LEN sizeof(TYPE_KW)-1
ba46c7
+#define FLAGS_KW "flags="
ba46c7
+#define FLAGS_LEN sizeof(FLAGS_KW)-1
ba46c7
+#define VALUES_KW "values="
ba46c7
+#define VALUES_LEN sizeof(VALUES_KW)-1
ba46c7
+#define FLAGS_AND_KW "AND"
ba46c7
+#define FLAGS_AND_LEN sizeof(FLAGS_AND_KW)-1
ba46c7
+
ba46c7
+static int
ba46c7
+attr_index_parse_idlistsize_values(Slapi_Attr *attr, struct index_idlistsizeinfo *idlinfo, char *values, const char *strval, char *returntext)
ba46c7
+{
ba46c7
+	int rc = 0;
ba46c7
+	/* if we are here, values is non-NULL and not an empty string - parse it */
ba46c7
+	char *ptr = NULL;
ba46c7
+	char *lasts = NULL;
ba46c7
+	char *val;
ba46c7
+	int syntaxcheck = config_get_syntaxcheck();
ba46c7
+	IFP syntax_validate_fn = syntaxcheck ? attr->a_plugin->plg_syntax_validate : NULL;
ba46c7
+	char staticfiltstrbuf[1024]; /* for small filter strings */
ba46c7
+	char *filtstrbuf = staticfiltstrbuf; /* default if not malloc'd */
ba46c7
+	size_t filtstrbuflen = sizeof(staticfiltstrbuf); /* default if not malloc'd */
ba46c7
+	Slapi_Filter *filt = NULL; /* for filter converting/unescaping config values */
ba46c7
+
ba46c7
+	/* caller should have already checked that values is valid and contains a "=" */
ba46c7
+	PR_ASSERT(values);
ba46c7
+	ptr = PL_strchr(values, '=');
ba46c7
+	PR_ASSERT(ptr);
ba46c7
+	++ptr;
ba46c7
+	for (val = ldap_utf8strtok_r(ptr, ",", &lasts); val;
ba46c7
+	     val = ldap_utf8strtok_r(NULL, ",", &lasts)) {
ba46c7
+		Slapi_Value **ivals= NULL; /* for config values converted to keys */
ba46c7
+		int ii;
ba46c7
+#define FILT_TEMPL_BEGIN "(a="
ba46c7
+#define FILT_TEMPL_END ")"
ba46c7
+		size_t filttemplen = sizeof(FILT_TEMPL_BEGIN) - 1 + sizeof(FILT_TEMPL_END) - 1;
ba46c7
+		size_t vallen = strlen(val);
ba46c7
+
ba46c7
+		if ((vallen + filttemplen + 1) > filtstrbuflen) {
ba46c7
+			filtstrbuflen = vallen + filttemplen + 1;
ba46c7
+			if (filtstrbuf == staticfiltstrbuf) {
ba46c7
+				filtstrbuf = (char *)slapi_ch_malloc(sizeof(char) * filtstrbuflen);
ba46c7
+			} else {
ba46c7
+				filtstrbuf = (char *)slapi_ch_realloc(filtstrbuf, sizeof(char) * filtstrbuflen);
ba46c7
+			}
ba46c7
+		}
ba46c7
+		/* each value is a value from a filter which should be escaped like a filter value
ba46c7
+		 * for each value, create a dummy filter string, then parse and unescape it just
ba46c7
+		 * like a filter
ba46c7
+		 */
ba46c7
+		PR_snprintf(filtstrbuf, filtstrbuflen, FILT_TEMPL_BEGIN "%s" FILT_TEMPL_END, val);
ba46c7
+		filt = slapi_str2filter(filtstrbuf);
ba46c7
+		if (!filt) {
ba46c7
+			rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+				    "attr_index_parse_idlistsize: invalid value %s in %s",
ba46c7
+				    val, strval);
ba46c7
+			break;
ba46c7
+		}
ba46c7
+
ba46c7
+		if (idlinfo->ai_indextype == INDEX_SUB) {
ba46c7
+			if (syntax_validate_fn) {
ba46c7
+				/* see if the values match the syntax, but only if checking is enabled */
ba46c7
+				char **subany = filt->f_sub_any;
ba46c7
+				struct berval bv;
ba46c7
+
ba46c7
+				if (filt->f_sub_initial && *filt->f_sub_initial) {
ba46c7
+					bv.bv_val = filt->f_sub_initial;
ba46c7
+					bv.bv_len = strlen(bv.bv_val);
ba46c7
+					if ((rc = syntax_validate_fn(&bv))) {
ba46c7
+						rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+						PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+						            "attr_index_parse_idlistsize: initial substring value %s "
ba46c7
+						            "in value %s violates syntax for attribute %s",
ba46c7
+						            bv.bv_val, val, attr->a_type);
ba46c7
+						break;
ba46c7
+					}
ba46c7
+				}
ba46c7
+				for (; !rc && subany && *subany; ++subany) {
ba46c7
+					char *subval = *subany;
ba46c7
+					if (*subval) {
ba46c7
+						bv.bv_val = subval;
ba46c7
+						bv.bv_len = strlen(bv.bv_val);
ba46c7
+						if ((rc = syntax_validate_fn(&bv))) {
ba46c7
+							rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+							PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+								    "attr_index_parse_idlistsize: initial substring value %s in "
ba46c7
+								    "value %s violates syntax for attribute %s",
ba46c7
+								    bv.bv_val, val, attr->a_type);
ba46c7
+							break;
ba46c7
+						}
ba46c7
+					}
ba46c7
+				}
ba46c7
+				if (rc) {
ba46c7
+					break;
ba46c7
+				}
ba46c7
+				if (filt->f_sub_final) {
ba46c7
+					bv.bv_val = filt->f_sub_final;
ba46c7
+					bv.bv_len = strlen(bv.bv_val);
ba46c7
+					if ((rc = syntax_validate_fn(&bv))) {
ba46c7
+						rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+						PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+							    "attr_index_parse_idlistsize: final substring value %s in value "
ba46c7
+							    "%s violates syntax for attribute %s",
ba46c7
+							    bv.bv_val, val, attr->a_type);
ba46c7
+						break;
ba46c7
+					}
ba46c7
+				}
ba46c7
+			}
ba46c7
+			/* if we are here, values passed syntax or no checking */
ba46c7
+			/* generate index keys */
ba46c7
+			(void)slapi_attr_assertion2keys_sub_sv(attr, filt->f_sub_initial, filt->f_sub_any, filt->f_sub_final, &ivals);
ba46c7
+
ba46c7
+		} else if (idlinfo->ai_indextype == INDEX_EQUALITY) {
ba46c7
+			Slapi_Value sval;
ba46c7
+			/* see if the value matches the syntax, but only if checking is enabled */
ba46c7
+			if (syntax_validate_fn && ((rc = syntax_validate_fn(&filt->f_avvalue)))) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: value %s violates syntax for attribute %s",
ba46c7
+					    val, attr->a_type);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+
ba46c7
+			sval.bv.bv_val = filt->f_avvalue.bv_val;
ba46c7
+			sval.bv.bv_len = filt->f_avvalue.bv_len;
ba46c7
+			sval.v_flags = 0;
ba46c7
+			sval.v_csnset = NULL;
ba46c7
+			(void)slapi_attr_assertion2keys_ava_sv(attr, &sval, (Slapi_Value ***)&ivals, LDAP_FILTER_EQUALITY);
ba46c7
+		}
ba46c7
+		/* don't need filter any more */
ba46c7
+		slapi_filter_free(filt, 1);
ba46c7
+		filt = NULL;
ba46c7
+
ba46c7
+		/* add value(s) in ivals to our value set - disallow duplicates with error */
ba46c7
+		for (ii = 0; !rc && ivals && ivals[ii]; ++ii) {
ba46c7
+			if (idlinfo->ai_values &&
ba46c7
+			    slapi_valueset_find(attr, idlinfo->ai_values, ivals[ii])) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: duplicate value %s in %s",
ba46c7
+					    slapi_value_get_string(ivals[ii]), val);
ba46c7
+				slapi_value_free(&ivals[ii]);
ba46c7
+			} else {
ba46c7
+				if (!idlinfo->ai_values) {
ba46c7
+					idlinfo->ai_values = slapi_valueset_new();
ba46c7
+				}
ba46c7
+				slapi_valueset_add_value_ext(idlinfo->ai_values, ivals[ii], SLAPI_VALUE_FLAG_PASSIN);
ba46c7
+			}
ba46c7
+		}
ba46c7
+		/* only free members of ivals that were not moved to ai_values */
ba46c7
+		valuearray_free_ext(&ivals, ii);
ba46c7
+		ivals = NULL;
ba46c7
+	}
ba46c7
+
ba46c7
+	slapi_filter_free(filt, 1);
ba46c7
+
ba46c7
+	if (filtstrbuf != staticfiltstrbuf) {
ba46c7
+		slapi_ch_free_string(&filtstrbuf);
ba46c7
+	}
ba46c7
+
ba46c7
+	return rc;
ba46c7
+}
ba46c7
+
ba46c7
+static int
ba46c7
+attr_index_parse_idlistsize_limit(char *ptr, struct index_idlistsizeinfo *idlinfo, char *returntext)
ba46c7
+{
ba46c7
+	int rc = 0;
ba46c7
+	char *endptr;
ba46c7
+
ba46c7
+	PR_ASSERT(ptr && (*ptr == '='));
ba46c7
+	ptr++;
ba46c7
+	idlinfo->ai_idlistsizelimit = strtol(ptr, &endptr, 10);
ba46c7
+	if (*endptr) { /* error in parsing */
ba46c7
+		rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+		PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+		            "attr_index_parse_idlistsize: value %s for %s is not valid - "
ba46c7
+		            "must be an integer >= -1",
ba46c7
+		            ptr, LIMIT_KW);
ba46c7
+	} else if (idlinfo->ai_idlistsizelimit < -1) {
ba46c7
+		rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+		PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+			    "attr_index_parse_idlistsize: value %s for %s "
ba46c7
+			    "must be an integer >= -1",
ba46c7
+			    ptr, LIMIT_KW);
ba46c7
+	}
ba46c7
+	return rc;
ba46c7
+}
ba46c7
+
ba46c7
+static int
ba46c7
+attr_index_parse_idlistsize_type(char *ptr, struct attrinfo *ai, struct index_idlistsizeinfo *idlinfo, const char *val, const char *strval, char *returntext)
ba46c7
+{
ba46c7
+	int rc = 0;
ba46c7
+	char *ptr_next;
ba46c7
+	size_t len;
ba46c7
+	size_t preslen = strlen(indextype_PRESENCE);
ba46c7
+	size_t eqlen = strlen(indextype_EQUALITY);
ba46c7
+	size_t sublen = strlen(indextype_SUB);
ba46c7
+
ba46c7
+	PR_ASSERT(ptr && (*ptr == '='));
ba46c7
+	do {
ba46c7
+		++ptr;
ba46c7
+		ptr_next = PL_strchr(ptr, ','); /* find next comma */
ba46c7
+		if (!ptr_next) {
ba46c7
+			ptr_next = PL_strchr(ptr, '\0'); /* find end of string */
ba46c7
+		}
ba46c7
+		len = ptr_next-ptr;
ba46c7
+		if ((len == preslen) && !PL_strncmp(ptr, indextype_PRESENCE, len)) {
ba46c7
+			if (idlinfo->ai_indextype & INDEX_PRESENCE) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: duplicate %s in value %s for %s",
ba46c7
+					    indextype_PRESENCE, val, strval);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			if (!(ai->ai_indexmask & INDEX_PRESENCE)) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: attribute %s does not have index type %s",
ba46c7
+					    ai->ai_type, indextype_PRESENCE);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			idlinfo->ai_indextype |= INDEX_PRESENCE;
ba46c7
+		} else if ((len == eqlen) && !PL_strncmp(ptr, indextype_EQUALITY, len)) {
ba46c7
+			if (idlinfo->ai_indextype & INDEX_EQUALITY) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: duplicate %s in value %s for %s",
ba46c7
+					    indextype_EQUALITY, val, strval);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			if (!(ai->ai_indexmask & INDEX_EQUALITY)) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: attribute %s does not have index type %s",
ba46c7
+					    ai->ai_type, indextype_EQUALITY);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			idlinfo->ai_indextype |= INDEX_EQUALITY;
ba46c7
+		} else if ((len == sublen) && !PL_strncmp(ptr, indextype_SUB, len)) {
ba46c7
+			if (idlinfo->ai_indextype & INDEX_SUB) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: duplicate %s in value %s for %s",
ba46c7
+					    indextype_SUB, val, strval);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			if (!(ai->ai_indexmask & INDEX_SUB)) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+				PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+					    "attr_index_parse_idlistsize: attribute %s does not have index type %s",
ba46c7
+					    ai->ai_type, indextype_SUB);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			idlinfo->ai_indextype |= INDEX_SUB;
ba46c7
+		} else {
ba46c7
+			rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+				    "attr_index_parse_idlistsize: unknown or unsupported index type "
ba46c7
+				    "%s in value %s for %s",
ba46c7
+				    ptr, val, strval);
ba46c7
+			break;
ba46c7
+		}
ba46c7
+	} while ((ptr = PL_strchr(ptr, ',')));
ba46c7
+
ba46c7
+	return rc;
ba46c7
+}
ba46c7
+
ba46c7
+static int
ba46c7
+attr_index_parse_idlistsize_flags(char *ptr, struct index_idlistsizeinfo *idlinfo, const char *val, const char *strval, char *returntext)
ba46c7
+{
ba46c7
+	int rc = 0;
ba46c7
+	char *ptr_next;
ba46c7
+	size_t len;
ba46c7
+
ba46c7
+	PR_ASSERT(ptr && (*ptr == '='));
ba46c7
+	do {
ba46c7
+		++ptr;
ba46c7
+		ptr_next = PL_strchr(ptr, ','); /* find next comma */
ba46c7
+		if (!ptr_next) {
ba46c7
+			ptr_next = PL_strchr(ptr, '\0'); /* find end of string */
ba46c7
+		}
ba46c7
+		len = ptr_next-ptr;
ba46c7
+		if ((len == FLAGS_AND_LEN) && !PL_strncmp(ptr, FLAGS_AND_KW, len)) {
ba46c7
+			if (idlinfo->ai_flags & INDEX_ALLIDS_FLAG_AND) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+			        	    "attr_index_parse_idlistsize: duplicate %s in value %s for %s",
ba46c7
+			        	    FLAGS_AND_KW, val, strval);
ba46c7
+				break;
ba46c7
+			}
ba46c7
+			idlinfo->ai_flags |= INDEX_ALLIDS_FLAG_AND;
ba46c7
+		} else {
ba46c7
+			rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+		        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+		        	    "attr_index_parse_idlistsize: unknown or unsupported flags %s in value %s for %s",
ba46c7
+	        	            ptr, val, strval);
ba46c7
+			break;
ba46c7
+		}
ba46c7
+
ba46c7
+	} while ((ptr = PL_strchr(ptr, ',')));
ba46c7
+	return rc;
ba46c7
+}
ba46c7
+
ba46c7
+static int
ba46c7
+attr_index_parse_idlistsize(struct attrinfo *ai, const char *strval, struct index_idlistsizeinfo *idlinfo, char *returntext)
ba46c7
+{
ba46c7
+	int rc = 0; /* assume success */
ba46c7
+	char *mystr = slapi_ch_strdup(strval); /* copy for strtok */
ba46c7
+	char *values = NULL;
ba46c7
+	char *lasts, *val, *ptr;
ba46c7
+	int seen_limit = 0, seen_type = 0, seen_flags = 0, seen_values = 0;
ba46c7
+	Slapi_Attr *attr = &ai->ai_sattr;
ba46c7
+
ba46c7
+	if (!mystr) {
ba46c7
+		rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+	        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+	        	    "attr_index_parse_idlistsize: value is empty");
ba46c7
+		goto done;
ba46c7
+	}
ba46c7
+
ba46c7
+	for (val = ldap_utf8strtok_r(mystr, " ", &lasts); val;
ba46c7
+	     val = ldap_utf8strtok_r(NULL, " ", &lasts)) {
ba46c7
+		ptr = PL_strchr(val, '=');
ba46c7
+		if (!ptr || !(*(ptr+1))) {
ba46c7
+			rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+		        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+		        	    "attr_index_parse_idlistsize: invalid value %s - should be keyword=value - in %s",
ba46c7
+		                    val, strval);
ba46c7
+			goto done;
ba46c7
+		}
ba46c7
+		/* ptr points at first '=' in val */
ba46c7
+		if (!PL_strncmp(val, LIMIT_KW, LIMIT_LEN)) {
ba46c7
+			if (seen_limit) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+			        	    "attr_index_parse_idlistsize: can have only 1 %s in value %s",
ba46c7
+			                    LIMIT_KW, strval);
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+			if ((rc = attr_index_parse_idlistsize_limit(ptr, idlinfo, returntext))) {
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+			seen_limit = 1;
ba46c7
+		} else if (!PL_strncmp(val, TYPE_KW, TYPE_LEN)) {
ba46c7
+			if (seen_type) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+			        	    "attr_index_parse_idlistsize: can have only 1 %s in value %s",
ba46c7
+			                    TYPE_KW, strval);
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+			if ((rc = attr_index_parse_idlistsize_type(ptr, ai, idlinfo, val, strval, returntext))) {
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+
ba46c7
+			seen_type = 1;
ba46c7
+		} else if (!PL_strncmp(val, FLAGS_KW, FLAGS_LEN)) {
ba46c7
+			if (seen_flags) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+			        	    "attr_index_parse_idlistsize: can have only 1 %s in value %s",
ba46c7
+			                    FLAGS_KW, strval);
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+			if ((rc = attr_index_parse_idlistsize_flags(ptr, idlinfo, val, strval, returntext))) {
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+			seen_flags = 1;
ba46c7
+		} else if (!PL_strncmp(val, VALUES_KW, VALUES_LEN)) {
ba46c7
+			if (seen_values) {
ba46c7
+				rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+			        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+			        	    "attr_index_parse_idlistsize: can have only 1 %s in value %s",
ba46c7
+			                    VALUES_KW, strval);
ba46c7
+				goto done;
ba46c7
+			}
ba46c7
+			values = val;
ba46c7
+			seen_values = 1;
ba46c7
+		} else {
ba46c7
+			rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+		        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+		        	    "attr_index_parse_idlistsize: unknown keyword %s in %s",
ba46c7
+		                    val, strval);
ba46c7
+			goto done;
ba46c7
+		}
ba46c7
+	}
ba46c7
+
ba46c7
+	if (!seen_limit) {
ba46c7
+		rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+	        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+	        	    "attr_index_parse_idlistsize: no limit specified in %s",
ba46c7
+	                    strval);
ba46c7
+		goto done;
ba46c7
+	}
ba46c7
+
ba46c7
+	/* parse values last
ba46c7
+	 * can only have values if type is eq or sub, and only eq by itself or sub by itself
ba46c7
+	 * eq and sub type values cannot be mixed, so error in that case
ba46c7
+	 * cannot have type pres,eq and values - pres must be by itself with no values
ba46c7
+	 */
ba46c7
+	if (values) {
ba46c7
+		if (idlinfo->ai_indextype == INDEX_EQUALITY) {
ba46c7
+			; /* ok */
ba46c7
+		} else if (idlinfo->ai_indextype == INDEX_SUB) {
ba46c7
+			; /* ok */
ba46c7
+		} else {
ba46c7
+			rc = LDAP_UNWILLING_TO_PERFORM;
ba46c7
+		        PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE,
ba46c7
+		        	    "attr_index_parse_idlistsize: if %s is specified, the %s "
ba46c7
+		                    "must be %s or %s - not both, and not any other types",
ba46c7
+		                    VALUES_KW, TYPE_KW, indextype_PRESENCE, indextype_SUB);
ba46c7
+			goto done;
ba46c7
+		}
ba46c7
+	} else {
ba46c7
+		goto done;
ba46c7
+	}
ba46c7
+
ba46c7
+	/* if we are here, values contains something - parse it */
ba46c7
+	rc = attr_index_parse_idlistsize_values(attr, idlinfo, values, strval, returntext);
ba46c7
+
ba46c7
+done:
ba46c7
+	slapi_ch_free_string(&mystr);
ba46c7
+	return rc;
ba46c7
+}
ba46c7
+
ba46c7
+static int
ba46c7
+attr_index_idlistsize_config(Slapi_Entry *e, struct attrinfo *ai, char *returntext)
ba46c7
+{
ba46c7
+	int rc = 0;
ba46c7
+	int ii;
ba46c7
+	Slapi_Attr *idlattr;
ba46c7
+	Slapi_Value *sval;
ba46c7
+	struct index_idlistsizeinfo *idlinfo;
ba46c7
+
ba46c7
+	slapi_entry_attr_find(e, NS_INDEX_IDLISTSCANLIMIT, &idlattr);
ba46c7
+	if (!idlattr) {
ba46c7
+		return rc;
ba46c7
+	}
ba46c7
+	for (ii = slapi_attr_first_value(idlattr, &sval); !rc && (ii != -1); ii = slapi_attr_next_value(idlattr, ii, &sval)) {
ba46c7
+		idlinfo = (struct index_idlistsizeinfo *)slapi_ch_calloc(1, sizeof(struct index_idlistsizeinfo));
ba46c7
+		if ((rc = attr_index_parse_idlistsize(ai, slapi_value_get_string(sval), idlinfo, returntext))) {
ba46c7
+			attr_index_idlistsize_free(&idlinfo);
ba46c7
+			attrinfo_delete_idlistinfo(&ai->ai_idlistinfo);
ba46c7
+		} else {
ba46c7
+			if (!ai->ai_idlistinfo) {
ba46c7
+				ai->ai_idlistinfo = dl_new();
ba46c7
+				dl_init(ai->ai_idlistinfo, 1);
ba46c7
+			}
ba46c7
+			dl_add(ai->ai_idlistinfo, idlinfo);
ba46c7
+		}
ba46c7
+	}
ba46c7
+	return rc;
ba46c7
+}
ba46c7
+
ba46c7
 void
ba46c7
 attr_index_config(
ba46c7
     backend *be,
ba46c7
@@ -188,6 +676,7 @@ attr_index_config(
ba46c7
 	Slapi_Value *sval;
ba46c7
 	Slapi_Attr *attr;
ba46c7
 	int mr_count = 0;
ba46c7
+	char myreturntext[SLAPI_DSE_RETURNTEXT_SIZE];
ba46c7
 
ba46c7
 	/* Get the cn */
ba46c7
 	if (0 == slapi_entry_attr_find(e, "cn", &attr)) {
ba46c7
@@ -364,6 +853,11 @@ attr_index_config(
ba46c7
 		}
ba46c7
 	}
ba46c7
 
ba46c7
+	if ((return_value = attr_index_idlistsize_config(e, a, myreturntext))) {
ba46c7
+		LDAPDebug(LDAP_DEBUG_ANY,"attr_index_config: %s: Failed to parse idscanlimit info: %d:%s\n",
ba46c7
+		          fname, return_value, myreturntext);
ba46c7
+	}
ba46c7
+
ba46c7
 	/* initialize the IDL code's private data */
ba46c7
 	return_value = idl_init_private(be, a);
ba46c7
 	if (0 != return_value) {
ba46c7
diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h
ba46c7
index e87c900..3b2e586 100644
ba46c7
--- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h
ba46c7
+++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h
ba46c7
@@ -300,7 +300,7 @@ int id_array_init(Id_Array *new_guy, int size);
ba46c7
 
ba46c7
 IDList* index_read( backend *be, char *type, const char* indextype, const struct berval* val, back_txn *txn, int *err );
ba46c7
 IDList* index_read_ext( backend *be, char *type, const char* indextype, const struct berval* val, back_txn *txn, int *err, int *unindexed );
ba46c7
-IDList* index_read_ext_allids( backend *be, char *type, const char* indextype, const struct berval* val, back_txn *txn, int *err, int *unindexed, int allidslimit );
ba46c7
+IDList* index_read_ext_allids( Slapi_PBlock *pb, backend *be, char *type, const char* indextype, const struct berval* val, back_txn *txn, int *err, int *unindexed, int allidslimit );
ba46c7
 IDList* index_range_read( Slapi_PBlock *pb, backend *be, char *type, const char* indextype, int ftype, struct berval* val, struct berval* nextval, int range, back_txn *txn, int *err );
ba46c7
 IDList* index_range_read_ext( Slapi_PBlock *pb, backend *be, char *type, const char* indextype, int ftype, struct berval* val, struct berval* nextval, int range, back_txn *txn, int *err, int allidslimit );
ba46c7
 const char *encode( const struct berval* data, char buf[BUFSIZ] );
ba46c7
diff --git a/ldap/servers/slapd/proto-slap.h b/ldap/servers/slapd/proto-slap.h
ba46c7
index 37f5f85..4c1dab9 100644
ba46c7
--- a/ldap/servers/slapd/proto-slap.h
ba46c7
+++ b/ldap/servers/slapd/proto-slap.h
ba46c7
@@ -158,6 +158,7 @@ int valuearray_init_bervalarray(struct berval **bvals, Slapi_Value ***cvals);
ba46c7
 int valuearray_init_bervalarray_with_flags(struct berval **bvals, Slapi_Value ***cvals, unsigned long flags);
ba46c7
 int valuearray_get_bervalarray(Slapi_Value **cvals, struct berval ***bvals); /* JCM SLOW FUNCTION */
ba46c7
 void valuearray_free(Slapi_Value ***va);
ba46c7
+void valuearray_free_ext(Slapi_Value ***va, int ii);
ba46c7
 Slapi_Value *valuearray_remove_value(const Slapi_Attr *a, Slapi_Value **va, const Slapi_Value *v);
ba46c7
 void valuearray_remove_value_atindex(Slapi_Value **va, int index);
ba46c7
 int valuearray_isempty( Slapi_Value **va);
ba46c7
-- 
ba46c7
1.7.1
ba46c7