Blame SOURCES/0019-Ticket-50727-correct-mistaken-options-in-filter-vali.patch

8394b4
From 918df0a60a9cf1e3a836165f1044ea63c88bdd72 Mon Sep 17 00:00:00 2001
8394b4
From: William Brown <william@blackhats.net.au>
8394b4
Date: Fri, 6 Dec 2019 14:04:45 +1000
8394b4
Subject: [PATCH] Ticket 50727 - correct mistaken options in filter validation
8394b4
 patch
8394b4
8394b4
Bug Description: Because William of the past missed (forgot) to make
8394b4
some agreed upon changes, we shipped the feature for filter validation
8394b4
in a state that was a bit unclear for users.
8394b4
8394b4
Fix Description: Fix the options to now be clearer in what is
8394b4
expected/demaned from admins. We now have 4 possible states for
8394b4
the value of the config:
8394b4
8394b4
* reject-invalid  (prev on)
8394b4
* process-safe  (prev warn)
8394b4
* warn-invalid (new!)
8394b4
* off (prev off)
8394b4
8394b4
These behave as:
8394b4
8394b4
* reject-invalid - reject queries that contain unknown attributes
8394b4
* process-safe - log a notes=F that an attr is missing, and idl_alloc(0)
8394b4
  the missing attribute for RFC4511 compliance.
8394b4
* warn-invalid - log a notes=F that an attr is missing, and process
8394b4
  as ALLIDS (the legacy behaviour)
8394b4
* off - process as ALLIDs (the legacy behaviour)
8394b4
8394b4
The default is "process-safe".
8394b4
8394b4
https://pagure.io/389-ds-base/issue/50727
8394b4
8394b4
Author: William Brown <william@blackhats.net.au>
8394b4
8394b4
Review by: tbordaz, lkrispen (thanks)
8394b4
---
8394b4
 .../suites/filter/schema_validation_test.py   | 112 ++++++++++++++++--
8394b4
 ldap/servers/slapd/back-ldbm/filterindex.c    |  28 +++--
8394b4
 ldap/servers/slapd/libglobs.c                 |  78 +++++++-----
8394b4
 ldap/servers/slapd/schema.c                   |  26 ++--
8394b4
 ldap/servers/slapd/slap.h                     |  21 ++--
8394b4
 ldap/servers/slapd/slapi-plugin.h             |   1 +
8394b4
 ldap/servers/slapd/slapi-private.h            |   3 +-
8394b4
 src/lib389/lib389/extensibleobject.py         |  53 +++++++++
8394b4
 8 files changed, 258 insertions(+), 64 deletions(-)
8394b4
 create mode 100644 src/lib389/lib389/extensibleobject.py
8394b4
8394b4
diff --git a/dirsrvtests/tests/suites/filter/schema_validation_test.py b/dirsrvtests/tests/suites/filter/schema_validation_test.py
8394b4
index 4ac9fa8ff..d0d67ca95 100644
8394b4
--- a/dirsrvtests/tests/suites/filter/schema_validation_test.py
8394b4
+++ b/dirsrvtests/tests/suites/filter/schema_validation_test.py
8394b4
@@ -9,14 +9,29 @@
8394b4
 
8394b4
 import pytest
8394b4
 import ldap
8394b4
-from lib389.topologies import topology_st
8394b4
+from lib389.topologies import topology_st as topology_st_pre
8394b4
 from lib389.dirsrv_log import DirsrvAccessLog
8394b4
 from lib389._mapped_object import DSLdapObjects
8394b4
 from lib389._constants import DEFAULT_SUFFIX
8394b4
+from lib389.extensibleobject import UnsafeExtensibleObjects
8394b4
 
8394b4
-def _check_value(inst_cfg, value):
8394b4
+def _check_value(inst_cfg, value, exvalue=None):
8394b4
+    if exvalue is None:
8394b4
+        exvalue = value
8394b4
     inst_cfg.set('nsslapd-verify-filter-schema', value)
8394b4
-    assert(inst_cfg.get_attr_val_utf8('nsslapd-verify-filter-schema') == value)
8394b4
+    assert(inst_cfg.get_attr_val_utf8('nsslapd-verify-filter-schema') == exvalue)
8394b4
+
8394b4
+@pytest.fixture(scope="module")
8394b4
+def topology_st(topology_st_pre):
8394b4
+    raw_objects = UnsafeExtensibleObjects(topology_st_pre.standalone, basedn=DEFAULT_SUFFIX)
8394b4
+    # Add an object that won't be able to be queried due to invalid attrs.
8394b4
+    raw_objects.create(properties = {
8394b4
+        "cn": "test_obj",
8394b4
+        "a": "a",
8394b4
+        "b": "b",
8394b4
+        "uid": "foo"
8394b4
+    })
8394b4
+    return topology_st_pre
8394b4
 
8394b4
 
8394b4
 @pytest.mark.ds50349
8394b4
@@ -51,8 +66,14 @@ def test_filter_validation_config(topology_st):
8394b4
 
8394b4
     initial_value = inst_cfg.get_attr_val_utf8('nsslapd-verify-filter-schema')
8394b4
 
8394b4
-    _check_value(inst_cfg, "on")
8394b4
-    _check_value(inst_cfg, "warn")
8394b4
+    # Check legacy values that may have been set
8394b4
+    _check_value(inst_cfg, "on", "reject-invalid")
8394b4
+    _check_value(inst_cfg, "warn", "process-safe")
8394b4
+    _check_value(inst_cfg, "off")
8394b4
+    # Check the more descriptive values
8394b4
+    _check_value(inst_cfg, "reject-invalid")
8394b4
+    _check_value(inst_cfg, "process-safe")
8394b4
+    _check_value(inst_cfg, "warn-invalid")
8394b4
     _check_value(inst_cfg, "off")
8394b4
 
8394b4
     # This should fail
8394b4
@@ -85,7 +106,7 @@ def test_filter_validation_enabled(topology_st):
8394b4
     inst = topology_st.standalone
8394b4
 
8394b4
     # In case the default has changed, we set the value to warn.
8394b4
-    inst.config.set("nsslapd-verify-filter-schema", "on")
8394b4
+    inst.config.set("nsslapd-verify-filter-schema", "reject-invalid")
8394b4
     raw_objects = DSLdapObjects(inst, basedn=DEFAULT_SUFFIX)
8394b4
 
8394b4
     # Check a good query has no errors.
8394b4
@@ -104,9 +125,9 @@ def test_filter_validation_enabled(topology_st):
8394b4
 
8394b4
 
8394b4
 @pytest.mark.ds50349
8394b4
-def test_filter_validation_warning(topology_st):
8394b4
+def test_filter_validation_warn_safe(topology_st):
8394b4
     """Test that queries which are invalid, are correctly marked as "notes=F" in
8394b4
-    the access log.
8394b4
+    the access log, and return no entries or partial sets.
8394b4
 
8394b4
     :id: 8b2b23fe-d878-435c-bc84-8c298be4ca1f
8394b4
     :setup: Standalone instance
8394b4
@@ -122,7 +143,7 @@ def test_filter_validation_warning(topology_st):
8394b4
     inst = topology_st.standalone
8394b4
 
8394b4
     # In case the default has changed, we set the value to warn.
8394b4
-    inst.config.set("nsslapd-verify-filter-schema", "warn")
8394b4
+    inst.config.set("nsslapd-verify-filter-schema", "process-safe")
8394b4
     # Set the access log to un-buffered so we get it immediately.
8394b4
     inst.config.set("nsslapd-accesslog-logbuffering", "off")
8394b4
 
8394b4
@@ -139,20 +160,93 @@ def test_filter_validation_warning(topology_st):
8394b4
 
8394b4
     # Check a good query has no warnings.
8394b4
     r = raw_objects.filter("(objectClass=*)")
8394b4
+    assert(len(r) > 0)
8394b4
     r_s1 = access_log.match(".*notes=F.*")
8394b4
     # Should be the same number of log lines IE 0.
8394b4
     assert(len(r_init) == len(r_s1))
8394b4
 
8394b4
     # Check a bad one DOES emit a warning.
8394b4
     r = raw_objects.filter("(a=a)")
8394b4
+    assert(len(r) == 0)
8394b4
     r_s2 = access_log.match(".*notes=F.*")
8394b4
     # Should be the greate number of log lines IE +1
8394b4
     assert(len(r_init) + 1 == len(r_s2))
8394b4
 
8394b4
     # Check a bad complex one does emit a warning.
8394b4
     r = raw_objects.filter("(&(a=a)(b=b)(objectClass=*))")
8394b4
+    assert(len(r) == 0)
8394b4
     r_s3 = access_log.match(".*notes=F.*")
8394b4
     # Should be the greate number of log lines IE +2
8394b4
     assert(len(r_init) + 2 == len(r_s3))
8394b4
 
8394b4
+    # Check that we can still get things when partial
8394b4
+    r = raw_objects.filter("(|(a=a)(b=b)(uid=foo))")
8394b4
+    assert(len(r) == 1)
8394b4
+    r_s4 = access_log.match(".*notes=F.*")
8394b4
+    # Should be the greate number of log lines IE +2
8394b4
+    assert(len(r_init) + 3 == len(r_s4))
8394b4
+
8394b4
+
8394b4
+@pytest.mark.ds50349
8394b4
+def test_filter_validation_warn_unsafe(topology_st):
8394b4
+    """Test that queries which are invalid, are correctly marked as "notes=F" in
8394b4
+    the access log, and uses the legacy query behaviour to return unsafe sets.
8394b4
+
8394b4
+    :id: 8b2b23fe-d878-435c-bc84-8c298be4ca1f
8394b4
+    :setup: Standalone instance
8394b4
+    :steps:
8394b4
+        1. Search a well formed query
8394b4
+        2. Search a poorly formed query
8394b4
+        3. Search a poorly formed complex (and/or) query
8394b4
+    :expectedresults:
8394b4
+        1. No warnings
8394b4
+        2. notes=F is present
8394b4
+        3. notes=F is present
8394b4
+    """
8394b4
+    inst = topology_st.standalone
8394b4
+
8394b4
+    # In case the default has changed, we set the value to warn.
8394b4
+    inst.config.set("nsslapd-verify-filter-schema", "warn-invalid")
8394b4
+    # Set the access log to un-buffered so we get it immediately.
8394b4
+    inst.config.set("nsslapd-accesslog-logbuffering", "off")
8394b4
+
8394b4
+    # Setup the query object.
8394b4
+    # Now we don't care if there are any results, we only care about good/bad queries.
8394b4
+    # To do this we have to bypass some of the lib389 magic, and just emit raw queries
8394b4
+    # to check them. Turns out lib389 is well designed and this just works as expected
8394b4
+    # if you use a single DSLdapObjects and filter. :)
8394b4
+    raw_objects = DSLdapObjects(inst, basedn=DEFAULT_SUFFIX)
8394b4
+
8394b4
+    # Find any initial notes=F
8394b4
+    access_log = DirsrvAccessLog(inst)
8394b4
+    r_init = access_log.match(".*notes=(U,)?F.*")
8394b4
+
8394b4
+    # Check a good query has no warnings.
8394b4
+    r = raw_objects.filter("(objectClass=*)")
8394b4
+    assert(len(r) > 0)
8394b4
+    r_s1 = access_log.match(".*notes=(U,)?F.*")
8394b4
+    # Should be the same number of log lines IE 0.
8394b4
+    assert(len(r_init) == len(r_s1))
8394b4
+
8394b4
+    # Check a bad one DOES emit a warning.
8394b4
+    r = raw_objects.filter("(a=a)")
8394b4
+    assert(len(r) == 1)
8394b4
+    # NOTE: Unlike warn-process-safely, these become UNINDEXED and show in the logs.
8394b4
+    r_s2 = access_log.match(".*notes=(U,)?F.*")
8394b4
+    # Should be the greate number of log lines IE +1
8394b4
+    assert(len(r_init) + 1 == len(r_s2))
8394b4
+
8394b4
+    # Check a bad complex one does emit a warning.
8394b4
+    r = raw_objects.filter("(&(a=a)(b=b)(objectClass=*))")
8394b4
+    assert(len(r) == 1)
8394b4
+    r_s3 = access_log.match(".*notes=(U,)?F.*")
8394b4
+    # Should be the greate number of log lines IE +2
8394b4
+    assert(len(r_init) + 2 == len(r_s3))
8394b4
+
8394b4
+    # Check that we can still get things when partial
8394b4
+    r = raw_objects.filter("(|(a=a)(b=b)(uid=foo))")
8394b4
+    assert(len(r) == 1)
8394b4
+    r_s4 = access_log.match(".*notes=(U,)?F.*")
8394b4
+    # Should be the greate number of log lines IE +2
8394b4
+    assert(len(r_init) + 3 == len(r_s4))
8394b4
 
8394b4
diff --git a/ldap/servers/slapd/back-ldbm/filterindex.c b/ldap/servers/slapd/back-ldbm/filterindex.c
8394b4
index 7e65f73ca..8a79848c3 100644
8394b4
--- a/ldap/servers/slapd/back-ldbm/filterindex.c
8394b4
+++ b/ldap/servers/slapd/back-ldbm/filterindex.c
8394b4
@@ -223,13 +223,15 @@ ava_candidates(
8394b4
 
8394b4
     switch (ftype) {
8394b4
     case LDAP_FILTER_GE:
8394b4
-        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
             /*
8394b4
              * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
              * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
              * be set on the filter at all!
8394b4
              */
8394b4
             slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+        }
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
             idl = idl_alloc(0);
8394b4
         } else {
8394b4
             idl = range_candidates(pb, be, type, bval, NULL, err, &sattr, allidslimit);
8394b4
@@ -239,13 +241,15 @@ ava_candidates(
8394b4
         goto done;
8394b4
         break;
8394b4
     case LDAP_FILTER_LE:
8394b4
-        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
             /*
8394b4
              * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
              * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
              * be set on the filter at all!
8394b4
              */
8394b4
             slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+        }
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
             idl = idl_alloc(0);
8394b4
         } else {
8394b4
             idl = range_candidates(pb, be, type, NULL, bval, err, &sattr, allidslimit);
8394b4
@@ -293,13 +297,15 @@ ava_candidates(
8394b4
         ptr[1] = NULL;
8394b4
         ivals = ptr;
8394b4
 
8394b4
-        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
             /*
8394b4
              * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
              * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
              * be set on the filter at all!
8394b4
              */
8394b4
             slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+        }
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
             idl = idl_alloc(0);
8394b4
         } else {
8394b4
             slapi_attr_assertion2keys_ava_sv(&sattr, &tmp, (Slapi_Value ***)&ivals, LDAP_FILTER_EQUALITY_FAST);
8394b4
@@ -326,13 +332,15 @@ ava_candidates(
8394b4
             slapi_ch_free((void **)&ivals);
8394b4
         }
8394b4
     } else {
8394b4
-        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
             /*
8394b4
              * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
              * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
              * be set on the filter at all!
8394b4
              */
8394b4
             slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+        }
8394b4
+        if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
             idl = idl_alloc(0);
8394b4
         } else {
8394b4
             slapi_value_init_berval(&sv, bval);
8394b4
@@ -382,13 +390,15 @@ presence_candidates(
8394b4
     }
8394b4
     slapi_pblock_get(pb, SLAPI_TXN, &txn.back_txn_txn);
8394b4
 
8394b4
-    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
         /*
8394b4
          * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
          * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
          * be set on the filter at all!
8394b4
          */
8394b4
         slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+    }
8394b4
+    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
         idl = idl_alloc(0);
8394b4
     } else {
8394b4
         idl = index_read_ext_allids(pb, be, type, indextype_PRESENCE,
8394b4
@@ -485,13 +495,15 @@ extensible_candidates(
8394b4
                         slapi_pblock_get(pb, SLAPI_PLUGIN_MR_KEYS, &keys)) {
8394b4
                         /* something went wrong.  bail. */
8394b4
                         break;
8394b4
-                    } else if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+                    } else if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
                         /*
8394b4
                          * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
                          * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
                          * be set on the filter at all!
8394b4
                          */
8394b4
                         slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+                    }
8394b4
+                    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
                         idl = idl_alloc(0);
8394b4
                     } else if (keys == NULL || keys[0] == NULL) {
8394b4
                         /* no keys */
8394b4
@@ -986,13 +998,15 @@ substring_candidates(
8394b4
      * look up each key in the index, ANDing the resulting
8394b4
      * IDLists together.
8394b4
      */
8394b4
-    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR) {
8394b4
+    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_WARN) {
8394b4
         /*
8394b4
          * REMEMBER: this flag is only set on WARN levels. If the filter verify
8394b4
          * is on strict, we reject in search.c, if we ar off, the flag will NOT
8394b4
          * be set on the filter at all!
8394b4
          */
8394b4
         slapi_pblock_set_flag_operation_notes(pb, SLAPI_OP_NOTE_FILTER_INVALID);
8394b4
+    }
8394b4
+    if (f->f_flags & SLAPI_FILTER_INVALID_ATTR_UNDEFINE) {
8394b4
         idl = idl_alloc(0);
8394b4
     } else {
8394b4
         slapi_pblock_get(pb, SLAPI_TXN, &txn.back_txn_txn);
8394b4
diff --git a/ldap/servers/slapd/libglobs.c b/ldap/servers/slapd/libglobs.c
8394b4
index b9cdb6b37..66170ebc6 100644
8394b4
--- a/ldap/servers/slapd/libglobs.c
8394b4
+++ b/ldap/servers/slapd/libglobs.c
8394b4
@@ -166,7 +166,7 @@ typedef enum {
8394b4
     CONFIG_SPECIAL_VALIDATE_CERT_SWITCH, /* maps strings to an enumeration */
8394b4
     CONFIG_SPECIAL_UNHASHED_PW_SWITCH,   /* unhashed pw: on/off/nolog */
8394b4
     CONFIG_SPECIAL_TLS_CHECK_CRL,        /* maps enum tls_check_crl_t to char * */
8394b4
-    CONFIG_ON_OFF_WARN,                  /* maps to a config on/warn/off enum */
8394b4
+    CONFIG_SPECIAL_FILTER_VERIFY,      /* maps to a config strict/warn-strict/warn/off enum */
8394b4
 } ConfigVarType;
8394b4
 
8394b4
 static int32_t config_set_onoff(const char *attrname, char *value, int32_t *configvalue, char *errorbuf, int apply);
8394b4
@@ -256,7 +256,7 @@ slapi_int_t init_malloc_mmap_threshold;
8394b4
 slapi_onoff_t init_extract_pem;
8394b4
 slapi_onoff_t init_ignore_vattrs;
8394b4
 slapi_onoff_t init_enable_upgrade_hash;
8394b4
-slapi_onwarnoff_t init_verify_filter_schema;
8394b4
+slapi_special_filter_verify_t init_verify_filter_schema;
8394b4
 
8394b4
 static int
8394b4
 isInt(ConfigVarType type)
8394b4
@@ -1248,7 +1248,7 @@ static struct config_get_and_set
8394b4
     {CONFIG_VERIFY_FILTER_SCHEMA, config_set_verify_filter_schema,
8394b4
      NULL, 0,
8394b4
      (void **)&global_slapdFrontendConfig.verify_filter_schema,
8394b4
-     CONFIG_ON_OFF_WARN, (ConfigGetFunc)config_get_verify_filter_schema,
8394b4
+     CONFIG_SPECIAL_FILTER_VERIFY, (ConfigGetFunc)config_get_verify_filter_schema,
8394b4
      &init_verify_filter_schema},
8394b4
     /* End config */
8394b4
     };
8394b4
@@ -7659,18 +7659,21 @@ config_initvalue_to_onoff(struct config_get_and_set *cgas, char *initvalbuf, siz
8394b4
 }
8394b4
 
8394b4
 static char *
8394b4
-config_initvalue_to_onwarnoff(struct config_get_and_set *cgas, char *initvalbuf, size_t initvalbufsize) {
8394b4
+config_initvalue_to_special_filter_verify(struct config_get_and_set *cgas, char *initvalbuf, size_t initvalbufsize) {
8394b4
     char *retval = NULL;
8394b4
-    if (cgas->config_var_type == CONFIG_ON_OFF_WARN) {
8394b4
-        slapi_onwarnoff_t *value = (slapi_onwarnoff_t *)(intptr_t)cgas->initvalue;
8394b4
+    if (cgas->config_var_type == CONFIG_SPECIAL_FILTER_VERIFY) {
8394b4
+        slapi_special_filter_verify_t *value = (slapi_special_filter_verify_t *)(intptr_t)cgas->initvalue;
8394b4
         if (value != NULL) {
8394b4
-            if (*value == SLAPI_ON) {
8394b4
-                PR_snprintf(initvalbuf, initvalbufsize, "%s", "on");
8394b4
+            if (*value == SLAPI_STRICT) {
8394b4
+                PR_snprintf(initvalbuf, initvalbufsize, "%s", "reject-invalid");
8394b4
                 retval = initvalbuf;
8394b4
-            } else if (*value == SLAPI_WARN) {
8394b4
-                PR_snprintf(initvalbuf, initvalbufsize, "%s", "warn");
8394b4
+            } else if (*value == SLAPI_WARN_SAFE) {
8394b4
+                PR_snprintf(initvalbuf, initvalbufsize, "%s", "process-safe");
8394b4
                 retval = initvalbuf;
8394b4
-            } else if (*value == SLAPI_OFF) {
8394b4
+            } else if (*value == SLAPI_WARN_UNSAFE) {
8394b4
+                PR_snprintf(initvalbuf, initvalbufsize, "%s", "warn-invalid");
8394b4
+                retval = initvalbuf;
8394b4
+            } else if (*value == SLAPI_OFF_UNSAFE) {
8394b4
                 PR_snprintf(initvalbuf, initvalbufsize, "%s", "off");
8394b4
                 retval = initvalbuf;
8394b4
             }
8394b4
@@ -7680,7 +7683,7 @@ config_initvalue_to_onwarnoff(struct config_get_and_set *cgas, char *initvalbuf,
8394b4
 }
8394b4
 
8394b4
 static int32_t
8394b4
-config_set_onoffwarn(slapdFrontendConfig_t *slapdFrontendConfig, slapi_onwarnoff_t *target, const char *attrname, char *value, char *errorbuf, int apply) {
8394b4
+config_set_specialfilterverify(slapdFrontendConfig_t *slapdFrontendConfig, slapi_special_filter_verify_t *target, const char *attrname, char *value, char *errorbuf, int apply) {
8394b4
     if (target == NULL) {
8394b4
         return LDAP_OPERATIONS_ERROR;
8394b4
     }
8394b4
@@ -7691,15 +7694,23 @@ config_set_onoffwarn(slapdFrontendConfig_t *slapdFrontendConfig, slapi_onwarnoff
8394b4
 
8394b4
     slapi_special_filter_verify_t p_val = SLAPI_WARN_UNSAFE;
8394b4
 
8394b4
+    /* on/warn/off retained for legacy reasons due to wbrown making terrible mistakes :( :( */
8394b4
     if (strcasecmp(value, "on") == 0) {
8394b4
-        p_val = SLAPI_ON;
8394b4
+        p_val = SLAPI_STRICT;
8394b4
     } else if (strcasecmp(value, "warn") == 0) {
8394b4
-        p_val = SLAPI_WARN;
8394b4
+        p_val = SLAPI_WARN_SAFE;
8394b4
+    /* The new fixed/descriptive names */
8394b4
+    } else if (strcasecmp(value, "reject-invalid") == 0) {
8394b4
+        p_val = SLAPI_STRICT;
8394b4
+    } else if (strcasecmp(value, "process-safe") == 0) {
8394b4
+        p_val = SLAPI_WARN_SAFE;
8394b4
+    } else if (strcasecmp(value, "warn-invalid") == 0) {
8394b4
+        p_val = SLAPI_WARN_UNSAFE;
8394b4
     } else if (strcasecmp(value, "off") == 0) {
8394b4
-        p_val = SLAPI_OFF;
8394b4
+        p_val = SLAPI_OFF_UNSAFE;
8394b4
     } else {
8394b4
         slapi_create_errormsg(errorbuf, SLAPI_DSE_RETURNTEXT_SIZE,
8394b4
-                              "%s: invalid value \"%s\". Valid values are \"on\", \"warn\" or \"off\".", attrname, value);
8394b4
+                              "%s: invalid value \"%s\". Valid values are \"reject-invalid\", \"process-safe\", \"warn-invalid\" or \"off\". If in doubt, choose \"process-safe\"", attrname, value);
8394b4
         return LDAP_OPERATIONS_ERROR;
8394b4
     }
8394b4
 
8394b4
@@ -7718,14 +7729,14 @@ config_set_onoffwarn(slapdFrontendConfig_t *slapdFrontendConfig, slapi_onwarnoff
8394b4
 int32_t
8394b4
 config_set_verify_filter_schema(const char *attrname, char *value, char *errorbuf, int apply) {
8394b4
     slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
8394b4
-    slapi_onwarnoff_t *target = &(slapdFrontendConfig->verify_filter_schema);
8394b4
-    return config_set_onoffwarn(slapdFrontendConfig, target, attrname, value, errorbuf, apply);
8394b4
+    slapi_special_filter_verify_t *target = &(slapdFrontendConfig->verify_filter_schema);
8394b4
+    return config_set_specialfilterverify(slapdFrontendConfig, target, attrname, value, errorbuf, apply);
8394b4
 }
8394b4
 
8394b4
 Slapi_Filter_Policy
8394b4
 config_get_verify_filter_schema()
8394b4
 {
8394b4
-    slapi_onwarnoff_t retVal;
8394b4
+    slapi_special_filter_verify_t retVal;
8394b4
     slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
8394b4
     CFG_LOCK_READ(slapdFrontendConfig);
8394b4
     retVal = slapdFrontendConfig->verify_filter_schema;
8394b4
@@ -7733,10 +7744,13 @@ config_get_verify_filter_schema()
8394b4
 
8394b4
     /* Now map this to a policy that the fns understand. */
8394b4
     switch (retVal) {
8394b4
-    case SLAPI_ON:
8394b4
+    case SLAPI_STRICT:
8394b4
         return FILTER_POLICY_STRICT;
8394b4
         break;
8394b4
-    case SLAPI_WARN:
8394b4
+    case SLAPI_WARN_SAFE:
8394b4
+        return FILTER_POLICY_PROTECT;
8394b4
+        break;
8394b4
+    case SLAPI_WARN_UNSAFE:
8394b4
         return FILTER_POLICY_WARNING;
8394b4
         break;
8394b4
     default:
8394b4
@@ -7794,8 +7808,8 @@ config_set(const char *attr, struct berval **values, char *errorbuf, int apply)
8394b4
             void *initval = cgas->initvalue;
8394b4
             if (cgas->config_var_type == CONFIG_ON_OFF) {
8394b4
                 initval = (void *)config_initvalue_to_onoff(cgas, initvalbuf, sizeof(initvalbuf));
8394b4
-            } else if (cgas->config_var_type == CONFIG_ON_OFF_WARN) {
8394b4
-                initval = (void *)config_initvalue_to_onwarnoff(cgas, initvalbuf, sizeof(initvalbuf));
8394b4
+            } else if (cgas->config_var_type == CONFIG_SPECIAL_FILTER_VERIFY) {
8394b4
+                initval = (void *)config_initvalue_to_special_filter_verify(cgas, initvalbuf, sizeof(initvalbuf));
8394b4
             }
8394b4
             if (cgas->setfunc) {
8394b4
                 retval = (cgas->setfunc)(cgas->attr_name, initval, errorbuf, apply);
8394b4
@@ -8021,20 +8035,24 @@ config_set_value(
8394b4
 
8394b4
         break;
8394b4
 
8394b4
-    case CONFIG_ON_OFF_WARN:
8394b4
+    case CONFIG_SPECIAL_FILTER_VERIFY:
8394b4
         /* Is this the right default here? */
8394b4
         if (!value) {
8394b4
-            slapi_entry_attr_set_charptr(e, cgas->attr_name, "off");
8394b4
+            slapi_entry_attr_set_charptr(e, cgas->attr_name, "process-safe");
8394b4
             break;
8394b4
         }
8394b4
 
8394b4
-        if (*((slapi_onwarnoff_t *)value) == SLAPI_ON) {
8394b4
-            slapi_entry_attr_set_charptr(e, cgas->attr_name, "on");
8394b4
-        } else if (*((slapi_onwarnoff_t *)value) == SLAPI_WARN) {
8394b4
-            slapi_entry_attr_set_charptr(e, cgas->attr_name, "warn");
8394b4
+        if (*((slapi_special_filter_verify_t *)value) == SLAPI_STRICT) {
8394b4
+            slapi_entry_attr_set_charptr(e, cgas->attr_name, "reject-invalid");
8394b4
+        } else if (*((slapi_special_filter_verify_t *)value) == SLAPI_WARN_SAFE) {
8394b4
+            slapi_entry_attr_set_charptr(e, cgas->attr_name, "process-safe");
8394b4
+        } else if (*((slapi_special_filter_verify_t *)value) == SLAPI_WARN_UNSAFE) {
8394b4
+            slapi_entry_attr_set_charptr(e, cgas->attr_name, "warn-invalid");
8394b4
+        } else if (*((slapi_special_filter_verify_t *)value) == SLAPI_OFF_UNSAFE) {
8394b4
+            slapi_entry_attr_set_charptr(e, cgas->attr_name, "off");
8394b4
         } else {
8394b4
             /* Default to safe warn-proccess-safely */
8394b4
-            slapi_entry_attr_set_charptr(e, cgas->attr_name, "warn-invalid");
8394b4
+            slapi_entry_attr_set_charptr(e, cgas->attr_name, "process-safe");
8394b4
         }
8394b4
 
8394b4
         break;
8394b4
diff --git a/ldap/servers/slapd/schema.c b/ldap/servers/slapd/schema.c
8394b4
index 6e853fc7c..d7c3c139e 100644
8394b4
--- a/ldap/servers/slapd/schema.c
8394b4
+++ b/ldap/servers/slapd/schema.c
8394b4
@@ -698,7 +698,7 @@ out:
8394b4
 }
8394b4
 
8394b4
 static Slapi_Filter_Result
8394b4
-slapi_filter_schema_check_inner(Slapi_Filter *f) {
8394b4
+slapi_filter_schema_check_inner(Slapi_Filter *f, slapi_filter_flags flags) {
8394b4
     /*
8394b4
      * Default response to Ok. If any more severe things happen we
8394b4
      * alter this to reflect it. IE we bubble up more severe errors
8394b4
@@ -712,26 +712,26 @@ slapi_filter_schema_check_inner(Slapi_Filter *f) {
8394b4
     case LDAP_FILTER_LE:
8394b4
     case LDAP_FILTER_APPROX:
8394b4
         if (!attr_syntax_exist_by_name_nolock(f->f_avtype)) {
8394b4
-            f->f_flags |= SLAPI_FILTER_INVALID_ATTR;
8394b4
+            f->f_flags |= flags;
8394b4
             r = FILTER_SCHEMA_WARNING;
8394b4
         }
8394b4
         break;
8394b4
     case LDAP_FILTER_PRESENT:
8394b4
         if (!attr_syntax_exist_by_name_nolock(f->f_type)) {
8394b4
-            f->f_flags |= SLAPI_FILTER_INVALID_ATTR;
8394b4
+            f->f_flags |= flags;
8394b4
             r = FILTER_SCHEMA_WARNING;
8394b4
         }
8394b4
         break;
8394b4
     case LDAP_FILTER_SUBSTRINGS:
8394b4
         if (!attr_syntax_exist_by_name_nolock(f->f_sub_type)) {
8394b4
-            f->f_flags |= SLAPI_FILTER_INVALID_ATTR;
8394b4
+            f->f_flags |= flags;
8394b4
             r = FILTER_SCHEMA_WARNING;
8394b4
         }
8394b4
         break;
8394b4
     case LDAP_FILTER_EXTENDED:
8394b4
         /* I don't have any examples of this, so I'm not 100% on how to check it */
8394b4
         if (!attr_syntax_exist_by_name_nolock(f->f_mr_type)) {
8394b4
-            f->f_flags |= SLAPI_FILTER_INVALID_ATTR;
8394b4
+            f->f_flags |= flags;
8394b4
             r = FILTER_SCHEMA_WARNING;
8394b4
         }
8394b4
         break;
8394b4
@@ -740,7 +740,7 @@ slapi_filter_schema_check_inner(Slapi_Filter *f) {
8394b4
     case LDAP_FILTER_NOT:
8394b4
         /* Recurse and check all elemments of the filter */
8394b4
         for (Slapi_Filter *f_child = f->f_list; f_child != NULL; f_child = f_child->f_next) {
8394b4
-            Slapi_Filter_Result ri = slapi_filter_schema_check_inner(f_child);
8394b4
+            Slapi_Filter_Result ri = slapi_filter_schema_check_inner(f_child, flags);
8394b4
             if (ri > r) {
8394b4
                 r = ri;
8394b4
             }
8394b4
@@ -769,12 +769,24 @@ slapi_filter_schema_check(Slapi_Filter *f, Slapi_Filter_Policy fp) {
8394b4
         return FILTER_SCHEMA_SUCCESS;
8394b4
     }
8394b4
 
8394b4
+    /*
8394b4
+     * There are two possible warning types - it's not up to us to warn into
8394b4
+     * the logs, that's the backends job. So we have to flag a hint into the
8394b4
+     * filter about what it should do. This is why there are two FILTER_INVALID
8394b4
+     * types in filter_flags, one for logging it, and one for actually doing
8394b4
+     * the rejection.
8394b4
+     */
8394b4
+    slapi_filter_flags flags = SLAPI_FILTER_INVALID_ATTR_WARN;
8394b4
+    if (fp == FILTER_POLICY_PROTECT) {
8394b4
+        flags |= SLAPI_FILTER_INVALID_ATTR_UNDEFINE;
8394b4
+    }
8394b4
+
8394b4
     /*
8394b4
      * Filters are nested, recursive structures, so we actually have to call an inner
8394b4
      * function until we have a result!
8394b4
      */
8394b4
     attr_syntax_read_lock();
8394b4
-    Slapi_Filter_Result r = slapi_filter_schema_check_inner(f);
8394b4
+    Slapi_Filter_Result r = slapi_filter_schema_check_inner(f, flags);
8394b4
     attr_syntax_unlock_read();
8394b4
 
8394b4
     /* If any warning occured, ensure we fail it. */
8394b4
diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h
8394b4
index 8a2748519..d73e9aaae 100644
8394b4
--- a/ldap/servers/slapd/slap.h
8394b4
+++ b/ldap/servers/slapd/slap.h
8394b4
@@ -457,12 +457,12 @@ typedef enum _tls_check_crl_t {
8394b4
     TLS_CHECK_ALL = 2,
8394b4
 } tls_check_crl_t;
8394b4
 
8394b4
-typedef enum _slapi_onwarnoff_t {
8394b4
-    SLAPI_OFF = 0,
8394b4
-    SLAPI_WARN = 1,
8394b4
-    SLAPI_ON = 2,
8394b4
-} slapi_onwarnoff_t;
8394b4
-
8394b4
+typedef enum _slapi_special_filter_verify_t {
8394b4
+    SLAPI_STRICT = 0,
8394b4
+    SLAPI_WARN_SAFE = 1,
8394b4
+    SLAPI_WARN_UNSAFE = 2,
8394b4
+    SLAPI_OFF_UNSAFE = 3,
8394b4
+} slapi_special_filter_verify_t;
8394b4
 
8394b4
 struct subfilt
8394b4
 {
8394b4
@@ -2547,11 +2547,12 @@ typedef struct _slapdFrontendConfig
8394b4
     slapi_onoff_t enable_upgrade_hash; /* If on, upgrade hashes for PW at bind */
8394b4
     /*
8394b4
      * Do we verify the filters we recieve by schema?
8394b4
-     * on - yes, and reject if attribute not found
8394b4
-     * warn - yes, and warn that the attribute is unknown and unindexed
8394b4
-     * off - no, do whatever (old status-quo)
8394b4
+     * reject-invalid - reject filter if there is anything invalid
8394b4
+     * process-safe - allow the filter, warn about what's invalid, and then idl_alloc(0) with rfc compliance
8394b4
+     * warn-invalid - allow the filter, warn about the invalid, and then do a ALLIDS (may lead to full table scan)
8394b4
+     * off - don't warn, just allow anything. This is the legacy behaviour.
8394b4
      */
8394b4
-    slapi_onwarnoff_t verify_filter_schema;
8394b4
+    slapi_special_filter_verify_t verify_filter_schema;
8394b4
 } slapdFrontendConfig_t;
8394b4
 
8394b4
 /* possible values for slapdFrontendConfig_t.schemareplace */
8394b4
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
8394b4
index 50b8d12c8..40b5c911a 100644
8394b4
--- a/ldap/servers/slapd/slapi-plugin.h
8394b4
+++ b/ldap/servers/slapd/slapi-plugin.h
8394b4
@@ -1571,6 +1571,7 @@ int slapi_entry_syntax_check(Slapi_PBlock *pb, Slapi_Entry *e, int override);
8394b4
 typedef enum {
8394b4
     FILTER_POLICY_OFF,
8394b4
     FILTER_POLICY_WARNING,
8394b4
+    FILTER_POLICY_PROTECT,
8394b4
     FILTER_POLICY_STRICT,
8394b4
 } Slapi_Filter_Policy;
8394b4
 
8394b4
diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h
8394b4
index 04c045532..1f17eda12 100644
8394b4
--- a/ldap/servers/slapd/slapi-private.h
8394b4
+++ b/ldap/servers/slapd/slapi-private.h
8394b4
@@ -54,7 +54,8 @@ typedef enum _slapi_filter_flags_t {
8394b4
     SLAPI_FILTER_RUV = 4,
8394b4
     SLAPI_FILTER_NORMALIZED_TYPE = 8,
8394b4
     SLAPI_FILTER_NORMALIZED_VALUE = 16,
8394b4
-    SLAPI_FILTER_INVALID_ATTR = 32,
8394b4
+    SLAPI_FILTER_INVALID_ATTR_UNDEFINE = 32,
8394b4
+    SLAPI_FILTER_INVALID_ATTR_WARN = 64,
8394b4
 } slapi_filter_flags;
8394b4
 
8394b4
 #define SLAPI_ENTRY_LDAPSUBENTRY 2
8394b4
diff --git a/src/lib389/lib389/extensibleobject.py b/src/lib389/lib389/extensibleobject.py
8394b4
new file mode 100644
8394b4
index 000000000..8fe37f980
8394b4
--- /dev/null
8394b4
+++ b/src/lib389/lib389/extensibleobject.py
8394b4
@@ -0,0 +1,53 @@
8394b4
+# --- BEGIN COPYRIGHT BLOCK ---
8394b4
+# Copyright (C) 2019, William Brown <william at="" blackhats.net.au="">
8394b4
+# All rights reserved.
8394b4
+#
8394b4
+# License: GPL (version 3 or any later version).
8394b4
+# See LICENSE for details.
8394b4
+# --- END COPYRIGHT BLOCK ---
8394b4
+
8394b4
+from lib389._mapped_object import DSLdapObject, DSLdapObjects
8394b4
+from lib389.utils import ensure_str
8394b4
+
8394b4
+class UnsafeExtensibleObject(DSLdapObject):
8394b4
+    """A single instance of an extensible object. Extensible object by it's
8394b4
+    nature is unsafe, eliminating rules around attribute checking. It may
8394b4
+    cause unsafe or other unknown behaviour if not handled correctly.
8394b4
+
8394b4
+    :param instance: An instance
8394b4
+    :type instance: lib389.DirSrv
8394b4
+    :param dn: Entry DN
8394b4
+    :type dn: str
8394b4
+    """
8394b4
+
8394b4
+    def __init__(self, instance, dn=None):
8394b4
+        super(UnsafeExtensibleObject, self).__init__(instance, dn)
8394b4
+        self._rdn_attribute = "cn"
8394b4
+        # Can I generate these from schema?
8394b4
+        self._must_attributes = []
8394b4
+        self._create_objectclasses = [
8394b4
+            'top',
8394b4
+            'extensibleObject',
8394b4
+        ]
8394b4
+        self._protected = False
8394b4
+
8394b4
+class UnsafeExtensibleObjects(DSLdapObjects):
8394b4
+    """DSLdapObjects that represents all extensible objects. Extensible Objects
8394b4
+    are unsafe in their nature, disabling many checks around schema and attribute
8394b4
+    handling. You should really really REALLY not use this unless you have specific
8394b4
+    needs for testing.
8394b4
+
8394b4
+    :param instance: An instance
8394b4
+    :type instance: lib389.DirSrv
8394b4
+    :param basedn: Base DN for all group entries below
8394b4
+    :type basedn: str
8394b4
+    """
8394b4
+
8394b4
+    def __init__(self, instance, basedn):
8394b4
+        super(UnsafeExtensibleObjects, self).__init__(instance)
8394b4
+        self._objectclasses = [
8394b4
+            'extensibleObject',
8394b4
+        ]
8394b4
+        self._filterattrs = ["cn"]
8394b4
+        self._childobject = UnsafeExtensibleObject
8394b4
+        self._basedn = ensure_str(basedn)
8394b4
-- 
8394b4
2.21.1
8394b4