|
 |
8394b4 |
From 74525da09050809a55e2c6bba4c42b27d5326a8f Mon Sep 17 00:00:00 2001
|
|
|
8394b4 |
From: Mark Reynolds <mreynolds@redhat.com>
|
|
|
8394b4 |
Date: Sun, 12 Jan 2020 20:11:07 -0500
|
|
|
8394b4 |
Subject: [PATCH] Issue 50599 - Remove db region files prior to db recovery
|
|
|
8394b4 |
|
|
|
8394b4 |
Bug Description: If the server crashes then the region files can become
|
|
|
8394b4 |
corrupted and this prevents the server from starting.
|
|
|
8394b4 |
|
|
|
8394b4 |
Fix Description: If we encounter a disorderly shutdown, then remove
|
|
|
8394b4 |
the region files so there is a clean slate to start
|
|
|
8394b4 |
with.
|
|
|
8394b4 |
|
|
|
8394b4 |
Also cleaned up function typo: slapi_disordely_shutdown
|
|
|
8394b4 |
|
|
|
8394b4 |
relates: https://pagure.io/389-ds-base/issue/50599
|
|
|
8394b4 |
|
|
|
8394b4 |
Reviewed by: firstyear & lkrispen (Thanks!!)
|
|
|
8394b4 |
---
|
|
|
8394b4 |
.../plugins/replication/repl5_replica.c | 4 +-
|
|
|
8394b4 |
.../slapd/back-ldbm/db-bdb/bdb_layer.c | 88 ++++++++++---------
|
|
|
8394b4 |
ldap/servers/slapd/plugin.c | 8 +-
|
|
|
8394b4 |
ldap/servers/slapd/slapi-plugin.h | 2 +-
|
|
|
8394b4 |
4 files changed, 54 insertions(+), 48 deletions(-)
|
|
|
8394b4 |
|
|
|
8394b4 |
diff --git a/ldap/servers/plugins/replication/repl5_replica.c b/ldap/servers/plugins/replication/repl5_replica.c
|
|
|
8394b4 |
index 94507bff8..02caa88d9 100644
|
|
|
8394b4 |
--- a/ldap/servers/plugins/replication/repl5_replica.c
|
|
|
8394b4 |
+++ b/ldap/servers/plugins/replication/repl5_replica.c
|
|
|
8394b4 |
@@ -1657,7 +1657,7 @@ replica_check_for_data_reload(Replica *r, void *arg __attribute__((unused)))
|
|
|
8394b4 |
* sessions.
|
|
|
8394b4 |
*/
|
|
|
8394b4 |
|
|
|
8394b4 |
- if (slapi_disordely_shutdown(PR_FALSE)) {
|
|
|
8394b4 |
+ if (slapi_disorderly_shutdown(PR_FALSE)) {
|
|
|
8394b4 |
slapi_log_err(SLAPI_LOG_WARNING, repl_plugin_name, "replica_check_for_data_reload - "
|
|
|
8394b4 |
"Disorderly shutdown for replica %s. Check if DB RUV needs to be updated\n",
|
|
|
8394b4 |
slapi_sdn_get_dn(r->repl_root));
|
|
|
8394b4 |
@@ -1701,7 +1701,7 @@ replica_check_for_data_reload(Replica *r, void *arg __attribute__((unused)))
|
|
|
8394b4 |
slapi_sdn_get_dn(r->repl_root));
|
|
|
8394b4 |
rc = 0;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
- } /* slapi_disordely_shutdown */
|
|
|
8394b4 |
+ } /* slapi_disorderly_shutdown */
|
|
|
8394b4 |
|
|
|
8394b4 |
object_release(ruv_obj);
|
|
|
8394b4 |
} else /* we have no changes currently logged for this replica */
|
|
|
8394b4 |
diff --git a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c
|
|
|
8394b4 |
index 10f6d401e..2103dac38 100644
|
|
|
8394b4 |
--- a/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c
|
|
|
8394b4 |
+++ b/ldap/servers/slapd/back-ldbm/db-bdb/bdb_layer.c
|
|
|
8394b4 |
@@ -15,6 +15,8 @@
|
|
|
8394b4 |
#include <prclist.h>
|
|
|
8394b4 |
#include <sys/types.h>
|
|
|
8394b4 |
#include <sys/statvfs.h>
|
|
|
8394b4 |
+#include <glob.h>
|
|
|
8394b4 |
+
|
|
|
8394b4 |
|
|
|
8394b4 |
#define DB_OPEN(oflags, db, txnid, file, database, type, flags, mode, rval) \
|
|
|
8394b4 |
{ \
|
|
|
8394b4 |
@@ -990,10 +992,9 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
return_value = dblayer_grok_directory(region_dir,
|
|
|
8394b4 |
DBLAYER_DIRECTORY_READWRITE_ACCESS);
|
|
|
8394b4 |
if (0 != return_value) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT, "bdb_start", "Can't start because the database "
|
|
|
8394b4 |
- "directory \"%s\" either doesn't exist, or is not "
|
|
|
8394b4 |
- "accessible\n",
|
|
|
8394b4 |
- region_dir);
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Can't start because the database directory \"%s\" either doesn't exist, or is not accessible\n",
|
|
|
8394b4 |
+ region_dir);
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
|
|
|
8394b4 |
@@ -1003,10 +1004,9 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
return_value = dblayer_grok_directory(log_dir,
|
|
|
8394b4 |
DBLAYER_DIRECTORY_READWRITE_ACCESS);
|
|
|
8394b4 |
if (0 != return_value) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT, "bdb_start", "Can't start because the log "
|
|
|
8394b4 |
- "directory \"%s\" either doesn't exist, or is not "
|
|
|
8394b4 |
- "accessible\n",
|
|
|
8394b4 |
- log_dir);
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Can't start because the log directory \"%s\" either doesn't exist, or is not accessible\n",
|
|
|
8394b4 |
+ log_dir);
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
}
|
|
|
8394b4 |
@@ -1057,15 +1057,27 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
if (conf->bdb_recovery_required) {
|
|
|
8394b4 |
open_flags |= DB_RECOVER;
|
|
|
8394b4 |
if (DBLAYER_RESTORE_MODE & dbmode) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start", "Recovering database after restore "
|
|
|
8394b4 |
- "from archive.\n");
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start",
|
|
|
8394b4 |
+ "Recovering database after restore from archive.\n");
|
|
|
8394b4 |
} else if (DBLAYER_CLEAN_RECOVER_MODE & dbmode) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start", "Clean up db environment and start "
|
|
|
8394b4 |
- "from archive.\n");
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start",
|
|
|
8394b4 |
+ "Clean up db environment and start from archive.\n");
|
|
|
8394b4 |
} else {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start", "Detected Disorderly Shutdown last "
|
|
|
8394b4 |
- "time Directory Server was running, recovering database.\n");
|
|
|
8394b4 |
- slapi_disordely_shutdown(PR_TRUE);
|
|
|
8394b4 |
+ glob_t globbuf;
|
|
|
8394b4 |
+ char file_pattern[MAXPATHLEN];
|
|
|
8394b4 |
+
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start",
|
|
|
8394b4 |
+ "Detected Disorderly Shutdown last time Directory Server was running, recovering database.\n");
|
|
|
8394b4 |
+ slapi_disorderly_shutdown(PR_TRUE);
|
|
|
8394b4 |
+
|
|
|
8394b4 |
+ /* Better wipe out the region files to help ensure a clean start */
|
|
|
8394b4 |
+ PR_snprintf(file_pattern, MAXPATHLEN, "%s/%s", region_dir, "__db.*");
|
|
|
8394b4 |
+ if (glob(file_pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) {
|
|
|
8394b4 |
+ for (size_t i = 0; i < globbuf.gl_pathc; i++) {
|
|
|
8394b4 |
+ remove(globbuf.gl_pathv[i]);
|
|
|
8394b4 |
+ }
|
|
|
8394b4 |
+ globfree(&globbuf);
|
|
|
8394b4 |
+ }
|
|
|
8394b4 |
}
|
|
|
8394b4 |
}
|
|
|
8394b4 |
switch (dbmode & DBLAYER_RESTORE_MASK) {
|
|
|
8394b4 |
@@ -1121,7 +1133,7 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
*/
|
|
|
8394b4 |
if (conf->bdb_lock_config <= BDB_LOCK_NB_MIN) {
|
|
|
8394b4 |
slapi_log_err(SLAPI_LOG_NOTICE, "bdb_start", "New max db lock count is too small. "
|
|
|
8394b4 |
- "Resetting it to the default value %d.\n",
|
|
|
8394b4 |
+ "Resetting it to the default value %d.\n",
|
|
|
8394b4 |
BDB_LOCK_NB_MIN);
|
|
|
8394b4 |
conf->bdb_lock_config = BDB_LOCK_NB_MIN;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
@@ -1165,29 +1177,26 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
if ((open_flags & DB_RECOVER) || (open_flags & DB_RECOVER_FATAL)) {
|
|
|
8394b4 |
/* Recover, then close, then open again */
|
|
|
8394b4 |
int recover_flags = open_flags & ~DB_THREAD;
|
|
|
8394b4 |
-
|
|
|
8394b4 |
if (DBLAYER_CLEAN_RECOVER_MODE & dbmode) /* upgrade case */
|
|
|
8394b4 |
{
|
|
|
8394b4 |
DB_ENV *thisenv = pEnv->bdb_DB_ENV;
|
|
|
8394b4 |
return_value = thisenv->remove(thisenv, region_dir, DB_FORCE);
|
|
|
8394b4 |
if (0 != return_value) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
- "bdb_start", "Failed to remove old db env "
|
|
|
8394b4 |
- "in %s: %s\n",
|
|
|
8394b4 |
- region_dir,
|
|
|
8394b4 |
- dblayer_strerror(return_value));
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Failed to remove old db env in %s: %s\n",
|
|
|
8394b4 |
+ region_dir, dblayer_strerror(return_value));
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
dbmode = DBLAYER_NORMAL_MODE;
|
|
|
8394b4 |
|
|
|
8394b4 |
if ((return_value = bdb_make_env(&pEnv, li)) != 0) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
- "bdb_start", "Failed to create DBENV (returned: %d).\n",
|
|
|
8394b4 |
- return_value);
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Failed to create DBENV (returned: %d).\n", return_value);
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
}
|
|
|
8394b4 |
|
|
|
8394b4 |
+
|
|
|
8394b4 |
return_value = (pEnv->bdb_DB_ENV->open)(
|
|
|
8394b4 |
pEnv->bdb_DB_ENV,
|
|
|
8394b4 |
region_dir,
|
|
|
8394b4 |
@@ -1201,27 +1210,25 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
*/
|
|
|
8394b4 |
slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
"bdb_start", "mmap in opening database environment (recovery mode) "
|
|
|
8394b4 |
- "failed trying to allocate %" PRIu64 " bytes. (OS err %d - %s)\n",
|
|
|
8394b4 |
+ "failed trying to allocate %" PRIu64 " bytes. (OS err %d - %s)\n",
|
|
|
8394b4 |
li->li_dbcachesize, return_value, dblayer_strerror(return_value));
|
|
|
8394b4 |
bdb_free_env(&priv->dblayer_env);
|
|
|
8394b4 |
priv->dblayer_env = CATASTROPHIC;
|
|
|
8394b4 |
} else {
|
|
|
8394b4 |
slapi_log_err(SLAPI_LOG_CRIT, "bdb_start", "Database Recovery Process FAILED. "
|
|
|
8394b4 |
- "The database is not recoverable. err=%d: %s\n",
|
|
|
8394b4 |
+ "The database is not recoverable. err=%d: %s\n",
|
|
|
8394b4 |
return_value, dblayer_strerror(return_value));
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
- "bdb_start", "Please make sure there is enough disk space for "
|
|
|
8394b4 |
- "dbcache (%" PRIu64 " bytes) and db region files\n",
|
|
|
8394b4 |
- li->li_dbcachesize);
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Please make sure there is enough disk space for dbcache (%" PRIu64 " bytes) and db region files\n",
|
|
|
8394b4 |
+ li->li_dbcachesize);
|
|
|
8394b4 |
}
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
} else {
|
|
|
8394b4 |
open_flags &= ~(DB_RECOVER | DB_RECOVER_FATAL);
|
|
|
8394b4 |
pEnv->bdb_DB_ENV->close(pEnv->bdb_DB_ENV, 0);
|
|
|
8394b4 |
if ((return_value = bdb_make_env(&pEnv, li)) != 0) {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
- "bdb_start", "Failed to create DBENV (returned: %d).\n",
|
|
|
8394b4 |
- return_value);
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Failed to create DBENV (returned: %d).\n", return_value);
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
bdb_free_env(&priv->dblayer_env);
|
|
|
8394b4 |
@@ -1288,16 +1295,15 @@ bdb_start(struct ldbminfo *li, int dbmode)
|
|
|
8394b4 |
* https://blackflag.mcom.com/show_bug.cgi?id=557319
|
|
|
8394b4 |
* Crash ns-slapd while running scalab01 after restart slapd
|
|
|
8394b4 |
*/
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
- "bdb_start", "mmap in opening database environment "
|
|
|
8394b4 |
- "failed trying to allocate %" PRIu64 " bytes. (OS err %d - %s)\n",
|
|
|
8394b4 |
- li->li_dbcachesize, return_value, dblayer_strerror(return_value));
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "mmap in opening database environment failed trying to allocate %" PRIu64 " bytes. (OS err %d - %s)\n",
|
|
|
8394b4 |
+ li->li_dbcachesize, return_value, dblayer_strerror(return_value));
|
|
|
8394b4 |
bdb_free_env(&priv->dblayer_env);
|
|
|
8394b4 |
priv->dblayer_env = CATASTROPHIC;
|
|
|
8394b4 |
} else {
|
|
|
8394b4 |
- slapi_log_err(SLAPI_LOG_CRIT,
|
|
|
8394b4 |
- "bdb_start", "Opening database environment (%s) failed. err=%d: %s\n",
|
|
|
8394b4 |
- region_dir, return_value, dblayer_strerror(return_value));
|
|
|
8394b4 |
+ slapi_log_err(SLAPI_LOG_CRIT, "bdb_start",
|
|
|
8394b4 |
+ "Opening database environment (%s) failed. err=%d: %s\n",
|
|
|
8394b4 |
+ region_dir, return_value, dblayer_strerror(return_value));
|
|
|
8394b4 |
}
|
|
|
8394b4 |
}
|
|
|
8394b4 |
return return_value;
|
|
|
8394b4 |
diff --git a/ldap/servers/slapd/plugin.c b/ldap/servers/slapd/plugin.c
|
|
|
8394b4 |
index b00c1bd8f..282b98738 100644
|
|
|
8394b4 |
--- a/ldap/servers/slapd/plugin.c
|
|
|
8394b4 |
+++ b/ldap/servers/slapd/plugin.c
|
|
|
8394b4 |
@@ -4383,14 +4383,14 @@ slapi_set_plugin_open_rootdn_bind(Slapi_PBlock *pb)
|
|
|
8394b4 |
}
|
|
|
8394b4 |
|
|
|
8394b4 |
PRBool
|
|
|
8394b4 |
-slapi_disordely_shutdown(PRBool set)
|
|
|
8394b4 |
+slapi_disorderly_shutdown(PRBool set)
|
|
|
8394b4 |
{
|
|
|
8394b4 |
- static PRBool is_disordely_shutdown = PR_FALSE;
|
|
|
8394b4 |
+ static PRBool is_disorderly_shutdown = PR_FALSE;
|
|
|
8394b4 |
|
|
|
8394b4 |
if (set) {
|
|
|
8394b4 |
- is_disordely_shutdown = PR_TRUE;
|
|
|
8394b4 |
+ is_disorderly_shutdown = PR_TRUE;
|
|
|
8394b4 |
}
|
|
|
8394b4 |
- return (is_disordely_shutdown);
|
|
|
8394b4 |
+ return (is_disorderly_shutdown);
|
|
|
8394b4 |
}
|
|
|
8394b4 |
|
|
|
8394b4 |
/*
|
|
|
8394b4 |
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
|
|
|
8394b4 |
index 29a6238d9..50b8d12c8 100644
|
|
|
8394b4 |
--- a/ldap/servers/slapd/slapi-plugin.h
|
|
|
8394b4 |
+++ b/ldap/servers/slapd/slapi-plugin.h
|
|
|
8394b4 |
@@ -7900,7 +7900,7 @@ uint64_t slapi_str_to_u64(const char *s);
|
|
|
8394b4 |
|
|
|
8394b4 |
void slapi_set_plugin_open_rootdn_bind(Slapi_PBlock *pb);
|
|
|
8394b4 |
|
|
|
8394b4 |
-PRBool slapi_disordely_shutdown(PRBool set);
|
|
|
8394b4 |
+PRBool slapi_disorderly_shutdown(PRBool set);
|
|
|
8394b4 |
|
|
|
8394b4 |
/*
|
|
|
8394b4 |
* Public entry extension getter/setter functions
|
|
|
8394b4 |
--
|
|
|
8394b4 |
2.21.1
|
|
|
8394b4 |
|