Blame SOURCES/ruby-2.5.0-st.c-retry-operations-if-rebuilt.patch

e076ba
From 4663c224fa6c925ce54af32fd1c1cbac9508f5ec Mon Sep 17 00:00:00 2001
e076ba
From: normal <normal@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
e076ba
Date: Tue, 13 Feb 2018 10:02:07 +0000
e076ba
Subject: [PATCH] st.c: retry operations if rebuilt
e076ba
e076ba
Calling the .eql? and .hash methods during a Hash operation can
e076ba
result in a thread switch or a signal handler to run: allowing
e076ba
one execution context to rebuild the hash table while another is
e076ba
still reading or writing the table.  This results in a
e076ba
use-after-free bug affecting the thread_safe-0.3.6 test suite
e076ba
and likely other bugs.
e076ba
e076ba
This bug did not affect users of commonly keys (String, Symbol,
e076ba
Fixnum) as those are optimized to avoid method dispatch
e076ba
for .eql? and .hash methods.
e076ba
e076ba
A separate version of this change needs to be ported to Ruby 2.3.x
e076ba
which had a different implementation of st.c but was affected
e076ba
by the same bug.
e076ba
e076ba
* st.c: Add comment about table rebuilding during comparison.
e076ba
  (DO_PTR_EQUAL_CHECK): New macro.
e076ba
  (REBUILT_TABLE_ENTRY_IND, REBUILT_TABLE_BIN_IND): New macros.
e076ba
  (find_entry, find_table_entry_ind, find_table_bin_ind): Use new
e076ba
  macros.  Return the rebuild flag.
e076ba
  (find_table_bin_ptr_and_reserve): Ditto.
e076ba
  (st_lookup, st_get_key, st_insert, st_insert2): Retry the
e076ba
  operation if the table was rebuilt.
e076ba
  (st_general_delete, st_shift, st_update, st_general_foreach):
e076ba
  Ditto.
e076ba
  (st_rehash_linear, st_rehash_indexed): Use DO_PTR_EQUAL_CHECK.
e076ba
  Return the rebuild flag.
e076ba
  (st_rehash): Retry the operation if the table was rebuilt.
e076ba
  [ruby-core:85510] [Ruby trunk Bug#14357]
e076ba
e076ba
Thanks to Vit Ondruch for reporting the bug.
e076ba
e076ba
From: Vladimir Makarov <vmakarov@redhat.com>
e076ba
e076ba
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@62396 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
e076ba
---
e076ba
 st.c | 258 ++++++++++++++++++++++++++++++++++++++++++++++++-------------------
e076ba
 1 file changed, 185 insertions(+), 73 deletions(-)
e076ba
e076ba
diff --git a/st.c b/st.c
e076ba
index 56ae30ce47..0c52e7a2ef 100644
e076ba
--- a/st.c
e076ba
+++ b/st.c
e076ba
@@ -90,6 +90,11 @@
e076ba
    o To save more memory we use 8-, 16-, 32- and 64- bit indexes in
e076ba
      bins depending on the current hash table size.
e076ba
 
e076ba
+   o The implementation takes into account that the table can be
e076ba
+     rebuilt during hashing or comparison functions.  It can happen if
e076ba
+     the functions are implemented in Ruby and a thread switch occurs
e076ba
+     during their execution.
e076ba
+
e076ba
    This implementation speeds up the Ruby hash table benchmarks in
e076ba
    average by more 40% on Intel Haswell CPU.
e076ba
 
e076ba
@@ -174,6 +179,15 @@ static const struct st_hash_type type_strcasehash = {
e076ba
 #define PTR_EQUAL(tab, ptr, hash_val, key_) \
e076ba
     ((ptr)->hash == (hash_val) && EQUAL((tab), (key_), (ptr)->key))
e076ba
 
e076ba
+/* As PRT_EQUAL only its result is returned in RES.  REBUILT_P is set
e076ba
+   up to TRUE if the table is rebuilt during the comparison.  */
e076ba
+#define DO_PTR_EQUAL_CHECK(tab, ptr, hash_val, key, res, rebuilt_p) \
e076ba
+    do {							    \
e076ba
+	unsigned int _old_rebuilds_num = (tab)->rebuilds_num;       \
e076ba
+	res = PTR_EQUAL(tab, ptr, hash_val, key);		    \
e076ba
+	rebuilt_p = _old_rebuilds_num != (tab)->rebuilds_num;	    \
e076ba
+    } while (FALSE)
e076ba
+
e076ba
 /* Features of a table.  */
e076ba
 struct st_features {
e076ba
     /* Power of 2 used for number of allocated entries.  */
e076ba
@@ -380,6 +394,11 @@ set_bin(st_index_t *bins, int s, st_index_t n, st_index_t v)
e076ba
 #define UNDEFINED_ENTRY_IND (~(st_index_t) 0)
e076ba
 #define UNDEFINED_BIN_IND (~(st_index_t) 0)
e076ba
 
e076ba
+/* Entry and bin values returned when we found a table rebuild during
e076ba
+   the search.  */
e076ba
+#define REBUILT_TABLE_ENTRY_IND (~(st_index_t) 1)
e076ba
+#define REBUILT_TABLE_BIN_IND (~(st_index_t) 1)
e076ba
+
e076ba
 /* Mark I-th bin of table TAB as corresponding to a deleted table
e076ba
    entry.  Update number of entries in the table and number of bins
e076ba
    corresponding to deleted entries. */
e076ba
@@ -823,17 +842,22 @@ secondary_hash(st_index_t ind, st_table *tab, st_index_t *perterb)
e076ba
 
e076ba
 /* Find an entry with HASH_VALUE and KEY in TABLE using a linear
e076ba
    search.  Return the index of the found entry in array `entries`.
e076ba
-   If it is not found, return UNDEFINED_ENTRY_IND.  */
e076ba
+   If it is not found, return UNDEFINED_ENTRY_IND.  If the table was
e076ba
+   rebuilt during the search, return REBUILT_TABLE_ENTRY_IND.  */
e076ba
 static inline st_index_t
e076ba
 find_entry(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
 {
e076ba
+    int eq_p, rebuilt_p;
e076ba
     st_index_t i, bound;
e076ba
     st_table_entry *entries;
e076ba
 
e076ba
     bound = tab->entries_bound;
e076ba
     entries = tab->entries;
e076ba
     for (i = tab->entries_start; i < bound; i++) {
e076ba
-	if (PTR_EQUAL(tab, &entries[i], hash_value, key))
e076ba
+	DO_PTR_EQUAL_CHECK(tab, &entries[i], hash_value, key, eq_p, rebuilt_p);
e076ba
+	if (EXPECT(rebuilt_p, 0))
e076ba
+	    return REBUILT_TABLE_ENTRY_IND;
e076ba
+	if (eq_p)
e076ba
 	    return i;
e076ba
     }
e076ba
     return UNDEFINED_ENTRY_IND;
e076ba
@@ -845,10 +869,12 @@ find_entry(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
 /*#define QUADRATIC_PROBE*/
e076ba
 
e076ba
 /* Return index of entry with HASH_VALUE and KEY in table TAB.  If
e076ba
-   there is no such entry, return UNDEFINED_ENTRY_IND.  */
e076ba
+   there is no such entry, return UNDEFINED_ENTRY_IND.  If the table
e076ba
+   was rebuilt during the search, return REBUILT_TABLE_ENTRY_IND.  */
e076ba
 static st_index_t
e076ba
 find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
 {
e076ba
+    int eq_p, rebuilt_p;
e076ba
     st_index_t ind;
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
     st_index_t d;
e076ba
@@ -869,10 +895,13 @@ find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
     FOUND_BIN;
e076ba
     for (;;) {
e076ba
         bin = get_bin(tab->bins, get_size_ind(tab), ind);
e076ba
-        if (! EMPTY_OR_DELETED_BIN_P(bin)
e076ba
-            && PTR_EQUAL(tab, &entries[bin - ENTRY_BASE], hash_value, key))
e076ba
-            break;
e076ba
-        else if (EMPTY_BIN_P(bin))
e076ba
+        if (! EMPTY_OR_DELETED_BIN_P(bin)) {
e076ba
+	    DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
e076ba
+	    if (EXPECT(rebuilt_p, 0))
e076ba
+		return REBUILT_TABLE_ENTRY_IND;
e076ba
+	    if (eq_p)
e076ba
+		break;
e076ba
+	} else if (EMPTY_BIN_P(bin))
e076ba
             return UNDEFINED_ENTRY_IND;
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
 	ind = hash_bin(ind + d, tab);
e076ba
@@ -887,10 +916,12 @@ find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
 
e076ba
 /* Find and return index of table TAB bin corresponding to an entry
e076ba
    with HASH_VALUE and KEY.  If there is no such bin, return
e076ba
-   UNDEFINED_BIN_IND.  */
e076ba
+   UNDEFINED_BIN_IND.  If the table was rebuilt during the search,
e076ba
+   return REBUILT_TABLE_BIN_IND.  */
e076ba
 static st_index_t
e076ba
 find_table_bin_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
 {
e076ba
+    int eq_p, rebuilt_p;
e076ba
     st_index_t ind;
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
     st_index_t d;
e076ba
@@ -911,10 +942,13 @@ find_table_bin_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
     FOUND_BIN;
e076ba
     for (;;) {
e076ba
         bin = get_bin(tab->bins, get_size_ind(tab), ind);
e076ba
-        if (! EMPTY_OR_DELETED_BIN_P(bin)
e076ba
-            && PTR_EQUAL(tab, &entries[bin - ENTRY_BASE], hash_value, key))
e076ba
-            break;
e076ba
-        else if (EMPTY_BIN_P(bin))
e076ba
+        if (! EMPTY_OR_DELETED_BIN_P(bin)) {
e076ba
+	    DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
e076ba
+	    if (EXPECT(rebuilt_p, 0))
e076ba
+		return REBUILT_TABLE_BIN_IND;
e076ba
+	    if (eq_p)
e076ba
+		break;
e076ba
+	} else if (EMPTY_BIN_P(bin))
e076ba
             return UNDEFINED_BIN_IND;
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
 	ind = hash_bin(ind + d, tab);
e076ba
@@ -955,7 +989,7 @@ find_table_bin_ind_direct(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
         bin = get_bin(tab->bins, get_size_ind(tab), ind);
e076ba
         if (EMPTY_OR_DELETED_BIN_P(bin))
e076ba
 	    return ind;
e076ba
-	st_assert (! PTR_EQUAL(tab, &entries[bin - ENTRY_BASE], hash_value, key));
e076ba
+	st_assert (entries[bin - ENTRY_BASE].hash != hash_value);
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
 	ind = hash_bin(ind + d, tab);
e076ba
 	d++;
e076ba
@@ -973,11 +1007,13 @@ find_table_bin_ind_direct(st_table *tab, st_hash_t hash_value, st_data_t key)
e076ba
    bigger entries array.  Although we can reuse a deleted bin, the
e076ba
    result bin value is always empty if the table has no entry with
e076ba
    KEY.  Return the entries array index of the found entry or
e076ba
-   UNDEFINED_ENTRY_IND if it is not found.  */
e076ba
+   UNDEFINED_ENTRY_IND if it is not found.  If the table was rebuilt
e076ba
+   during the search, return REBUILT_TABLE_ENTRY_IND.  */
e076ba
 static st_index_t
e076ba
 find_table_bin_ptr_and_reserve(st_table *tab, st_hash_t *hash_value,
e076ba
 			       st_data_t key, st_index_t *bin_ind)
e076ba
 {
e076ba
+    int eq_p, rebuilt_p;
e076ba
     st_index_t ind;
e076ba
     st_hash_t curr_hash_value = *hash_value;
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
@@ -1015,7 +1051,10 @@ find_table_bin_ptr_and_reserve(st_table *tab, st_hash_t *hash_value,
e076ba
             break;
e076ba
 	}
e076ba
 	else if (! DELETED_BIN_P(entry_index)) {
e076ba
-            if (PTR_EQUAL(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key))
e076ba
+	    DO_PTR_EQUAL_CHECK(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key, eq_p, rebuilt_p);
e076ba
+	    if (EXPECT(rebuilt_p, 0))
e076ba
+		return REBUILT_TABLE_ENTRY_IND;
e076ba
+            if (eq_p)
e076ba
                 break;
e076ba
 	}
e076ba
 	else if (first_deleted_bin_ind == UNDEFINED_BIN_IND)
e076ba
@@ -1040,13 +1079,18 @@ st_lookup(st_table *tab, st_data_t key, st_data_t *value)
e076ba
     st_index_t bin;
e076ba
     st_hash_t hash = do_hash(key, tab);
e076ba
 
e076ba
+ retry:
e076ba
     if (tab->bins == NULL) {
e076ba
         bin = find_entry(tab, hash, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	if (bin == UNDEFINED_ENTRY_IND)
e076ba
 	    return 0;
e076ba
     }
e076ba
     else {
e076ba
         bin = find_table_entry_ind(tab, hash, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	if (bin == UNDEFINED_ENTRY_IND)
e076ba
 	    return 0;
e076ba
 	bin -= ENTRY_BASE;
e076ba
@@ -1064,13 +1108,18 @@ st_get_key(st_table *tab, st_data_t key, st_data_t *result)
e076ba
     st_index_t bin;
e076ba
     st_hash_t hash = do_hash(key, tab);
e076ba
 
e076ba
+ retry:
e076ba
     if (tab->bins == NULL) {
e076ba
         bin = find_entry(tab, hash, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	if (bin == UNDEFINED_ENTRY_IND)
e076ba
 	    return 0;
e076ba
     }
e076ba
     else {
e076ba
         bin = find_table_entry_ind(tab, hash, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	if (bin == UNDEFINED_ENTRY_IND)
e076ba
 	    return 0;
e076ba
 	bin -= ENTRY_BASE;
e076ba
@@ -1104,10 +1153,13 @@ st_insert(st_table *tab, st_data_t key, st_data_t value)
e076ba
     st_index_t bin_ind;
e076ba
     int new_p;
e076ba
 
e076ba
-    rebuild_table_if_necessary(tab);
e076ba
     hash_value = do_hash(key, tab);
e076ba
+ retry:
e076ba
+    rebuild_table_if_necessary(tab);
e076ba
     if (tab->bins == NULL) {
e076ba
         bin = find_entry(tab, hash_value, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	new_p = bin == UNDEFINED_ENTRY_IND;
e076ba
 	if (new_p)
e076ba
 	    tab->num_entries++;
e076ba
@@ -1116,6 +1168,8 @@ st_insert(st_table *tab, st_data_t key, st_data_t value)
e076ba
     else {
e076ba
         bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
e076ba
 					     key, &bin_ind);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	new_p = bin == UNDEFINED_ENTRY_IND;
e076ba
 	bin -= ENTRY_BASE;
e076ba
     }
e076ba
@@ -1192,10 +1246,13 @@ st_insert2(st_table *tab, st_data_t key, st_data_t value,
e076ba
     st_index_t bin_ind;
e076ba
     int new_p;
e076ba
 
e076ba
-    rebuild_table_if_necessary (tab);
e076ba
     hash_value = do_hash(key, tab);
e076ba
+ retry:
e076ba
+    rebuild_table_if_necessary (tab);
e076ba
     if (tab->bins == NULL) {
e076ba
         bin = find_entry(tab, hash_value, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	new_p = bin == UNDEFINED_ENTRY_IND;
e076ba
 	if (new_p)
e076ba
 	    tab->num_entries++;
e076ba
@@ -1204,6 +1261,8 @@ st_insert2(st_table *tab, st_data_t key, st_data_t value,
e076ba
     else {
e076ba
         bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
e076ba
 					     key, &bin_ind);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	new_p = bin == UNDEFINED_ENTRY_IND;
e076ba
 	bin -= ENTRY_BASE;
e076ba
     }
e076ba
@@ -1212,7 +1271,6 @@ st_insert2(st_table *tab, st_data_t key, st_data_t value,
e076ba
         check = tab->rebuilds_num;
e076ba
         key = (*func)(key);
e076ba
         st_assert(check == tab->rebuilds_num);
e076ba
-	st_assert(do_hash(key, tab) == hash_value);
e076ba
         ind = tab->entries_bound++;
e076ba
         entry = &tab->entries[ind];
e076ba
         entry->hash = hash_value;
e076ba
@@ -1220,6 +1278,7 @@ st_insert2(st_table *tab, st_data_t key, st_data_t value,
e076ba
         entry->record = value;
e076ba
 	if (bin_ind != UNDEFINED_BIN_IND)
e076ba
 	    set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
e076ba
+	st_assert(do_hash(key, tab) == hash_value);
e076ba
 #ifdef ST_DEBUG
e076ba
 	st_check(tab);
e076ba
 #endif
e076ba
@@ -1281,8 +1340,11 @@ st_general_delete(st_table *tab, st_data_t *key, st_data_t *value)
e076ba
 
e076ba
     st_assert(tab != NULL);
e076ba
     hash = do_hash(*key, tab);
e076ba
+ retry:
e076ba
     if (tab->bins == NULL) {
e076ba
         bin = find_entry(tab, hash, *key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	if (bin == UNDEFINED_ENTRY_IND) {
e076ba
 	    if (value != 0) *value = 0;
e076ba
 	    return 0;
e076ba
@@ -1290,6 +1352,8 @@ st_general_delete(st_table *tab, st_data_t *key, st_data_t *value)
e076ba
     }
e076ba
     else {
e076ba
         bin_ind = find_table_bin_ind(tab, hash, *key);
e076ba
+	if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
e076ba
+	    goto retry;
e076ba
 	if (bin_ind == UNDEFINED_BIN_IND) {
e076ba
 	    if (value != 0) *value = 0;
e076ba
 	    return 0;
e076ba
@@ -1344,21 +1408,33 @@ st_shift(st_table *tab, st_data_t *key, st_data_t *value)
e076ba
     for (i = tab->entries_start; i < bound; i++) {
e076ba
         curr_entry_ptr = &entries[i];
e076ba
 	if (! DELETED_ENTRY_P(curr_entry_ptr)) {
e076ba
+	    st_hash_t entry_hash = curr_entry_ptr->hash;
e076ba
+	    st_data_t entry_key = curr_entry_ptr->key;
e076ba
+
e076ba
 	    if (value != 0) *value = curr_entry_ptr->record;
e076ba
-	    *key = curr_entry_ptr->key;
e076ba
+	    *key = entry_key;
e076ba
+	retry:
e076ba
 	    if (tab->bins == NULL) {
e076ba
-	        bin = find_entry(tab, curr_entry_ptr->hash, curr_entry_ptr->key);
e076ba
+	        bin = find_entry(tab, entry_hash, entry_key);
e076ba
+		if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0)) {
e076ba
+		    entries = tab->entries;
e076ba
+		    goto retry;
e076ba
+		}
e076ba
 		st_assert(bin != UNDEFINED_ENTRY_IND);
e076ba
-		st_assert(&entries[bin] == curr_entry_ptr);
e076ba
+		curr_entry_ptr = &entries[bin];
e076ba
 	    }
e076ba
 	    else {
e076ba
-	        bin_ind = find_table_bin_ind(tab, curr_entry_ptr->hash,
e076ba
-					     curr_entry_ptr->key);
e076ba
+	        bin_ind = find_table_bin_ind(tab, entry_hash, entry_key);
e076ba
+		if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0)) {
e076ba
+		    entries = tab->entries;
e076ba
+		    goto retry;
e076ba
+		}
e076ba
 		st_assert(bin_ind != UNDEFINED_BIN_IND);
e076ba
-		st_assert(&entries[get_bin(tab->bins, get_size_ind(tab), bin_ind)
e076ba
-				      - ENTRY_BASE] == curr_entry_ptr);
e076ba
+		curr_entry_ptr = &entries[get_bin(tab->bins, get_size_ind(tab), bin_ind)
e076ba
+					  - ENTRY_BASE];
e076ba
 		MARK_BIN_DELETED(tab, bin_ind);
e076ba
 	    }
e076ba
+	    st_assert(entry_hash != curr_entry_ptr->hash && entry_key == curr_entry_ptr->key);
e076ba
 	    MARK_ENTRY_DELETED(curr_entry_ptr);
e076ba
 	    tab->num_entries--;
e076ba
 	    update_range_for_deleted(tab, i);
e076ba
@@ -1402,15 +1478,20 @@ st_update(st_table *tab, st_data_t key,
e076ba
     int retval, existing;
e076ba
     st_hash_t hash = do_hash(key, tab);
e076ba
 
e076ba
+ retry:
e076ba
     entries = tab->entries;
e076ba
     if (tab->bins == NULL) {
e076ba
         bin = find_entry(tab, hash, key);
e076ba
+	if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+	    goto retry;
e076ba
 	existing = bin != UNDEFINED_ENTRY_IND;
e076ba
 	entry = &entries[bin];
e076ba
 	bin_ind = UNDEFINED_BIN_IND;
e076ba
     }
e076ba
     else {
e076ba
         bin_ind = find_table_bin_ind(tab, hash, key);
e076ba
+	if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
e076ba
+	    goto retry;
e076ba
 	existing = bin_ind != UNDEFINED_BIN_IND;
e076ba
 	if (existing) {
e076ba
 	    bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
e076ba
@@ -1489,14 +1570,19 @@ st_general_foreach(st_table *tab, int (*func)(ANYARGS), st_data_t arg,
e076ba
 	hash = curr_entry_ptr->hash;
e076ba
 	retval = (*func)(key, curr_entry_ptr->record, arg, 0);
e076ba
 	if (rebuilds_num != tab->rebuilds_num) {
e076ba
+	retry:
e076ba
 	    entries = tab->entries;
e076ba
 	    packed_p = tab->bins == NULL;
e076ba
 	    if (packed_p) {
e076ba
 	        i = find_entry(tab, hash, key);
e076ba
+		if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+		    goto retry;
e076ba
 		error_p = i == UNDEFINED_ENTRY_IND;
e076ba
 	    }
e076ba
 	    else {
e076ba
 	        i = find_table_entry_ind(tab, hash, key);
e076ba
+		if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+		    goto retry;
e076ba
 		error_p = i == UNDEFINED_ENTRY_IND;
e076ba
 		i -= ENTRY_BASE;
e076ba
 	    }
e076ba
@@ -1512,36 +1598,44 @@ st_general_foreach(st_table *tab, int (*func)(ANYARGS), st_data_t arg,
e076ba
 	}
e076ba
 	switch (retval) {
e076ba
 	  case ST_CONTINUE:
e076ba
-	    break;
e076ba
+	      break;
e076ba
 	  case ST_CHECK:
e076ba
-	    if (check_p)
e076ba
-		break;
e076ba
+	      if (check_p)
e076ba
+		  break;
e076ba
 	  case ST_STOP:
e076ba
 #ifdef ST_DEBUG
e076ba
-	    st_check(tab);
e076ba
-#endif
e076ba
-	    return 0;
e076ba
-	  case ST_DELETE:
e076ba
-	    if (packed_p) {
e076ba
-	        bin = find_entry(tab, hash, curr_entry_ptr->key);
e076ba
-		if (bin == UNDEFINED_ENTRY_IND)
e076ba
-		    break;
e076ba
-	    }
e076ba
-	    else {
e076ba
-	        bin_ind = find_table_bin_ind(tab, hash, curr_entry_ptr->key);
e076ba
-		if (bin_ind == UNDEFINED_BIN_IND)
e076ba
-		    break;
e076ba
-		bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
e076ba
-		MARK_BIN_DELETED(tab, bin_ind);
e076ba
-	    }
e076ba
-	    st_assert(&entries[bin] == curr_entry_ptr);
e076ba
-	    MARK_ENTRY_DELETED(curr_entry_ptr);
e076ba
-	    tab->num_entries--;
e076ba
-	    update_range_for_deleted(tab, bin);
e076ba
+	      st_check(tab);
e076ba
+#endif
e076ba
+	      return 0;
e076ba
+	  case ST_DELETE: {
e076ba
+	      st_data_t key = curr_entry_ptr->key;
e076ba
+
e076ba
+	      again:
e076ba
+	      if (packed_p) {
e076ba
+		  bin = find_entry(tab, hash, key);
e076ba
+		  if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
e076ba
+		      goto again;
e076ba
+		  if (bin == UNDEFINED_ENTRY_IND)
e076ba
+		      break;
e076ba
+	      }
e076ba
+	      else {
e076ba
+		  bin_ind = find_table_bin_ind(tab, hash, key);
e076ba
+		  if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
e076ba
+		      goto again;
e076ba
+		  if (bin_ind == UNDEFINED_BIN_IND)
e076ba
+		      break;
e076ba
+		  bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
e076ba
+		  MARK_BIN_DELETED(tab, bin_ind);
e076ba
+	      }
e076ba
+	      curr_entry_ptr = &entries[bin];
e076ba
+	      MARK_ENTRY_DELETED(curr_entry_ptr);
e076ba
+	      tab->num_entries--;
e076ba
+	      update_range_for_deleted(tab, bin);
e076ba
 #ifdef ST_DEBUG
e076ba
-	    st_check(tab);
e076ba
+	      st_check(tab);
e076ba
 #endif
e076ba
-	    break;
e076ba
+	      break;
e076ba
+	  }
e076ba
 	}
e076ba
     }
e076ba
 #ifdef ST_DEBUG
e076ba
@@ -2015,10 +2109,12 @@ st_expand_table(st_table *tab, st_index_t siz)
e076ba
     free(tmp);
e076ba
 }
e076ba
 
e076ba
-/* Rehash using linear search. */
e076ba
-static void
e076ba
+/* Rehash using linear search.  Return TRUE if we found that the table
e076ba
+   was rebuilt.  */
e076ba
+static int
e076ba
 st_rehash_linear(st_table *tab)
e076ba
 {
e076ba
+    int eq_p, rebuilt_p;
e076ba
     st_index_t i, j;
e076ba
     st_table_entry *p, *q;
e076ba
     if (tab->bins) {
e076ba
@@ -2033,7 +2129,10 @@ st_rehash_linear(st_table *tab)
e076ba
             q = &tab->entries[j];
e076ba
             if (DELETED_ENTRY_P(q))
e076ba
                 continue;
e076ba
-            if (PTR_EQUAL(tab, p, q->hash, q->key)) {
e076ba
+	    DO_PTR_EQUAL_CHECK(tab, p, q->hash, q->key, eq_p, rebuilt_p);
e076ba
+	    if (EXPECT(rebuilt_p, 0))
e076ba
+		return TRUE;
e076ba
+	    if (eq_p) {
e076ba
                 st_assert(p < q);
e076ba
                 *p = *q;
e076ba
                 MARK_ENTRY_DELETED(q);
e076ba
@@ -2042,12 +2141,15 @@ st_rehash_linear(st_table *tab)
e076ba
             }
e076ba
         }
e076ba
     }
e076ba
+    return FALSE;
e076ba
 }
e076ba
 
e076ba
-/* Rehash using index */
e076ba
-static void
e076ba
+/* Rehash using index.  Return TRUE if we found that the table was
e076ba
+   rebuilt.  */
e076ba
+static int
e076ba
 st_rehash_indexed(st_table *tab)
e076ba
 {
e076ba
+    int eq_p, rebuilt_p;
e076ba
     st_index_t i;
e076ba
     st_index_t const n = bins_size(tab);
e076ba
     unsigned int const size_ind = get_size_ind(tab);
e076ba
@@ -2076,26 +2178,32 @@ st_rehash_indexed(st_table *tab)
e076ba
                 set_bin(bins, size_ind, ind, i + ENTRY_BASE);
e076ba
                 break;
e076ba
             }
e076ba
-            else if (PTR_EQUAL(tab, q, p->hash, p->key)) {
e076ba
-                /* duplicated key; delete it */
e076ba
-                st_assert(q < p);
e076ba
-                q->record = p->record;
e076ba
-                MARK_ENTRY_DELETED(p);
e076ba
-                tab->num_entries--;
e076ba
-                update_range_for_deleted(tab, bin);
e076ba
-                break;
e076ba
-            }
e076ba
             else {
e076ba
-                /* hash collision; skip it */
e076ba
+		DO_PTR_EQUAL_CHECK(tab, q, p->hash, p->key, eq_p, rebuilt_p);
e076ba
+		if (EXPECT(rebuilt_p, 0))
e076ba
+		    return TRUE;
e076ba
+		if (eq_p) {
e076ba
+		    /* duplicated key; delete it */
e076ba
+		    st_assert(q < p);
e076ba
+		    q->record = p->record;
e076ba
+		    MARK_ENTRY_DELETED(p);
e076ba
+		    tab->num_entries--;
e076ba
+		    update_range_for_deleted(tab, bin);
e076ba
+		    break;
e076ba
+		}
e076ba
+		else {
e076ba
+		    /* hash collision; skip it */
e076ba
 #ifdef QUADRATIC_PROBE
e076ba
-                ind = hash_bin(ind + d, tab);
e076ba
-                d++;
e076ba
+		    ind = hash_bin(ind + d, tab);
e076ba
+		    d++;
e076ba
 #else
e076ba
-                ind = secondary_hash(ind, tab, &peterb);
e076ba
+		    ind = secondary_hash(ind, tab, &peterb);
e076ba
 #endif
e076ba
-            }
e076ba
+		}
e076ba
+	    }
e076ba
         }
e076ba
     }
e076ba
+    return FALSE;
e076ba
 }
e076ba
 
e076ba
 /* Reconstruct TAB's bins according to TAB's entries. This function
e076ba
@@ -2104,10 +2212,14 @@ st_rehash_indexed(st_table *tab)
e076ba
 static void
e076ba
 st_rehash(st_table *tab)
e076ba
 {
e076ba
-    if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
e076ba
-        st_rehash_linear(tab);
e076ba
-    else
e076ba
-        st_rehash_indexed(tab);
e076ba
+    int rebuilt_p;
e076ba
+
e076ba
+    do {
e076ba
+	if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
e076ba
+	    rebuilt_p = st_rehash_linear(tab);
e076ba
+	else
e076ba
+	    rebuilt_p = st_rehash_indexed(tab);
e076ba
+    } while (rebuilt_p);
e076ba
 }
e076ba
 
e076ba
 #ifdef RUBY
e076ba
-- 
e076ba
2.16.1
e076ba