5f7b84
commit f88c59f4657ac2e0bab8f51f60022ecbe7f12e2e
5f7b84
Author: Wilco Dijkstra <wdijkstr@arm.com>
5f7b84
Date:   Fri May 17 18:16:20 2019 +0100
5f7b84
5f7b84
    Small tcache improvements
5f7b84
    
5f7b84
    Change the tcache->counts[] entries to uint16_t - this removes
5f7b84
    the limit set by char and allows a larger tcache.  Remove a few
5f7b84
    redundant asserts.
5f7b84
    
5f7b84
    bench-malloc-thread with 4 threads is ~15% faster on Cortex-A72.
5f7b84
    
5f7b84
    Reviewed-by: DJ Delorie <dj@redhat.com>
5f7b84
    
5f7b84
            * malloc/malloc.c (MAX_TCACHE_COUNT): Increase to UINT16_MAX.
5f7b84
            (tcache_put): Remove redundant assert.
5f7b84
            (tcache_get): Remove redundant asserts.
5f7b84
            (__libc_malloc): Check tcache count is not zero.
5f7b84
            * manual/tunables.texi (glibc.malloc.tcache_count): Update maximum.
5f7b84
    
5f7b84
    (cherry picked from commit 1f50f2ad854c84ead522bfc7331b46dbe6057d53)
5f7b84
5f7b84
diff --git a/malloc/malloc.c b/malloc/malloc.c
5f7b84
index 998879aededf0d7c..e6a483d5cf7c4312 100644
5f7b84
--- a/malloc/malloc.c
5f7b84
+++ b/malloc/malloc.c
5f7b84
@@ -321,6 +321,10 @@ __malloc_assert (const char *assertion, const char *file, unsigned int line,
5f7b84
 /* This is another arbitrary limit, which tunables can change.  Each
5f7b84
    tcache bin will hold at most this number of chunks.  */
5f7b84
 # define TCACHE_FILL_COUNT 7
5f7b84
+
5f7b84
+/* Maximum chunks in tcache bins for tunables.  This value must fit the range
5f7b84
+   of tcache->counts[] entries, else they may overflow.  */
5f7b84
+# define MAX_TCACHE_COUNT UINT16_MAX
5f7b84
 #endif
5f7b84
 
5f7b84
 
5f7b84
@@ -2915,12 +2919,10 @@ typedef struct tcache_entry
5f7b84
    time), this is for performance reasons.  */
5f7b84
 typedef struct tcache_perthread_struct
5f7b84
 {
5f7b84
-  char counts[TCACHE_MAX_BINS];
5f7b84
+  uint16_t counts[TCACHE_MAX_BINS];
5f7b84
   tcache_entry *entries[TCACHE_MAX_BINS];
5f7b84
 } tcache_perthread_struct;
5f7b84
 
5f7b84
-#define MAX_TCACHE_COUNT 127	/* Maximum value of counts[] entries.  */
5f7b84
-
5f7b84
 static __thread bool tcache_shutting_down = false;
5f7b84
 static __thread tcache_perthread_struct *tcache = NULL;
5f7b84
 
5f7b84
@@ -2930,7 +2932,6 @@ static __always_inline void
5f7b84
 tcache_put (mchunkptr chunk, size_t tc_idx)
5f7b84
 {
5f7b84
   tcache_entry *e = (tcache_entry *) chunk2mem (chunk);
5f7b84
-  assert (tc_idx < TCACHE_MAX_BINS);
5f7b84
 
5f7b84
   /* Mark this chunk as "in the tcache" so the test in _int_free will
5f7b84
      detect a double free.  */
5f7b84
@@ -2947,8 +2948,6 @@ static __always_inline void *
5f7b84
 tcache_get (size_t tc_idx)
5f7b84
 {
5f7b84
   tcache_entry *e = tcache->entries[tc_idx];
5f7b84
-  assert (tc_idx < TCACHE_MAX_BINS);
5f7b84
-  assert (tcache->counts[tc_idx] > 0);
5f7b84
   tcache->entries[tc_idx] = e->next;
5f7b84
   --(tcache->counts[tc_idx]);
5f7b84
   e->key = NULL;
5f7b84
@@ -3053,9 +3052,8 @@ __libc_malloc (size_t bytes)
5f7b84
 
5f7b84
   DIAG_PUSH_NEEDS_COMMENT;
5f7b84
   if (tc_idx < mp_.tcache_bins
5f7b84
-      /*&& tc_idx < TCACHE_MAX_BINS*/ /* to appease gcc */
5f7b84
       && tcache
5f7b84
-      && tcache->entries[tc_idx] != NULL)
5f7b84
+      && tcache->counts[tc_idx] > 0)
5f7b84
     {
5f7b84
       return tcache_get (tc_idx);
5f7b84
     }
5f7b84
diff --git a/manual/tunables.texi b/manual/tunables.texi
5f7b84
index 9dccf2ee7f8eec17..f6c49250e3889ddd 100644
5f7b84
--- a/manual/tunables.texi
5f7b84
+++ b/manual/tunables.texi
5f7b84
@@ -188,7 +188,7 @@ per-thread cache.  The default (and maximum) value is 1032 bytes on
5f7b84
 
5f7b84
 @deftp Tunable glibc.malloc.tcache_count
5f7b84
 The maximum number of chunks of each size to cache. The default is 7.
5f7b84
-The upper limit is 127.  If set to zero, the per-thread cache is effectively
5f7b84
+The upper limit is 65535.  If set to zero, the per-thread cache is effectively
5f7b84
 disabled.
5f7b84
 
5f7b84
 The approximate maximum overhead of the per-thread cache is thus equal