From bcab8c3cd877506de75f50e0f9ed98827ed554b0 Mon Sep 17 00:00:00 2001

From: Peter Zhu <peter@peterzhu.ca>

Date: Tue, 23 Feb 2021 16:28:56 -0500

Subject: [PATCH] Use mmap for allocating heap pages

---

 configure.ac                 |  16 ++++

 gc.c                         | 149 ++++++++++++++++++++++++++---------

 test/ruby/test_gc_compact.rb |  41 ++++++----

 3 files changed, 155 insertions(+), 51 deletions(-)

diff --git a/configure.ac b/configure.ac

index 2dcebdde9f..b1b190004d 100644

--- a/configure.ac

+++ b/configure.ac

@@ -1944,6 +1944,7 @@ AC_CHECK_FUNCS(memmem)

 AC_CHECK_FUNCS(mkfifo)

 AC_CHECK_FUNCS(mknod)

 AC_CHECK_FUNCS(mktime)

+AC_CHECK_FUNCS(mmap)

 AC_CHECK_FUNCS(openat)

 AC_CHECK_FUNCS(pipe2)

 AC_CHECK_FUNCS(poll)

@@ -2666,6 +2667,21 @@ main(int argc, char *argv[])

 	rb_cv_fork_with_pthread=yes)])

     test x$rb_cv_fork_with_pthread = xyes || AC_DEFINE(CANNOT_FORK_WITH_PTHREAD)

 ])

+

+AC_CHECK_HEADERS([sys/user.h])

+AS_IF([test "x$ac_cv_func_mmap" = xyes], [

+    AC_CACHE_CHECK([whether PAGE_SIZE is compile-time const], rb_cv_const_page_size,

+	[malloc_headers=`sed -n '/MALLOC_HEADERS_BEGIN/,/MALLOC_HEADERS_END/p' ${srcdir}/gc.c`

+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$malloc_headers

+            typedef char conftest_page[PAGE_SIZE];

+        ]], [[]])],

+        [rb_cv_const_page_size=yes],

+        [rb_cv_const_page_size=no])])

+])

+AS_IF([test "x$rb_cv_const_page_size" = xyes],

+    [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 1)],

+    [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 0)]

+)

 }

 : "runtime section" && {

diff --git a/gc.c b/gc.c

index f6acf3e117..6f8e5f242d 100644

--- a/gc.c

+++ b/gc.c

@@ -32,6 +32,7 @@

 #include <stdarg.h>

 #include <stdio.h>

+/* MALLOC_HEADERS_BEGIN */

 #ifndef HAVE_MALLOC_USABLE_SIZE

 # ifdef _WIN32

 #  define HAVE_MALLOC_USABLE_SIZE

@@ -54,6 +55,12 @@

 # endif

 #endif

+#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)

+/* LIST_HEAD conflicts with sys/queue.h on macOS */

+# include <sys/user.h>

+#endif

+/* MALLOC_HEADERS_END */

+

 #ifdef HAVE_SYS_TIME_H

 # include <sys/time.h>

 #endif

@@ -821,6 +828,25 @@ enum {

     HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),

     HEAP_PAGE_BITMAP_PLANES = 4 /* RGENGC: mark, unprotected, uncollectible, marking */

 };

+#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)

+#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN

+

+#ifdef HAVE_MMAP

+# if HAVE_CONST_PAGE_SIZE

+/* If we have the HEAP_PAGE and it is a constant, then we can directly use it. */

+static const bool USE_MMAP_ALIGNED_ALLOC = (PAGE_SIZE <= HEAP_PAGE_SIZE);

+# elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)

+/* PAGE_SIZE <= HEAP_PAGE_SIZE */

+static const bool USE_MMAP_ALIGNED_ALLOC = true;

+# else

+/* Otherwise, fall back to determining if we can use mmap during runtime. */

+#  define USE_MMAP_ALIGNED_ALLOC (use_mmap_aligned_alloc != false)

+

+static bool use_mmap_aligned_alloc;

+# endif

+#elif !defined(__MINGW32__) && !defined(_WIN32)

+static const bool USE_MMAP_ALIGNED_ALLOC = false;

+#endif

 struct heap_page {

     short total_slots;

@@ -1760,14 +1786,14 @@ heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *pag

     heap->total_slots -= page->total_slots;

 }

-static void rb_aligned_free(void *ptr);

+static void rb_aligned_free(void *ptr, size_t size);

 static void

 heap_page_free(rb_objspace_t *objspace, struct heap_page *page)

 {

     heap_allocated_pages--;

     objspace->profile.total_freed_pages++;

-    rb_aligned_free(GET_PAGE_BODY(page->start));

+    rb_aligned_free(GET_PAGE_BODY(page->start), HEAP_PAGE_SIZE);

     free(page);

 }

@@ -1819,7 +1845,7 @@ heap_page_allocate(rb_objspace_t *objspace)

     /* assign heap_page entry */

     page = calloc1(sizeof(struct heap_page));

     if (page == 0) {

-        rb_aligned_free(page_body);

+        rb_aligned_free(page_body, HEAP_PAGE_SIZE);

 	rb_memerror();

     }

@@ -3159,15 +3185,18 @@ Init_heap(void)

 {

     rb_objspace_t *objspace = &rb_objspace;

-#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)

-    /* If Ruby's heap pages are not a multiple of the system page size, we

-     * cannot use mprotect for the read barrier, so we must disable automatic

-     * compaction. */

-    int pagesize;

-    pagesize = (int)sysconf(_SC_PAGE_SIZE);

-    if ((HEAP_PAGE_SIZE % pagesize) != 0) {

-        ruby_enable_autocompact = 0;

-    }

+#if defined(HAVE_MMAP) && !HAVE_CONST_PAGE_SIZE && !defined(PAGE_MAX_SIZE)

+    /* Need to determine if we can use mmap at runtime. */

+# ifdef PAGE_SIZE

+    /* If the PAGE_SIZE macro can be used. */

+    use_mmap_aligned_alloc = PAGE_SIZE <= HEAP_PAGE_SIZE;

+# elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)

+    /* If we can use sysconf to determine the page size. */

+    use_mmap_aligned_alloc = sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE;

+# else

+    /* Otherwise we can't determine the system page size, so don't use mmap. */

+    use_mmap_aligned_alloc = FALSE;

+# endif

 #endif

     objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL);

@@ -8533,6 +8562,14 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE

     /* For now, compact implies full mark / sweep, so ignore other flags */

     if (RTEST(compact)) {

+        /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for

+         * the read barrier, so we must disable compaction. */

+#if !defined(__MINGW32__) && !defined(_WIN32)

+        if (!USE_MMAP_ALIGNED_ALLOC) {

+            rb_raise(rb_eNotImpError, "Compaction isn't available on this platform");

+        }

+#endif

+

         reason |= GPR_FLAG_COMPACT;

     } else {

         if (!RTEST(full_mark))       reason &= ~GPR_FLAG_FULL_MARK;

@@ -9944,16 +9981,14 @@ gc_disable(rb_execution_context_t *ec, VALUE _)

 static VALUE

 gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v)

 {

-#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)

-    /* If Ruby's heap pages are not a multiple of the system page size, we

-     * cannot use mprotect for the read barrier, so we must disable automatic

-     * compaction. */

-    int pagesize;

-    pagesize = (int)sysconf(_SC_PAGE_SIZE);

-    if ((HEAP_PAGE_SIZE % pagesize) != 0) {

+    /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for

+     * the read barrier, so we must disable automatic compaction. */

+#if !defined(__MINGW32__) && !defined(_WIN32)

+    if (!USE_MMAP_ALIGNED_ALLOC) {

         rb_raise(rb_eNotImpError, "Automatic compaction isn't available on this platform");

     }

 #endif

+

     ruby_enable_autocompact = RTEST(v);

     return v;

 }

@@ -10350,22 +10385,54 @@ rb_aligned_malloc(size_t alignment, size_t size)

 #elif defined _WIN32

     void *_aligned_malloc(size_t, size_t);

     res = _aligned_malloc(size, alignment);

-#elif defined(HAVE_POSIX_MEMALIGN)

-    if (posix_memalign(&res, alignment, size) == 0) {

-        return res;

+#else

+    if (USE_MMAP_ALIGNED_ALLOC) {

+        GC_ASSERT(alignment % sysconf(_SC_PAGE_SIZE) == 0);

+

+        char *ptr = mmap(NULL, alignment + size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

+        if (ptr == MAP_FAILED) {

+            return NULL;

+        }

+

+        char *aligned = ptr + alignment;

+        aligned -= ((VALUE)aligned & (alignment - 1));

+        GC_ASSERT(aligned > ptr);

+        GC_ASSERT(aligned <= ptr + alignment);

+

+        size_t start_out_of_range_size = aligned - ptr;

+        GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);

+        if (start_out_of_range_size > 0) {

+            if (munmap(ptr, start_out_of_range_size)) {

+                rb_bug("rb_aligned_malloc: munmap failed for start");

+            }

+        }

+

+        size_t end_out_of_range_size = alignment - start_out_of_range_size;

+        GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);

+        if (end_out_of_range_size > 0) {

+            if (munmap(aligned + size, end_out_of_range_size)) {

+                rb_bug("rb_aligned_malloc: munmap failed for end");

+            }

+        }

+

+        res = (void *)aligned;

     }

     else {

-        return NULL;

+# if defined(HAVE_POSIX_MEMALIGN)

+        if (posix_memalign(&res, alignment, size) != 0) {

+            return NULL;

+        }

+# elif defined(HAVE_MEMALIGN)

+        res = memalign(alignment, size);

+# else

+        char* aligned;

+        res = malloc(alignment + size + sizeof(void*));

+        aligned = (char*)res + alignment + sizeof(void*);

+        aligned -= ((VALUE)aligned & (alignment - 1));

+        ((void**)aligned)[-1] = res;

+        res = (void*)aligned;

+# endif

     }

-#elif defined(HAVE_MEMALIGN)

-    res = memalign(alignment, size);

-#else

-    char* aligned;

-    res = malloc(alignment + size + sizeof(void*));

-    aligned = (char*)res + alignment + sizeof(void*);

-    aligned -= ((VALUE)aligned & (alignment - 1));

-    ((void**)aligned)[-1] = res;

-    res = (void*)aligned;

 #endif

     /* alignment must be a power of 2 */

@@ -10375,16 +10442,26 @@ rb_aligned_malloc(size_t alignment, size_t size)

 }

 static void

-rb_aligned_free(void *ptr)

+rb_aligned_free(void *ptr, size_t size)

 {

 #if defined __MINGW32__

     __mingw_aligned_free(ptr);

 #elif defined _WIN32

     _aligned_free(ptr);

-#elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN)

-    free(ptr);

 #else

-    free(((void**)ptr)[-1]);

+    if (USE_MMAP_ALIGNED_ALLOC) {

+        GC_ASSERT(size % sysconf(_SC_PAGE_SIZE) == 0);

+        if (munmap(ptr, size)) {

+            rb_bug("rb_aligned_free: munmap failed");

+        }

+    }

+    else {

+# if defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)

+        free(ptr);

+# else

+        free(((void**)ptr)[-1]);

+# endif

+    }

 #endif

 }

diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb

index 4a8cff33f4..f5cab55ba7 100644

--- a/test/ruby/test_gc_compact.rb

+++ b/test/ruby/test_gc_compact.rb

@@ -4,12 +4,32 @@

 require 'etc'

 class TestGCCompact < Test::Unit::TestCase

-  class AutoCompact < Test::Unit::TestCase

+  module SupportsCompact

     def setup

       skip "autocompact not supported on this platform" unless supports_auto_compact?

       super

     end

+    private

+

+    def supports_auto_compact?

+      return true unless defined?(Etc::SC_PAGE_SIZE)

+

+      begin

+        return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0

+      rescue NotImplementedError

+      rescue ArgumentError

+      end

+

+      true

+    end

+  end

+

+  include SupportsCompact

+

+  class AutoCompact < Test::Unit::TestCase

+    include SupportsCompact

+

     def test_enable_autocompact

       before = GC.auto_compact

       GC.auto_compact = true

@@ -59,26 +79,17 @@ def test_implicit_compaction_does_something

     ensure

       GC.auto_compact = before

     end

-

-    private

-

-    def supports_auto_compact?

-      return true unless defined?(Etc::SC_PAGE_SIZE)

-

-      begin

-        return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0

-      rescue NotImplementedError

-      rescue ArgumentError

-      end

-

-      true

-    end

   end

   def os_page_size

     return true unless defined?(Etc::SC_PAGE_SIZE)

   end

+  def setup

+    skip "autocompact not supported on this platform" unless supports_auto_compact?

+    super

+  end

+

   def test_gc_compact_stats

     list = []

-- 

2.30.1 (Apple Git-130)