From bcab8c3cd877506de75f50e0f9ed98827ed554b0 Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Tue, 23 Feb 2021 16:28:56 -0500
Subject: [PATCH] Use mmap for allocating heap pages
---
configure.ac | 16 ++++
gc.c | 149 ++++++++++++++++++++++++++---------
test/ruby/test_gc_compact.rb | 41 ++++++----
3 files changed, 155 insertions(+), 51 deletions(-)
diff --git a/configure.ac b/configure.ac
index 2dcebdde9f..b1b190004d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1944,6 +1944,7 @@ AC_CHECK_FUNCS(memmem)
AC_CHECK_FUNCS(mkfifo)
AC_CHECK_FUNCS(mknod)
AC_CHECK_FUNCS(mktime)
+AC_CHECK_FUNCS(mmap)
AC_CHECK_FUNCS(openat)
AC_CHECK_FUNCS(pipe2)
AC_CHECK_FUNCS(poll)
@@ -2666,6 +2667,21 @@ main(int argc, char *argv[])
rb_cv_fork_with_pthread=yes)])
test x$rb_cv_fork_with_pthread = xyes || AC_DEFINE(CANNOT_FORK_WITH_PTHREAD)
])
+
+AC_CHECK_HEADERS([sys/user.h])
+AS_IF([test "x$ac_cv_func_mmap" = xyes], [
+ AC_CACHE_CHECK([whether PAGE_SIZE is compile-time const], rb_cv_const_page_size,
+ [malloc_headers=`sed -n '/MALLOC_HEADERS_BEGIN/,/MALLOC_HEADERS_END/p' ${srcdir}/gc.c`
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$malloc_headers
+ typedef char conftest_page[PAGE_SIZE];
+ ]], [[]])],
+ [rb_cv_const_page_size=yes],
+ [rb_cv_const_page_size=no])])
+])
+AS_IF([test "x$rb_cv_const_page_size" = xyes],
+ [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 1)],
+ [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 0)]
+)
}
: "runtime section" && {
diff --git a/gc.c b/gc.c
index f6acf3e117..6f8e5f242d 100644
--- a/gc.c
+++ b/gc.c
@@ -32,6 +32,7 @@
#include <stdarg.h>
#include <stdio.h>
+/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
# define HAVE_MALLOC_USABLE_SIZE
@@ -54,6 +55,12 @@
# endif
#endif
+#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
+/* LIST_HEAD conflicts with sys/queue.h on macOS */
+# include <sys/user.h>
+#endif
+/* MALLOC_HEADERS_END */
+
#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
@@ -821,6 +828,25 @@ enum {
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
HEAP_PAGE_BITMAP_PLANES = 4 /* RGENGC: mark, unprotected, uncollectible, marking */
};
+#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
+#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
+
+#ifdef HAVE_MMAP
+# if HAVE_CONST_PAGE_SIZE
+/* If we have the HEAP_PAGE and it is a constant, then we can directly use it. */
+static const bool USE_MMAP_ALIGNED_ALLOC = (PAGE_SIZE <= HEAP_PAGE_SIZE);
+# elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
+/* PAGE_SIZE <= HEAP_PAGE_SIZE */
+static const bool USE_MMAP_ALIGNED_ALLOC = true;
+# else
+/* Otherwise, fall back to determining if we can use mmap during runtime. */
+# define USE_MMAP_ALIGNED_ALLOC (use_mmap_aligned_alloc != false)
+
+static bool use_mmap_aligned_alloc;
+# endif
+#elif !defined(__MINGW32__) && !defined(_WIN32)
+static const bool USE_MMAP_ALIGNED_ALLOC = false;
+#endif
struct heap_page {
short total_slots;
@@ -1760,14 +1786,14 @@ heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *pag
heap->total_slots -= page->total_slots;
}
-static void rb_aligned_free(void *ptr);
+static void rb_aligned_free(void *ptr, size_t size);
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
- rb_aligned_free(GET_PAGE_BODY(page->start));
+ rb_aligned_free(GET_PAGE_BODY(page->start), HEAP_PAGE_SIZE);
free(page);
}
@@ -1819,7 +1845,7 @@ heap_page_allocate(rb_objspace_t *objspace)
/* assign heap_page entry */
page = calloc1(sizeof(struct heap_page));
if (page == 0) {
- rb_aligned_free(page_body);
+ rb_aligned_free(page_body, HEAP_PAGE_SIZE);
rb_memerror();
}
@@ -3159,15 +3185,18 @@ Init_heap(void)
{
rb_objspace_t *objspace = &rb_objspace;
-#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
- /* If Ruby's heap pages are not a multiple of the system page size, we
- * cannot use mprotect for the read barrier, so we must disable automatic
- * compaction. */
- int pagesize;
- pagesize = (int)sysconf(_SC_PAGE_SIZE);
- if ((HEAP_PAGE_SIZE % pagesize) != 0) {
- ruby_enable_autocompact = 0;
- }
+#if defined(HAVE_MMAP) && !HAVE_CONST_PAGE_SIZE && !defined(PAGE_MAX_SIZE)
+ /* Need to determine if we can use mmap at runtime. */
+# ifdef PAGE_SIZE
+ /* If the PAGE_SIZE macro can be used. */
+ use_mmap_aligned_alloc = PAGE_SIZE <= HEAP_PAGE_SIZE;
+# elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
+ /* If we can use sysconf to determine the page size. */
+ use_mmap_aligned_alloc = sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE;
+# else
+ /* Otherwise we can't determine the system page size, so don't use mmap. */
+ use_mmap_aligned_alloc = FALSE;
+# endif
#endif
objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL);
@@ -8533,6 +8562,14 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE
/* For now, compact implies full mark / sweep, so ignore other flags */
if (RTEST(compact)) {
+ /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
+ * the read barrier, so we must disable compaction. */
+#if !defined(__MINGW32__) && !defined(_WIN32)
+ if (!USE_MMAP_ALIGNED_ALLOC) {
+ rb_raise(rb_eNotImpError, "Compaction isn't available on this platform");
+ }
+#endif
+
reason |= GPR_FLAG_COMPACT;
} else {
if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
@@ -9944,16 +9981,14 @@ gc_disable(rb_execution_context_t *ec, VALUE _)
static VALUE
gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v)
{
-#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
- /* If Ruby's heap pages are not a multiple of the system page size, we
- * cannot use mprotect for the read barrier, so we must disable automatic
- * compaction. */
- int pagesize;
- pagesize = (int)sysconf(_SC_PAGE_SIZE);
- if ((HEAP_PAGE_SIZE % pagesize) != 0) {
+ /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
+ * the read barrier, so we must disable automatic compaction. */
+#if !defined(__MINGW32__) && !defined(_WIN32)
+ if (!USE_MMAP_ALIGNED_ALLOC) {
rb_raise(rb_eNotImpError, "Automatic compaction isn't available on this platform");
}
#endif
+
ruby_enable_autocompact = RTEST(v);
return v;
}
@@ -10350,22 +10385,54 @@ rb_aligned_malloc(size_t alignment, size_t size)
#elif defined _WIN32
void *_aligned_malloc(size_t, size_t);
res = _aligned_malloc(size, alignment);
-#elif defined(HAVE_POSIX_MEMALIGN)
- if (posix_memalign(&res, alignment, size) == 0) {
- return res;
+#else
+ if (USE_MMAP_ALIGNED_ALLOC) {
+ GC_ASSERT(alignment % sysconf(_SC_PAGE_SIZE) == 0);
+
+ char *ptr = mmap(NULL, alignment + size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (ptr == MAP_FAILED) {
+ return NULL;
+ }
+
+ char *aligned = ptr + alignment;
+ aligned -= ((VALUE)aligned & (alignment - 1));
+ GC_ASSERT(aligned > ptr);
+ GC_ASSERT(aligned <= ptr + alignment);
+
+ size_t start_out_of_range_size = aligned - ptr;
+ GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
+ if (start_out_of_range_size > 0) {
+ if (munmap(ptr, start_out_of_range_size)) {
+ rb_bug("rb_aligned_malloc: munmap failed for start");
+ }
+ }
+
+ size_t end_out_of_range_size = alignment - start_out_of_range_size;
+ GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
+ if (end_out_of_range_size > 0) {
+ if (munmap(aligned + size, end_out_of_range_size)) {
+ rb_bug("rb_aligned_malloc: munmap failed for end");
+ }
+ }
+
+ res = (void *)aligned;
}
else {
- return NULL;
+# if defined(HAVE_POSIX_MEMALIGN)
+ if (posix_memalign(&res, alignment, size) != 0) {
+ return NULL;
+ }
+# elif defined(HAVE_MEMALIGN)
+ res = memalign(alignment, size);
+# else
+ char* aligned;
+ res = malloc(alignment + size + sizeof(void*));
+ aligned = (char*)res + alignment + sizeof(void*);
+ aligned -= ((VALUE)aligned & (alignment - 1));
+ ((void**)aligned)[-1] = res;
+ res = (void*)aligned;
+# endif
}
-#elif defined(HAVE_MEMALIGN)
- res = memalign(alignment, size);
-#else
- char* aligned;
- res = malloc(alignment + size + sizeof(void*));
- aligned = (char*)res + alignment + sizeof(void*);
- aligned -= ((VALUE)aligned & (alignment - 1));
- ((void**)aligned)[-1] = res;
- res = (void*)aligned;
#endif
/* alignment must be a power of 2 */
@@ -10375,16 +10442,26 @@ rb_aligned_malloc(size_t alignment, size_t size)
}
static void
-rb_aligned_free(void *ptr)
+rb_aligned_free(void *ptr, size_t size)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
#elif defined _WIN32
_aligned_free(ptr);
-#elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN)
- free(ptr);
#else
- free(((void**)ptr)[-1]);
+ if (USE_MMAP_ALIGNED_ALLOC) {
+ GC_ASSERT(size % sysconf(_SC_PAGE_SIZE) == 0);
+ if (munmap(ptr, size)) {
+ rb_bug("rb_aligned_free: munmap failed");
+ }
+ }
+ else {
+# if defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
+ free(ptr);
+# else
+ free(((void**)ptr)[-1]);
+# endif
+ }
#endif
}
diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb
index 4a8cff33f4..f5cab55ba7 100644
--- a/test/ruby/test_gc_compact.rb
+++ b/test/ruby/test_gc_compact.rb
@@ -4,12 +4,32 @@
require 'etc'
class TestGCCompact < Test::Unit::TestCase
- class AutoCompact < Test::Unit::TestCase
+ module SupportsCompact
def setup
skip "autocompact not supported on this platform" unless supports_auto_compact?
super
end
+ private
+
+ def supports_auto_compact?
+ return true unless defined?(Etc::SC_PAGE_SIZE)
+
+ begin
+ return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0
+ rescue NotImplementedError
+ rescue ArgumentError
+ end
+
+ true
+ end
+ end
+
+ include SupportsCompact
+
+ class AutoCompact < Test::Unit::TestCase
+ include SupportsCompact
+
def test_enable_autocompact
before = GC.auto_compact
GC.auto_compact = true
@@ -59,26 +79,17 @@ def test_implicit_compaction_does_something
ensure
GC.auto_compact = before
end
-
- private
-
- def supports_auto_compact?
- return true unless defined?(Etc::SC_PAGE_SIZE)
-
- begin
- return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0
- rescue NotImplementedError
- rescue ArgumentError
- end
-
- true
- end
end
def os_page_size
return true unless defined?(Etc::SC_PAGE_SIZE)
end
+ def setup
+ skip "autocompact not supported on this platform" unless supports_auto_compact?
+ super
+ end
+
def test_gc_compact_stats
list = []
--
2.30.1 (Apple Git-130)