commit bae7c7c764413b23e61cb099ce33be4c4ee259bb
Author: Florian Weimer <fweimer@redhat.com>
Date:   Thu Jan 28 13:59:11 2016 +0100

    Improve check against integer wraparound in hcreate_r [BZ #18240]

commit 2f5c1750558fe64bac361f52d6827ab1bcfe52bc
Author: Ondřej Bílka <neleai@seznam.cz>
Date:   Sat Jul 11 17:44:10 2015 +0200

    Handle overflow in __hcreate_r

--- glibc-2.17-c758a686/misc/hsearch_r.c
+++ glibc-2.17-c758a686/misc/hsearch_r.c
@@ -20,7 +20,7 @@
 #include <errno.h>
 #include <malloc.h>
 #include <string.h>
-
+#include <stdint.h>
 #include <search.h>
 
 /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986
@@ -47,12 +47,10 @@
 isprime (unsigned int number)
 {
   /* no even number will be passed */
-  unsigned int div = 3;
-
-  while (div * div < number && number % div != 0)
-    div += 2;
-
-  return number % div != 0;
+  for (unsigned int div = 3; div <= number / div; div += 2)
+    if (number % div == 0)
+      return 0;
+  return 1;
 }
 
 
@@ -74,6 +72,12 @@
       return 0;
     }
 
+  if (nel >= SIZE_MAX / sizeof (_ENTRY))
+    {
+      __set_errno (ENOMEM);
+      return 0;
+    }
+
   /* There is still another table active. Return with error. */
   if (htab->table != NULL)
     return 0;
@@ -82,10 +86,19 @@
      use will not work.  */
   if (nel < 3)
     nel = 3;
-  /* Change nel to the first prime number not smaller as nel. */
-  nel |= 1;      /* make odd */
-  while (!isprime (nel))
-    nel += 2;
+
+  /* Change nel to the first prime number in the range [nel, UINT_MAX - 2],
+     The '- 2' means 'nel += 2' cannot overflow.  */
+  for (nel |= 1; ; nel += 2)
+    {
+      if (UINT_MAX - 2 < nel)
+        {
+          __set_errno (ENOMEM);
+          return 0;
+        }
+      if (isprime (nel))
+        break;
+    }
 
   htab->size = nel;
   htab->filled = 0;
--- /dev/null
+++ glibc-2.17-c758a686/misc/bug18240.c
@@ -0,0 +1,97 @@
+/* Test integer wraparound in hcreate.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <limits.h>
+#include <search.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/resource.h>
+
+static void
+test_size (size_t size)
+{
+  int res = hcreate (size);
+  if (res == 0)
+    {
+      if (errno == ENOMEM)
+        return;
+      printf ("error: hcreate (%zu): %m\n", size);
+      exit (1);
+    }
+  char *keys[100];
+  for (int i = 0; i < 100; ++i)
+    {
+      if (asprintf (keys + i, "%d", i) < 0)
+        {
+          printf ("error: asprintf: %m\n");
+          exit (1);
+        }
+      ENTRY e = { keys[i], (char *) "value" };
+      if (hsearch (e, ENTER) == NULL)
+        {
+          printf ("error: hsearch (\"%s\"): %m\n", keys[i]);
+          exit (1);
+        }
+    }
+  hdestroy ();
+
+  for (int i = 0; i < 100; ++i)
+    free (keys[i]);
+}
+
+static int
+do_test (void)
+{
+  /* Limit the size of the process, so that memory allocation will
+     fail without impacting the entire system.  */
+  {
+    struct rlimit limit;
+    if (getrlimit (RLIMIT_AS, &limit) != 0)
+      {
+        printf ("getrlimit (RLIMIT_AS) failed: %m\n");
+        return 1;
+      }
+    long target = 100 * 1024 * 1024;
+    if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > target)
+      {
+        limit.rlim_cur = target;
+        if (setrlimit (RLIMIT_AS, &limit) != 0)
+          {
+            printf ("setrlimit (RLIMIT_AS) failed: %m\n");
+            return 1;
+          }
+      }
+  }
+
+  test_size (500);
+  test_size (-1);
+  test_size (-3);
+  test_size (INT_MAX - 2);
+  test_size (INT_MAX - 1);
+  test_size (INT_MAX);
+  test_size (((unsigned) INT_MAX) + 1);
+  test_size (UINT_MAX - 2);
+  test_size (UINT_MAX - 1);
+  test_size (UINT_MAX);
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"
--- glibc-2.17-c758a686/misc/Makefile
+++ glibc-2.17-c758a686/misc/Makefile
@@ -76,7 +76,7 @@
 gpl2lgpl := error.c error.h
 
 tests := tst-dirname tst-tsearch tst-fdset tst-efgcvt tst-mntent tst-hsearch \
-	 tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1
+	 tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1 bug18240
 ifeq ($(run-built-tests),yes)
 tests: $(objpfx)tst-error1-mem
 endif