commit bae7c7c764413b23e61cb099ce33be4c4ee259bb

Author: Florian Weimer <fweimer@redhat.com>

Date:   Thu Jan 28 13:59:11 2016 +0100

    Improve check against integer wraparound in hcreate_r [BZ #18240]

commit 2f5c1750558fe64bac361f52d6827ab1bcfe52bc

Author: Ondřej Bílka <neleai@seznam.cz>

Date:   Sat Jul 11 17:44:10 2015 +0200

    Handle overflow in __hcreate_r

--- glibc-2.17-c758a686/misc/hsearch_r.c

+++ glibc-2.17-c758a686/misc/hsearch_r.c

@@ -20,7 +20,7 @@

 #include <errno.h>

 #include <malloc.h>

 #include <string.h>

-

+#include <stdint.h>

 #include <search.h>

 /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986

@@ -47,12 +47,10 @@

 isprime (unsigned int number)

 {

   /* no even number will be passed */

-  unsigned int div = 3;

-

-  while (div * div < number && number % div != 0)

-    div += 2;

-

-  return number % div != 0;

+  for (unsigned int div = 3; div <= number / div; div += 2)

+    if (number % div == 0)

+      return 0;

+  return 1;

 }

@@ -74,6 +72,12 @@

       return 0;

     }

+  if (nel >= SIZE_MAX / sizeof (_ENTRY))

+    {

+      __set_errno (ENOMEM);

+      return 0;

+    }

+

   /* There is still another table active. Return with error. */

   if (htab->table != NULL)

     return 0;

@@ -82,10 +86,19 @@

      use will not work.  */

   if (nel < 3)

     nel = 3;

-  /* Change nel to the first prime number not smaller as nel. */

-  nel |= 1;      /* make odd */

-  while (!isprime (nel))

-    nel += 2;

+

+  /* Change nel to the first prime number in the range [nel, UINT_MAX - 2],

+     The '- 2' means 'nel += 2' cannot overflow.  */

+  for (nel |= 1; ; nel += 2)

+    {

+      if (UINT_MAX - 2 < nel)

+        {

+          __set_errno (ENOMEM);

+          return 0;

+        }

+      if (isprime (nel))

+        break;

+    }

   htab->size = nel;

   htab->filled = 0;

--- /dev/null

+++ glibc-2.17-c758a686/misc/bug18240.c

@@ -0,0 +1,97 @@

+/* Test integer wraparound in hcreate.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <errno.h>

+#include <limits.h>

+#include <search.h>

+#include <stdbool.h>

+#include <stdio.h>

+#include <stdlib.h>

+#include <sys/resource.h>

+

+static void

+test_size (size_t size)

+{

+  int res = hcreate (size);

+  if (res == 0)

+    {

+      if (errno == ENOMEM)

+        return;

+      printf ("error: hcreate (%zu): %m\n", size);

+      exit (1);

+    }

+  char *keys[100];

+  for (int i = 0; i < 100; ++i)

+    {

+      if (asprintf (keys + i, "%d", i) < 0)

+        {

+          printf ("error: asprintf: %m\n");

+          exit (1);

+        }

+      ENTRY e = { keys[i], (char *) "value" };

+      if (hsearch (e, ENTER) == NULL)

+        {

+          printf ("error: hsearch (\"%s\"): %m\n", keys[i]);

+          exit (1);

+        }

+    }

+  hdestroy ();

+

+  for (int i = 0; i < 100; ++i)

+    free (keys[i]);

+}

+

+static int

+do_test (void)

+{

+  /* Limit the size of the process, so that memory allocation will

+     fail without impacting the entire system.  */

+  {

+    struct rlimit limit;

+    if (getrlimit (RLIMIT_AS, &limit) != 0)

+      {

+        printf ("getrlimit (RLIMIT_AS) failed: %m\n");

+        return 1;

+      }

+    long target = 100 * 1024 * 1024;

+    if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > target)

+      {

+        limit.rlim_cur = target;

+        if (setrlimit (RLIMIT_AS, &limit) != 0)

+          {

+            printf ("setrlimit (RLIMIT_AS) failed: %m\n");

+            return 1;

+          }

+      }

+  }

+

+  test_size (500);

+  test_size (-1);

+  test_size (-3);

+  test_size (INT_MAX - 2);

+  test_size (INT_MAX - 1);

+  test_size (INT_MAX);

+  test_size (((unsigned) INT_MAX) + 1);

+  test_size (UINT_MAX - 2);

+  test_size (UINT_MAX - 1);

+  test_size (UINT_MAX);

+  return 0;

+}

+

+#define TEST_FUNCTION do_test ()

+#include "../test-skeleton.c"

--- glibc-2.17-c758a686/misc/Makefile

+++ glibc-2.17-c758a686/misc/Makefile

@@ -76,7 +76,7 @@

 gpl2lgpl := error.c error.h

 tests := tst-dirname tst-tsearch tst-fdset tst-efgcvt tst-mntent tst-hsearch \

-	 tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1

+	 tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1 bug18240

 ifeq ($(run-built-tests),yes)

 tests: $(objpfx)tst-error1-mem

 endif