commit 9333498794cde1d5cca518badf79533a24114b6f

Author: Joseph Myers <joseph@codesourcery.com>

Date:   Wed Feb 12 23:31:56 2020 +0000

    Avoid ldbl-96 stack corruption from range reduction of pseudo-zero (bug 25487).

    Bug 25487 reports stack corruption in ldbl-96 sinl on a pseudo-zero

    argument (an representation where all the significand bits, including

    the explicit high bit, are zero, but the exponent is not zero, which

    is not a valid representation for the long double type).

    Although this is not a valid long double representation, existing

    practice in this area (see bug 4586, originally marked invalid but

    subsequently fixed) is that we still seek to avoid invalid memory

    accesses as a result, in case of programs that treat arbitrary binary

    data as long double representations, although the invalid

    representations of the ldbl-96 format do not need to be consistently

    handled the same as any particular valid representation.

    This patch makes the range reduction detect pseudo-zero and unnormal

    representations that would otherwise go to __kernel_rem_pio2, and

    returns a NaN for them instead of continuing with the range reduction

    process.  (Pseudo-zero and unnormal representations whose unbiased

    exponent is less than -1 have already been safely returned from the

    function before this point without going through the rest of range

    reduction.)  Pseudo-zero representations would previously result in

    the value passed to __kernel_rem_pio2 being all-zero, which is

    definitely unsafe; unnormal representations would previously result in

    a value passed whose high bit is zero, which might well be unsafe

    since that is not a form of input expected by __kernel_rem_pio2.

    Tested for x86_64.

Revised for RHEL 8.3.0.

diff -Nrup a/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c b/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c

--- a/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c	2018-08-01 01:10:47.000000000 -0400

+++ b/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c	2020-04-03 13:05:02.609844427 -0400

@@ -209,6 +209,18 @@ __ieee754_rem_pio2l (long double x, long

       y[1] = y[0];

       return 0;

     }

+  

+  if ((i0 & 0x80000000) == 0)

+    {

+      /* Pseudo-zero and unnormal representations are not valid

+        representations of long double.  We need to avoid stack

+        corruption in __kernel_rem_pio2, which expects input in a

+        particular normal form, but those representations do not need

+        to be consistently handled like any particular floating-point

+        value.  */

+      y[1] = y[0] = __builtin_nanl ("");

+      return 0;

+    }

   /* Split the 64 bits of the mantissa into three 24-bit integers

      stored in a double array.  */

diff -Nrup a/sysdeps/ieee754/ldbl-96/Makefile b/sysdeps/ieee754/ldbl-96/Makefile

--- a/sysdeps/ieee754/ldbl-96/Makefile	2018-08-01 01:10:47.000000000 -0400

+++ b/sysdeps/ieee754/ldbl-96/Makefile	2020-04-03 13:03:20.233546734 -0400

@@ -17,5 +17,6 @@

 # <http://www.gnu.org/licenses/>.

 ifeq ($(subdir),math)

-tests += test-canonical-ldbl-96 test-totalorderl-ldbl-96

+tests += test-canonical-ldbl-96 test-totalorderl-ldbl-96 test-sinl-pseudo

+CFLAGS-test-sinl-pseudo.c += -fstack-protector-all

 endif

diff -Nrup a/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c b/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c

--- a/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c	1969-12-31 19:00:00.000000000 -0500

+++ b/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c	2020-04-03 13:05:37.857952212 -0400

@@ -0,0 +1,41 @@

+/* Test sinl for pseudo-zeros and unnormals for ldbl-96 (bug 25487).

+   Copyright (C) 2020 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <https://www.gnu.org/licenses/>.  */

+

+#include <math.h>

+#include <math_ldbl.h>

+#include <stdint.h>

+

+static int

+do_test (void)

+{

+  for (int i = 0; i < 64; i++)

+    {

+      uint64_t sig = i == 63 ? 0 : 1ULL << i;

+      long double ld;

+      SET_LDOUBLE_WORDS (ld, 0x4141,

+                        sig >> 32, sig & 0xffffffffULL);

+      /* The requirement is that no stack overflow occurs when the

+        pseudo-zero or unnormal goes through range reduction.  */

+      volatile long double ldr;

+      ldr = sinl (ld);

+      (void) ldr;

+    }

+  return 0;

+}

+

+#include <support/test-driver.c>