commit e6d90d675d4cae810be76a5ff41b8ae8bd6bc914

Author: Wilco Dijkstra <wdijkstr@arm.com>

Date:   Mon Jun 23 17:15:41 2014 +0100

    Add generic HAVE_RM_CTX implementation

    This patch adds a generic implementation of HAVE_RM_CTX using standard

    fenv calls. As a result math functions using SET_RESTORE_ROUND* macros

    do not suffer from a large slowdown on targets which do not implement

    optimized libc_fe*_ctx inline functions. Most of the libc_fe* inline

    functions are now unused and could be removed in the future (there are

    a few math functions left which use a mixture of standard fenv calls

    and libc_fe* inline functions - they could be updated to use

    SET_RESTORE_ROUND or improved to avoid expensive fenv manipulations

    across just a few FP instructions).

    libc_feholdsetround*_noex_ctx is added to enable better optimization of

    SET_RESTORE_ROUND_NOEX* implementations.

    Performance measurements on ARM and x86 of sin() show significant gains

    over the current default, fairly close to a highly optimized fenv_private:

                            ARM   x86

    no fenv_private      : 100%  100%

    generic HAVE_RM_CTX  : 250%  350%

    fenv_private (CTX)   : 250%  450%

    2014-06-23  Will Newton  <will.newton@linaro.org>

    	    Wilco  <wdijkstr@arm.com>

    	* sysdeps/generic/math_private.h: Add generic HAVE_RM_CTX

    	implementation.  Include get-rounding-mode.h.

    	[!HAVE_RM_CTX]: Define HAVE_RM_CTX to zero.

    	[!libc_feholdsetround_noex_ctx]: Define

    	libc_feholdsetround_noex_ctx.

    	[!libc_feholdsetround_noexf_ctx]: Define

    	libc_feholdsetround_noexf_ctx.

    	[!libc_feholdsetround_noexl_ctx]: Define

    	libc_feholdsetround_noexl_ctx.

    	(libc_feholdsetround_ctx): New function.

    	(libc_feresetround_ctx): New function.

    	(libc_feholdsetround_noex_ctx): New function.

    	(libc_feresetround_noex_ctx): New function.

diff --git a/sysdeps/generic/math_private.h b/sysdeps/generic/math_private.h

index 9b881a3..94c1e4a 100644

--- a/sysdeps/generic/math_private.h

+++ b/sysdeps/generic/math_private.h

@@ -20,6 +20,7 @@

 #include <stdint.h>

 #include <sys/types.h>

 #include <fenv.h>

+#include <get-rounding-mode.h>

 /* The original fdlibm code used statements like:

 	n0 = ((*(int*)&one)>>29)^1;		* index of high word *

@@ -551,12 +552,26 @@ default_libc_feupdateenv_test (fenv_t *e, int ex)

 # define libc_feresetround_noexl libc_fesetenvl

 #endif

+#ifndef HAVE_RM_CTX

+# define HAVE_RM_CTX 0

+#endif

+

 #if HAVE_RM_CTX

 /* Set/Restore Rounding Modes only when necessary.  If defined, these functions

    set/restore floating point state only if the state needed within the lexical

    block is different from the current state.  This saves a lot of time when

    the floating point unit is much slower than the fixed point units.  */

+# ifndef libc_feholdsetround_noex_ctx

+#   define libc_feholdsetround_noex_ctx  libc_feholdsetround_ctx

+# endif

+# ifndef libc_feholdsetround_noexf_ctx

+#   define libc_feholdsetround_noexf_ctx libc_feholdsetroundf_ctx

+# endif

+# ifndef libc_feholdsetround_noexl_ctx

+#   define libc_feholdsetround_noexl_ctx libc_feholdsetroundl_ctx

+# endif

+

 # ifndef libc_feresetround_noex_ctx

 #   define libc_feresetround_noex_ctx  libc_fesetenv_ctx

 # endif

@@ -567,24 +582,80 @@ default_libc_feupdateenv_test (fenv_t *e, int ex)

 #   define libc_feresetround_noexl_ctx libc_fesetenvl_ctx

 # endif

-# ifndef libc_feholdsetround_53bit_ctx

-#   define libc_feholdsetround_53bit_ctx libc_feholdsetround_ctx

-# endif

+#else

-# ifndef libc_feresetround_53bit_ctx

-#   define libc_feresetround_53bit_ctx libc_feresetround_ctx

-# endif

+/* Default implementation using standard fenv functions.

+   Avoid unnecessary rounding mode changes by first checking the

+   current rounding mode.  Note the use of __glibc_unlikely is

+   important for performance.  */

-# define SET_RESTORE_ROUND_GENERIC(RM,ROUNDFUNC,CLEANUPFUNC) \

-  struct rm_ctx ctx __attribute__((cleanup(CLEANUPFUNC ## _ctx)));	      \

-  ROUNDFUNC ## _ctx (&ctx, (RM))

-#else

-# define SET_RESTORE_ROUND_GENERIC(RM, ROUNDFUNC, CLEANUPFUNC) \

-  fenv_t __libc_save_rm __attribute__((cleanup(CLEANUPFUNC)));	\

-  ROUNDFUNC (&__libc_save_rm, (RM))

+static __always_inline void

+libc_feholdsetround_ctx (struct rm_ctx *ctx, int round)

+{

+  ctx->updated_status = false;

+

+  /* Update rounding mode only if different.  */

+  if (__glibc_unlikely (round != get_rounding_mode ()))

+    {

+      ctx->updated_status = true;

+      fegetenv (&ctx->env);

+      fesetround (round);

+    }

+}

+

+static __always_inline void

+libc_feresetround_ctx (struct rm_ctx *ctx)

+{

+  /* Restore the rounding mode if updated.  */

+  if (__glibc_unlikely (ctx->updated_status))

+    feupdateenv (&ctx->env);

+}

+

+static __always_inline void

+libc_feholdsetround_noex_ctx (struct rm_ctx *ctx, int round)

+{

+  /* Save exception flags and rounding mode.  */

+  fegetenv (&ctx->env);

+

+  /* Update rounding mode only if different.  */

+  if (__glibc_unlikely (round != get_rounding_mode ()))

+    fesetround (round);

+}

+

+static __always_inline void

+libc_feresetround_noex_ctx (struct rm_ctx *ctx)

+{

+  /* Restore exception flags and rounding mode.  */

+  fesetenv (&ctx->env);

+}

+

+# define libc_feholdsetroundf_ctx libc_feholdsetround_ctx

+# define libc_feholdsetroundl_ctx libc_feholdsetround_ctx

+# define libc_feresetroundf_ctx   libc_feresetround_ctx

+# define libc_feresetroundl_ctx   libc_feresetround_ctx

+

+# define libc_feholdsetround_noexf_ctx libc_feholdsetround_noex_ctx

+# define libc_feholdsetround_noexl_ctx libc_feholdsetround_noex_ctx

+# define libc_feresetround_noexf_ctx   libc_feresetround_noex_ctx

+# define libc_feresetround_noexl_ctx   libc_feresetround_noex_ctx

+

+#endif

+

+#ifndef libc_feholdsetround_53bit_ctx

+#  define libc_feholdsetround_53bit_ctx libc_feholdsetround_ctx

+#endif

+#ifndef libc_feresetround_53bit_ctx

+#  define libc_feresetround_53bit_ctx libc_feresetround_ctx

 #endif

-/* Save and restore the rounding mode within a lexical block.  */

+#define SET_RESTORE_ROUND_GENERIC(RM,ROUNDFUNC,CLEANUPFUNC) \

+  struct rm_ctx ctx __attribute__((cleanup (CLEANUPFUNC ## _ctx))); \

+  ROUNDFUNC ## _ctx (&ctx, (RM))

+

+/* Set the rounding mode within a lexical block.  Restore the rounding mode to

+   the value at the start of the block.  The exception mode must be preserved.

+   Exceptions raised within the block must be set in the exception flags.

+   Non-stop mode may be enabled inside the block.  */

 #define SET_RESTORE_ROUND(RM) \

   SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround, libc_feresetround)

@@ -593,15 +664,21 @@ default_libc_feupdateenv_test (fenv_t *e, int ex)

 #define SET_RESTORE_ROUNDL(RM) \

   SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetroundl, libc_feresetroundl)

-/* Save and restore the rounding mode within a lexical block, and also

-   the set of exceptions raised within the block may be discarded.  */

+/* Set the rounding mode within a lexical block.  Restore the rounding mode to

+   the value at the start of the block.  The exception mode must be preserved.

+   Exceptions raised within the block must be discarded, and exception flags

+   are restored to the value at the start of the block.

+   Non-stop mode may be enabled inside the block.  */

 #define SET_RESTORE_ROUND_NOEX(RM) \

-  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround, libc_feresetround_noex)

+  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_noex, \

+			     libc_feresetround_noex)

 #define SET_RESTORE_ROUND_NOEXF(RM) \

-  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetroundf, libc_feresetround_noexf)

+  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_noexf, \

+			     libc_feresetround_noexf)

 #define SET_RESTORE_ROUND_NOEXL(RM) \

-  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetroundl, libc_feresetround_noexl)

+  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_noexl, \

+			     libc_feresetround_noexl)

 /* Like SET_RESTORE_ROUND, but also set rounding precision to 53 bits.  */

 #define SET_RESTORE_ROUND_53BIT(RM) \