bca718
commit e6d90d675d4cae810be76a5ff41b8ae8bd6bc914
bca718
Author: Wilco Dijkstra <wdijkstr@arm.com>
bca718
Date:   Mon Jun 23 17:15:41 2014 +0100
bca718
bca718
    Add generic HAVE_RM_CTX implementation
bca718
    
bca718
    This patch adds a generic implementation of HAVE_RM_CTX using standard
bca718
    fenv calls. As a result math functions using SET_RESTORE_ROUND* macros
bca718
    do not suffer from a large slowdown on targets which do not implement
bca718
    optimized libc_fe*_ctx inline functions. Most of the libc_fe* inline
bca718
    functions are now unused and could be removed in the future (there are
bca718
    a few math functions left which use a mixture of standard fenv calls
bca718
    and libc_fe* inline functions - they could be updated to use
bca718
    SET_RESTORE_ROUND or improved to avoid expensive fenv manipulations
bca718
    across just a few FP instructions).
bca718
    
bca718
    libc_feholdsetround*_noex_ctx is added to enable better optimization of
bca718
    SET_RESTORE_ROUND_NOEX* implementations.
bca718
    
bca718
    Performance measurements on ARM and x86 of sin() show significant gains
bca718
    over the current default, fairly close to a highly optimized fenv_private:
bca718
    
bca718
                            ARM   x86
bca718
    no fenv_private      : 100%  100%
bca718
    generic HAVE_RM_CTX  : 250%  350%
bca718
    fenv_private (CTX)   : 250%  450%
bca718
    
bca718
    2014-06-23  Will Newton  <will.newton@linaro.org>
bca718
    	    Wilco  <wdijkstr@arm.com>
bca718
    
bca718
    	* sysdeps/generic/math_private.h: Add generic HAVE_RM_CTX
bca718
    	implementation.  Include get-rounding-mode.h.
bca718
    	[!HAVE_RM_CTX]: Define HAVE_RM_CTX to zero.
bca718
    	[!libc_feholdsetround_noex_ctx]: Define
bca718
    	libc_feholdsetround_noex_ctx.
bca718
    	[!libc_feholdsetround_noexf_ctx]: Define
bca718
    	libc_feholdsetround_noexf_ctx.
bca718
    	[!libc_feholdsetround_noexl_ctx]: Define
bca718
    	libc_feholdsetround_noexl_ctx.
bca718
    	(libc_feholdsetround_ctx): New function.
bca718
    	(libc_feresetround_ctx): New function.
bca718
    	(libc_feholdsetround_noex_ctx): New function.
bca718
    	(libc_feresetround_noex_ctx): New function.
bca718
bca718
diff --git a/sysdeps/generic/math_private.h b/sysdeps/generic/math_private.h
bca718
index 9b881a3..94c1e4a 100644
bca718
--- a/sysdeps/generic/math_private.h
bca718
+++ b/sysdeps/generic/math_private.h
bca718
@@ -20,6 +20,7 @@
bca718
 #include <stdint.h>
bca718
 #include <sys/types.h>
bca718
 #include <fenv.h>
bca718
+#include <get-rounding-mode.h>
bca718
 
bca718
 /* The original fdlibm code used statements like:
bca718
 	n0 = ((*(int*)&one)>>29)^1;		* index of high word *
bca718
@@ -551,12 +552,26 @@ default_libc_feupdateenv_test (fenv_t *e, int ex)
bca718
 # define libc_feresetround_noexl libc_fesetenvl
bca718
 #endif
bca718
 
bca718
+#ifndef HAVE_RM_CTX
bca718
+# define HAVE_RM_CTX 0
bca718
+#endif
bca718
+
bca718
 #if HAVE_RM_CTX
bca718
 /* Set/Restore Rounding Modes only when necessary.  If defined, these functions
bca718
    set/restore floating point state only if the state needed within the lexical
bca718
    block is different from the current state.  This saves a lot of time when
bca718
    the floating point unit is much slower than the fixed point units.  */
bca718
 
bca718
+# ifndef libc_feholdsetround_noex_ctx
bca718
+#   define libc_feholdsetround_noex_ctx  libc_feholdsetround_ctx
bca718
+# endif
bca718
+# ifndef libc_feholdsetround_noexf_ctx
bca718
+#   define libc_feholdsetround_noexf_ctx libc_feholdsetroundf_ctx
bca718
+# endif
bca718
+# ifndef libc_feholdsetround_noexl_ctx
bca718
+#   define libc_feholdsetround_noexl_ctx libc_feholdsetroundl_ctx
bca718
+# endif
bca718
+
bca718
 # ifndef libc_feresetround_noex_ctx
bca718
 #   define libc_feresetround_noex_ctx  libc_fesetenv_ctx
bca718
 # endif
bca718
@@ -567,24 +582,80 @@ default_libc_feupdateenv_test (fenv_t *e, int ex)
bca718
 #   define libc_feresetround_noexl_ctx libc_fesetenvl_ctx
bca718
 # endif
bca718
 
bca718
-# ifndef libc_feholdsetround_53bit_ctx
bca718
-#   define libc_feholdsetround_53bit_ctx libc_feholdsetround_ctx
bca718
-# endif
bca718
+#else
bca718
 
bca718
-# ifndef libc_feresetround_53bit_ctx
bca718
-#   define libc_feresetround_53bit_ctx libc_feresetround_ctx
bca718
-# endif
bca718
+/* Default implementation using standard fenv functions.
bca718
+   Avoid unnecessary rounding mode changes by first checking the
bca718
+   current rounding mode.  Note the use of __glibc_unlikely is
bca718
+   important for performance.  */
bca718
 
bca718
-# define SET_RESTORE_ROUND_GENERIC(RM,ROUNDFUNC,CLEANUPFUNC) \
bca718
-  struct rm_ctx ctx __attribute__((cleanup(CLEANUPFUNC ## _ctx)));	      \
bca718
-  ROUNDFUNC ## _ctx (&ctx, (RM))
bca718
-#else
bca718
-# define SET_RESTORE_ROUND_GENERIC(RM, ROUNDFUNC, CLEANUPFUNC) \
bca718
-  fenv_t __libc_save_rm __attribute__((cleanup(CLEANUPFUNC)));	\
bca718
-  ROUNDFUNC (&__libc_save_rm, (RM))
bca718
+static __always_inline void
bca718
+libc_feholdsetround_ctx (struct rm_ctx *ctx, int round)
bca718
+{
bca718
+  ctx->updated_status = false;
bca718
+
bca718
+  /* Update rounding mode only if different.  */
bca718
+  if (__glibc_unlikely (round != get_rounding_mode ()))
bca718
+    {
bca718
+      ctx->updated_status = true;
bca718
+      fegetenv (&ctx->env);
bca718
+      fesetround (round);
bca718
+    }
bca718
+}
bca718
+
bca718
+static __always_inline void
bca718
+libc_feresetround_ctx (struct rm_ctx *ctx)
bca718
+{
bca718
+  /* Restore the rounding mode if updated.  */
bca718
+  if (__glibc_unlikely (ctx->updated_status))
bca718
+    feupdateenv (&ctx->env);
bca718
+}
bca718
+
bca718
+static __always_inline void
bca718
+libc_feholdsetround_noex_ctx (struct rm_ctx *ctx, int round)
bca718
+{
bca718
+  /* Save exception flags and rounding mode.  */
bca718
+  fegetenv (&ctx->env);
bca718
+
bca718
+  /* Update rounding mode only if different.  */
bca718
+  if (__glibc_unlikely (round != get_rounding_mode ()))
bca718
+    fesetround (round);
bca718
+}
bca718
+
bca718
+static __always_inline void
bca718
+libc_feresetround_noex_ctx (struct rm_ctx *ctx)
bca718
+{
bca718
+  /* Restore exception flags and rounding mode.  */
bca718
+  fesetenv (&ctx->env);
bca718
+}
bca718
+
bca718
+# define libc_feholdsetroundf_ctx libc_feholdsetround_ctx
bca718
+# define libc_feholdsetroundl_ctx libc_feholdsetround_ctx
bca718
+# define libc_feresetroundf_ctx   libc_feresetround_ctx
bca718
+# define libc_feresetroundl_ctx   libc_feresetround_ctx
bca718
+
bca718
+# define libc_feholdsetround_noexf_ctx libc_feholdsetround_noex_ctx
bca718
+# define libc_feholdsetround_noexl_ctx libc_feholdsetround_noex_ctx
bca718
+# define libc_feresetround_noexf_ctx   libc_feresetround_noex_ctx
bca718
+# define libc_feresetround_noexl_ctx   libc_feresetround_noex_ctx
bca718
+
bca718
+#endif
bca718
+
bca718
+#ifndef libc_feholdsetround_53bit_ctx
bca718
+#  define libc_feholdsetround_53bit_ctx libc_feholdsetround_ctx
bca718
+#endif
bca718
+#ifndef libc_feresetround_53bit_ctx
bca718
+#  define libc_feresetround_53bit_ctx libc_feresetround_ctx
bca718
 #endif
bca718
 
bca718
-/* Save and restore the rounding mode within a lexical block.  */
bca718
+#define SET_RESTORE_ROUND_GENERIC(RM,ROUNDFUNC,CLEANUPFUNC) \
bca718
+  struct rm_ctx ctx __attribute__((cleanup (CLEANUPFUNC ## _ctx))); \
bca718
+  ROUNDFUNC ## _ctx (&ctx, (RM))
bca718
+
bca718
+/* Set the rounding mode within a lexical block.  Restore the rounding mode to
bca718
+   the value at the start of the block.  The exception mode must be preserved.
bca718
+   Exceptions raised within the block must be set in the exception flags.
bca718
+   Non-stop mode may be enabled inside the block.  */
bca718
 
bca718
 #define SET_RESTORE_ROUND(RM) \
bca718
   SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround, libc_feresetround)
bca718
@@ -593,15 +664,21 @@ default_libc_feupdateenv_test (fenv_t *e, int ex)
bca718
 #define SET_RESTORE_ROUNDL(RM) \
bca718
   SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetroundl, libc_feresetroundl)
bca718
 
bca718
-/* Save and restore the rounding mode within a lexical block, and also
bca718
-   the set of exceptions raised within the block may be discarded.  */
bca718
+/* Set the rounding mode within a lexical block.  Restore the rounding mode to
bca718
+   the value at the start of the block.  The exception mode must be preserved.
bca718
+   Exceptions raised within the block must be discarded, and exception flags
bca718
+   are restored to the value at the start of the block.
bca718
+   Non-stop mode may be enabled inside the block.  */
bca718
 
bca718
 #define SET_RESTORE_ROUND_NOEX(RM) \
bca718
-  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround, libc_feresetround_noex)
bca718
+  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_noex, \
bca718
+			     libc_feresetround_noex)
bca718
 #define SET_RESTORE_ROUND_NOEXF(RM) \
bca718
-  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetroundf, libc_feresetround_noexf)
bca718
+  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_noexf, \
bca718
+			     libc_feresetround_noexf)
bca718
 #define SET_RESTORE_ROUND_NOEXL(RM) \
bca718
-  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetroundl, libc_feresetround_noexl)
bca718
+  SET_RESTORE_ROUND_GENERIC (RM, libc_feholdsetround_noexl, \
bca718
+			     libc_feresetround_noexl)
bca718
 
bca718
 /* Like SET_RESTORE_ROUND, but also set rounding precision to 53 bits.  */
bca718
 #define SET_RESTORE_ROUND_53BIT(RM) \