978e96
This patch is based on the below upstream commit.
978e96
File deletions were altered to reflect file renames.
978e96
978e96
commit f50277c19df0937ea9691ab7e7c642ecd3ed3d94
978e96
Author: Torvald Riegel <triegel@redhat.com>
978e96
Date:   Sun Oct 19 21:59:26 2014 +0200
978e96
978e96
    pthread_once: Add fast path and remove x86 variants.
978e96
978e96
diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c
978e96
index 595bd7e298003e00..2afb79c01fe0a61e 100644
978e96
--- a/nptl/pthread_once.c
978e96
+++ b/nptl/pthread_once.c
978e96
@@ -58,11 +58,13 @@ clear_once_control (void *arg)
978e96
    initialization is interrupted, we then fork 2^30 times (30 bits of
978e96
    once_control are used for the fork generation), and try to initialize
978e96
    again, we can deadlock because we can't distinguish the in-progress and
978e96
-   interrupted cases anymore.  */
978e96
-int
978e96
-__pthread_once (once_control, init_routine)
978e96
-     pthread_once_t *once_control;
978e96
-     void (*init_routine) (void);
978e96
+   interrupted cases anymore.
978e96
+   XXX: We split out this slow path because current compilers do not generate
978e96
+   as efficient code when the fast path in __pthread_once below is not in a
978e96
+   separate function.  */
978e96
+static int
978e96
+__attribute__ ((noinline))
978e96
+__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
978e96
 {
978e96
   while (1)
978e96
     {
978e96
@@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine)
978e96
          signals that initialization has finished, we need to see any
978e96
          data modifications done during initialization.  */
978e96
       val = *once_control;
978e96
-      atomic_read_barrier();
978e96
+      atomic_read_barrier ();
978e96
       do
978e96
 	{
978e96
 	  /* Check if the initialization has already been done.  */
978e96
@@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine)
978e96
 
978e96
   return 0;
978e96
 }
978e96
+
978e96
+int
978e96
+__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
978e96
+{
978e96
+  /* Fast path.  See __pthread_once_slow.  */
978e96
+  int val;
978e96
+  val = *once_control;
978e96
+  atomic_read_barrier ();
978e96
+  if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
978e96
+    return 0;
978e96
+  else
978e96
+    return __pthread_once_slow (once_control, init_routine);
978e96
+}
978e96
 weak_alias (__pthread_once, pthread_once)
978e96
 hidden_def (__pthread_once)
978e96
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
978e96
deleted file mode 100644
978e96
index ca3b860a7f6f95ae..0000000000000000
978e96
--- a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
978e96
+++ /dev/null
978e96
@@ -1,178 +0,0 @@
978e96
-/* Copyright (C) 2002-2012 Free Software Foundation, Inc.
978e96
-   This file is part of the GNU C Library.
978e96
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
978e96
-
978e96
-   The GNU C Library is free software; you can redistribute it and/or
978e96
-   modify it under the terms of the GNU Lesser General Public
978e96
-   License as published by the Free Software Foundation; either
978e96
-   version 2.1 of the License, or (at your option) any later version.
978e96
-
978e96
-   The GNU C Library is distributed in the hope that it will be useful,
978e96
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
978e96
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
978e96
-   Lesser General Public License for more details.
978e96
-
978e96
-   You should have received a copy of the GNU Lesser General Public
978e96
-   License along with the GNU C Library; if not, see
978e96
-   <http://www.gnu.org/licenses/>.  */
978e96
-
978e96
-#include <unwindbuf.h>
978e96
-#include <sysdep.h>
978e96
-#include <kernel-features.h>
978e96
-#include <lowlevellock.h>
978e96
-
978e96
-
978e96
-	.comm	__fork_generation, 4, 4
978e96
-
978e96
-	.text
978e96
-
978e96
-
978e96
-	.globl	__pthread_once
978e96
-	.type	__pthread_once,@function
978e96
-	.align	16
978e96
-	cfi_startproc
978e96
-__pthread_once:
978e96
-	movl	4(%esp), %ecx
978e96
-	testl	$2, (%ecx)
978e96
-	jz	1f
978e96
-	xorl	%eax, %eax
978e96
-	ret
978e96
-
978e96
-1:	pushl	%ebx
978e96
-	cfi_adjust_cfa_offset (4)
978e96
-	cfi_rel_offset (3, 0)
978e96
-	pushl	%esi
978e96
-	cfi_adjust_cfa_offset (4)
978e96
-	cfi_rel_offset (6, 0)
978e96
-	movl	%ecx, %ebx
978e96
-	xorl	%esi, %esi
978e96
-
978e96
-	/* Not yet initialized or initialization in progress.
978e96
-	   Get the fork generation counter now.  */
978e96
-6:	movl	(%ebx), %eax
978e96
-#ifdef PIC
978e96
-	LOAD_PIC_REG(cx)
978e96
-#endif
978e96
-
978e96
-5:	movl	%eax, %edx
978e96
-
978e96
-	testl	$2, %eax
978e96
-	jnz	4f
978e96
-
978e96
-	andl	$3, %edx
978e96
-#ifdef PIC
978e96
-	orl	__fork_generation@GOTOFF(%ecx), %edx
978e96
-#else
978e96
-	orl	__fork_generation, %edx
978e96
-#endif
978e96
-	orl	$1, %edx
978e96
-
978e96
-	LOCK
978e96
-	cmpxchgl %edx, (%ebx)
978e96
-	jnz	5b
978e96
-
978e96
-	/* Check whether another thread already runs the initializer.  */
978e96
-	testl	$1, %eax
978e96
-	jz	3f	/* No -> do it.  */
978e96
-
978e96
-	/* Check whether the initializer execution was interrupted
978e96
-	   by a fork.  */
978e96
-	xorl	%edx, %eax
978e96
-	testl	$0xfffffffc, %eax
978e96
-	jnz	3f	/* Different for generation -> run initializer.  */
978e96
-
978e96
-	/* Somebody else got here first.  Wait.  */
978e96
-#ifdef __ASSUME_PRIVATE_FUTEX
978e96
-	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
978e96
-#else
978e96
-# if FUTEX_WAIT == 0
978e96
-	movl	%gs:PRIVATE_FUTEX, %ecx
978e96
-# else
978e96
-	movl	$FUTEX_WAIT, %ecx
978e96
-	orl	%gs:PRIVATE_FUTEX, %ecx
978e96
-# endif
978e96
-#endif
978e96
-	movl	$SYS_futex, %eax
978e96
-	ENTER_KERNEL
978e96
-	jmp	6b
978e96
-
978e96
-3:	/* Call the initializer function after setting up the
978e96
-	   cancellation handler.  Note that it is not possible here
978e96
-	   to use the unwind-based cleanup handling.  This would require
978e96
-	   that the user-provided function and all the code it calls
978e96
-	   is compiled with exceptions.  Unfortunately this cannot be
978e96
-	   guaranteed.  */
978e96
-	subl	$UNWINDBUFSIZE+8, %esp
978e96
-	cfi_adjust_cfa_offset (UNWINDBUFSIZE+8)
978e96
-	movl	%ecx, %ebx		/* PIC register value.  */
978e96
-
978e96
-	leal	8+UWJMPBUF(%esp), %eax
978e96
-	movl	$0, 4(%esp)
978e96
-	movl	%eax, (%esp)
978e96
-	call	__sigsetjmp@PLT
978e96
-	testl	%eax, %eax
978e96
-	jne	7f
978e96
-
978e96
-	leal	8(%esp), %eax
978e96
-	call	HIDDEN_JUMPTARGET(__pthread_register_cancel)
978e96
-
978e96
-	/* Call the user-provided initialization function.  */
978e96
-	call	*24+UNWINDBUFSIZE(%esp)
978e96
-
978e96
-	/* Pop the cleanup handler.  */
978e96
-	leal	8(%esp), %eax
978e96
-	call	HIDDEN_JUMPTARGET(__pthread_unregister_cancel)
978e96
-	addl	$UNWINDBUFSIZE+8, %esp
978e96
-	cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8)
978e96
-
978e96
-	/* Sucessful run of the initializer.  Signal that we are done.  */
978e96
-	movl	12(%esp), %ebx
978e96
-	LOCK
978e96
-	addl	$1, (%ebx)
978e96
-
978e96
-	/* Wake up all other threads.  */
978e96
-	movl	$0x7fffffff, %edx
978e96
-#ifdef __ASSUME_PRIVATE_FUTEX
978e96
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
978e96
-#else
978e96
-	movl	$FUTEX_WAKE, %ecx
978e96
-	orl	%gs:PRIVATE_FUTEX, %ecx
978e96
-#endif
978e96
-	movl	$SYS_futex, %eax
978e96
-	ENTER_KERNEL
978e96
-
978e96
-4:	popl	%esi
978e96
-	cfi_adjust_cfa_offset (-4)
978e96
-	cfi_restore (6)
978e96
-	popl	%ebx
978e96
-	cfi_adjust_cfa_offset (-4)
978e96
-	cfi_restore (3)
978e96
-	xorl	%eax, %eax
978e96
-	ret
978e96
-
978e96
-7:	/* __sigsetjmp returned for the second time.  */
978e96
-	movl	20+UNWINDBUFSIZE(%esp), %ebx
978e96
-	cfi_adjust_cfa_offset (UNWINDBUFSIZE+16)
978e96
-	cfi_offset (3, -8)
978e96
-	cfi_offset (6, -12)
978e96
-	movl	$0, (%ebx)
978e96
-
978e96
-	movl	$0x7fffffff, %edx
978e96
-#ifdef __ASSUME_PRIVATE_FUTEX
978e96
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
978e96
-#else
978e96
-	movl	$FUTEX_WAKE, %ecx
978e96
-	orl	%gs:PRIVATE_FUTEX, %ecx
978e96
-#endif
978e96
-	movl	$SYS_futex, %eax
978e96
-	ENTER_KERNEL
978e96
-
978e96
-	leal	8(%esp), %eax
978e96
-	call	HIDDEN_JUMPTARGET (__pthread_unwind_next)
978e96
-	/* NOTREACHED */
978e96
-	hlt
978e96
-	cfi_endproc
978e96
-	.size	__pthread_once,.-__pthread_once
978e96
-
978e96
-hidden_def (__pthread_once)
978e96
-strong_alias (__pthread_once, pthread_once)
978e96
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
978e96
deleted file mode 100644
978e96
index 7f5c0810fa16b987..0000000000000000
978e96
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
978e96
+++ /dev/null
978e96
@@ -1,193 +0,0 @@
978e96
-/* Copyright (C) 2002-2012 Free Software Foundation, Inc.
978e96
-   This file is part of the GNU C Library.
978e96
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
978e96
-
978e96
-   The GNU C Library is free software; you can redistribute it and/or
978e96
-   modify it under the terms of the GNU Lesser General Public
978e96
-   License as published by the Free Software Foundation; either
978e96
-   version 2.1 of the License, or (at your option) any later version.
978e96
-
978e96
-   The GNU C Library is distributed in the hope that it will be useful,
978e96
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
978e96
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
978e96
-   Lesser General Public License for more details.
978e96
-
978e96
-   You should have received a copy of the GNU Lesser General Public
978e96
-   License along with the GNU C Library; if not, see
978e96
-   <http://www.gnu.org/licenses/>.  */
978e96
-
978e96
-#include <sysdep.h>
978e96
-#include <kernel-features.h>
978e96
-#include <tcb-offsets.h>
978e96
-#include <lowlevellock.h>
978e96
-
978e96
-
978e96
-	.comm	__fork_generation, 4, 4
978e96
-
978e96
-	.text
978e96
-
978e96
-
978e96
-	.globl	__pthread_once
978e96
-	.type	__pthread_once,@function
978e96
-	.align	16
978e96
-__pthread_once:
978e96
-.LSTARTCODE:
978e96
-	cfi_startproc
978e96
-#ifdef SHARED
978e96
-	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
978e96
-			DW.ref.__gcc_personality_v0)
978e96
-	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
978e96
-#else
978e96
-	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
978e96
-	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
978e96
-#endif
978e96
-	testl	$2, (%rdi)
978e96
-	jz	1f
978e96
-	xorl	%eax, %eax
978e96
-	retq
978e96
-
978e96
-	/* Preserve the function pointer.  */
978e96
-1:	pushq	%rsi
978e96
-	cfi_adjust_cfa_offset(8)
978e96
-	xorq	%r10, %r10
978e96
-
978e96
-	/* Not yet initialized or initialization in progress.
978e96
-	   Get the fork generation counter now.  */
978e96
-6:	movl	(%rdi), %eax
978e96
-
978e96
-5:	movl	%eax, %edx
978e96
-
978e96
-	testl	$2, %eax
978e96
-	jnz	4f
978e96
-
978e96
-	andl	$3, %edx
978e96
-	orl	__fork_generation(%rip), %edx
978e96
-	orl	$1, %edx
978e96
-
978e96
-	LOCK
978e96
-	cmpxchgl %edx, (%rdi)
978e96
-	jnz	5b
978e96
-
978e96
-	/* Check whether another thread already runs the initializer.  */
978e96
-	testl	$1, %eax
978e96
-	jz	3f	/* No -> do it.  */
978e96
-
978e96
-	/* Check whether the initializer execution was interrupted
978e96
-	   by a fork.  */
978e96
-	xorl	%edx, %eax
978e96
-	testl	$0xfffffffc, %eax
978e96
-	jnz	3f	/* Different for generation -> run initializer.  */
978e96
-
978e96
-	/* Somebody else got here first.  Wait.  */
978e96
-#ifdef __ASSUME_PRIVATE_FUTEX
978e96
-	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
978e96
-#else
978e96
-# if FUTEX_WAIT == 0
978e96
-	movl	%fs:PRIVATE_FUTEX, %esi
978e96
-# else
978e96
-	movl	$FUTEX_WAIT, %esi
978e96
-	orl	%fs:PRIVATE_FUTEX, %esi
978e96
-# endif
978e96
-#endif
978e96
-	movl	$SYS_futex, %eax
978e96
-	syscall
978e96
-	jmp	6b
978e96
-
978e96
-	/* Preserve the pointer to the control variable.  */
978e96
-3:	pushq	%rdi
978e96
-	cfi_adjust_cfa_offset(8)
978e96
-	pushq	%rdi
978e96
-	cfi_adjust_cfa_offset(8)
978e96
-
978e96
-.LcleanupSTART:
978e96
-	callq	*16(%rsp)
978e96
-.LcleanupEND:
978e96
-
978e96
-	/* Get the control variable address back.  */
978e96
-	popq	%rdi
978e96
-	cfi_adjust_cfa_offset(-8)
978e96
-
978e96
-	/* Sucessful run of the initializer.  Signal that we are done.  */
978e96
-	LOCK
978e96
-	incl	(%rdi)
978e96
-
978e96
-	addq	$8, %rsp
978e96
-	cfi_adjust_cfa_offset(-8)
978e96
-
978e96
-	/* Wake up all other threads.  */
978e96
-	movl	$0x7fffffff, %edx
978e96
-#ifdef __ASSUME_PRIVATE_FUTEX
978e96
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
978e96
-#else
978e96
-	movl	$FUTEX_WAKE, %esi
978e96
-	orl	%fs:PRIVATE_FUTEX, %esi
978e96
-#endif
978e96
-	movl	$SYS_futex, %eax
978e96
-	syscall
978e96
-
978e96
-4:	addq	$8, %rsp
978e96
-	cfi_adjust_cfa_offset(-8)
978e96
-	xorl	%eax, %eax
978e96
-	retq
978e96
-	.size	__pthread_once,.-__pthread_once
978e96
-
978e96
-
978e96
-hidden_def (__pthread_once)
978e96
-strong_alias (__pthread_once, pthread_once)
978e96
-
978e96
-
978e96
-	.type	clear_once_control,@function
978e96
-	.align	16
978e96
-clear_once_control:
978e96
-	cfi_adjust_cfa_offset(3 * 8)
978e96
-	movq	(%rsp), %rdi
978e96
-	movq	%rax, %r8
978e96
-	movl	$0, (%rdi)
978e96
-
978e96
-	movl	$0x7fffffff, %edx
978e96
-#ifdef __ASSUME_PRIVATE_FUTEX
978e96
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
978e96
-#else
978e96
-	movl	$FUTEX_WAKE, %esi
978e96
-	orl	%fs:PRIVATE_FUTEX, %esi
978e96
-#endif
978e96
-	movl	$SYS_futex, %eax
978e96
-	syscall
978e96
-
978e96
-	movq	%r8, %rdi
978e96
-.LcallUR:
978e96
-	call	_Unwind_Resume@PLT
978e96
-	hlt
978e96
-.LENDCODE:
978e96
-	cfi_endproc
978e96
-	.size	clear_once_control,.-clear_once_control
978e96
-
978e96
-
978e96
-	.section .gcc_except_table,"a",@progbits
978e96
-.LexceptSTART:
978e96
-	.byte	DW_EH_PE_omit			# @LPStart format
978e96
-	.byte	DW_EH_PE_omit			# @TType format
978e96
-	.byte	DW_EH_PE_uleb128		# call-site format
978e96
-	.uleb128 .Lcstend-.Lcstbegin
978e96
-.Lcstbegin:
978e96
-	.uleb128 .LcleanupSTART-.LSTARTCODE
978e96
-	.uleb128 .LcleanupEND-.LcleanupSTART
978e96
-	.uleb128 clear_once_control-.LSTARTCODE
978e96
-	.uleb128  0
978e96
-	.uleb128 .LcallUR-.LSTARTCODE
978e96
-	.uleb128 .LENDCODE-.LcallUR
978e96
-	.uleb128 0
978e96
-	.uleb128  0
978e96
-.Lcstend:
978e96
-
978e96
-
978e96
-#ifdef SHARED
978e96
-	.hidden	DW.ref.__gcc_personality_v0
978e96
-	.weak	DW.ref.__gcc_personality_v0
978e96
-	.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
978e96
-	.align	LP_SIZE
978e96
-	.type	DW.ref.__gcc_personality_v0, @object
978e96
-	.size	DW.ref.__gcc_personality_v0, LP_SIZE
978e96
-DW.ref.__gcc_personality_v0:
978e96
-	ASM_ADDR __gcc_personality_v0
978e96
-#endif