| This patch is based on the below upstream commit. |
| File deletions were altered to reflect file renames. |
| |
| commit f50277c19df0937ea9691ab7e7c642ecd3ed3d94 |
| Author: Torvald Riegel <triegel@redhat.com> |
| Date: Sun Oct 19 21:59:26 2014 +0200 |
| |
| pthread_once: Add fast path and remove x86 variants. |
| |
| diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c |
| index 595bd7e298003e00..2afb79c01fe0a61e 100644 |
| |
| |
| @@ -58,11 +58,13 @@ clear_once_control (void *arg) |
| initialization is interrupted, we then fork 2^30 times (30 bits of |
| once_control are used for the fork generation), and try to initialize |
| again, we can deadlock because we can't distinguish the in-progress and |
| - interrupted cases anymore. */ |
| -int |
| -__pthread_once (once_control, init_routine) |
| - pthread_once_t *once_control; |
| - void (*init_routine) (void); |
| + interrupted cases anymore. |
| + XXX: We split out this slow path because current compilers do not generate |
| + as efficient code when the fast path in __pthread_once below is not in a |
| + separate function. */ |
| +static int |
| +__attribute__ ((noinline)) |
| +__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void)) |
| { |
| while (1) |
| { |
| @@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine) |
| signals that initialization has finished, we need to see any |
| data modifications done during initialization. */ |
| val = *once_control; |
| - atomic_read_barrier(); |
| + atomic_read_barrier (); |
| do |
| { |
| /* Check if the initialization has already been done. */ |
| @@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine) |
| |
| return 0; |
| } |
| + |
| +int |
| +__pthread_once (pthread_once_t *once_control, void (*init_routine) (void)) |
| +{ |
| + /* Fast path. See __pthread_once_slow. */ |
| + int val; |
| + val = *once_control; |
| + atomic_read_barrier (); |
| + if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0)) |
| + return 0; |
| + else |
| + return __pthread_once_slow (once_control, init_routine); |
| +} |
| weak_alias (__pthread_once, pthread_once) |
| hidden_def (__pthread_once) |
| diff --git a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S |
| deleted file mode 100644 |
| index ca3b860a7f6f95ae..0000000000000000 |
| |
| |
| @@ -1,178 +0,0 @@ |
| -/* Copyright (C) 2002-2012 Free Software Foundation, Inc. |
| - This file is part of the GNU C Library. |
| - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. |
| - |
| - The GNU C Library is free software; you can redistribute it and/or |
| - modify it under the terms of the GNU Lesser General Public |
| - License as published by the Free Software Foundation; either |
| - version 2.1 of the License, or (at your option) any later version. |
| - |
| - The GNU C Library is distributed in the hope that it will be useful, |
| - but WITHOUT ANY WARRANTY; without even the implied warranty of |
| - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| - Lesser General Public License for more details. |
| - |
| - You should have received a copy of the GNU Lesser General Public |
| - License along with the GNU C Library; if not, see |
| - <http://www.gnu.org/licenses/>. */ |
| - |
| -#include <unwindbuf.h> |
| -#include <sysdep.h> |
| -#include <kernel-features.h> |
| -#include <lowlevellock.h> |
| - |
| - |
| - .comm __fork_generation, 4, 4 |
| - |
| - .text |
| - |
| - |
| - .globl __pthread_once |
| - .type __pthread_once,@function |
| - .align 16 |
| - cfi_startproc |
| -__pthread_once: |
| - movl 4(%esp), %ecx |
| - testl $2, (%ecx) |
| - jz 1f |
| - xorl %eax, %eax |
| - ret |
| - |
| -1: pushl %ebx |
| - cfi_adjust_cfa_offset (4) |
| - cfi_rel_offset (3, 0) |
| - pushl %esi |
| - cfi_adjust_cfa_offset (4) |
| - cfi_rel_offset (6, 0) |
| - movl %ecx, %ebx |
| - xorl %esi, %esi |
| - |
| - /* Not yet initialized or initialization in progress. |
| - Get the fork generation counter now. */ |
| -6: movl (%ebx), %eax |
| -#ifdef PIC |
| - LOAD_PIC_REG(cx) |
| -#endif |
| - |
| -5: movl %eax, %edx |
| - |
| - testl $2, %eax |
| - jnz 4f |
| - |
| - andl $3, %edx |
| -#ifdef PIC |
| - orl __fork_generation@GOTOFF(%ecx), %edx |
| -#else |
| - orl __fork_generation, %edx |
| -#endif |
| - orl $1, %edx |
| - |
| - LOCK |
| - cmpxchgl %edx, (%ebx) |
| - jnz 5b |
| - |
| - /* Check whether another thread already runs the initializer. */ |
| - testl $1, %eax |
| - jz 3f /* No -> do it. */ |
| - |
| - /* Check whether the initializer execution was interrupted |
| - by a fork. */ |
| - xorl %edx, %eax |
| - testl $0xfffffffc, %eax |
| - jnz 3f /* Different for generation -> run initializer. */ |
| - |
| - /* Somebody else got here first. Wait. */ |
| -#ifdef __ASSUME_PRIVATE_FUTEX |
| - movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx |
| -#else |
| -# if FUTEX_WAIT == 0 |
| - movl %gs:PRIVATE_FUTEX, %ecx |
| -# else |
| - movl $FUTEX_WAIT, %ecx |
| - orl %gs:PRIVATE_FUTEX, %ecx |
| -# endif |
| -#endif |
| - movl $SYS_futex, %eax |
| - ENTER_KERNEL |
| - jmp 6b |
| - |
| -3: /* Call the initializer function after setting up the |
| - cancellation handler. Note that it is not possible here |
| - to use the unwind-based cleanup handling. This would require |
| - that the user-provided function and all the code it calls |
| - is compiled with exceptions. Unfortunately this cannot be |
| - guaranteed. */ |
| - subl $UNWINDBUFSIZE+8, %esp |
| - cfi_adjust_cfa_offset (UNWINDBUFSIZE+8) |
| - movl %ecx, %ebx /* PIC register value. */ |
| - |
| - leal 8+UWJMPBUF(%esp), %eax |
| - movl $0, 4(%esp) |
| - movl %eax, (%esp) |
| - call __sigsetjmp@PLT |
| - testl %eax, %eax |
| - jne 7f |
| - |
| - leal 8(%esp), %eax |
| - call HIDDEN_JUMPTARGET(__pthread_register_cancel) |
| - |
| - /* Call the user-provided initialization function. */ |
| - call *24+UNWINDBUFSIZE(%esp) |
| - |
| - /* Pop the cleanup handler. */ |
| - leal 8(%esp), %eax |
| - call HIDDEN_JUMPTARGET(__pthread_unregister_cancel) |
| - addl $UNWINDBUFSIZE+8, %esp |
| - cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8) |
| - |
| - /* Sucessful run of the initializer. Signal that we are done. */ |
| - movl 12(%esp), %ebx |
| - LOCK |
| - addl $1, (%ebx) |
| - |
| - /* Wake up all other threads. */ |
| - movl $0x7fffffff, %edx |
| -#ifdef __ASSUME_PRIVATE_FUTEX |
| - movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx |
| -#else |
| - movl $FUTEX_WAKE, %ecx |
| - orl %gs:PRIVATE_FUTEX, %ecx |
| -#endif |
| - movl $SYS_futex, %eax |
| - ENTER_KERNEL |
| - |
| -4: popl %esi |
| - cfi_adjust_cfa_offset (-4) |
| - cfi_restore (6) |
| - popl %ebx |
| - cfi_adjust_cfa_offset (-4) |
| - cfi_restore (3) |
| - xorl %eax, %eax |
| - ret |
| - |
| -7: /* __sigsetjmp returned for the second time. */ |
| - movl 20+UNWINDBUFSIZE(%esp), %ebx |
| - cfi_adjust_cfa_offset (UNWINDBUFSIZE+16) |
| - cfi_offset (3, -8) |
| - cfi_offset (6, -12) |
| - movl $0, (%ebx) |
| - |
| - movl $0x7fffffff, %edx |
| -#ifdef __ASSUME_PRIVATE_FUTEX |
| - movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx |
| -#else |
| - movl $FUTEX_WAKE, %ecx |
| - orl %gs:PRIVATE_FUTEX, %ecx |
| -#endif |
| - movl $SYS_futex, %eax |
| - ENTER_KERNEL |
| - |
| - leal 8(%esp), %eax |
| - call HIDDEN_JUMPTARGET (__pthread_unwind_next) |
| - /* NOTREACHED */ |
| - hlt |
| - cfi_endproc |
| - .size __pthread_once,.-__pthread_once |
| - |
| -hidden_def (__pthread_once) |
| -strong_alias (__pthread_once, pthread_once) |
| diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S |
| deleted file mode 100644 |
| index 7f5c0810fa16b987..0000000000000000 |
| |
| |
| @@ -1,193 +0,0 @@ |
| -/* Copyright (C) 2002-2012 Free Software Foundation, Inc. |
| - This file is part of the GNU C Library. |
| - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. |
| - |
| - The GNU C Library is free software; you can redistribute it and/or |
| - modify it under the terms of the GNU Lesser General Public |
| - License as published by the Free Software Foundation; either |
| - version 2.1 of the License, or (at your option) any later version. |
| - |
| - The GNU C Library is distributed in the hope that it will be useful, |
| - but WITHOUT ANY WARRANTY; without even the implied warranty of |
| - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| - Lesser General Public License for more details. |
| - |
| - You should have received a copy of the GNU Lesser General Public |
| - License along with the GNU C Library; if not, see |
| - <http://www.gnu.org/licenses/>. */ |
| - |
| -#include <sysdep.h> |
| -#include <kernel-features.h> |
| -#include <tcb-offsets.h> |
| -#include <lowlevellock.h> |
| - |
| - |
| - .comm __fork_generation, 4, 4 |
| - |
| - .text |
| - |
| - |
| - .globl __pthread_once |
| - .type __pthread_once,@function |
| - .align 16 |
| -__pthread_once: |
| -.LSTARTCODE: |
| - cfi_startproc |
| -#ifdef SHARED |
| - cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, |
| - DW.ref.__gcc_personality_v0) |
| - cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) |
| -#else |
| - cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) |
| - cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) |
| -#endif |
| - testl $2, (%rdi) |
| - jz 1f |
| - xorl %eax, %eax |
| - retq |
| - |
| - /* Preserve the function pointer. */ |
| -1: pushq %rsi |
| - cfi_adjust_cfa_offset(8) |
| - xorq %r10, %r10 |
| - |
| - /* Not yet initialized or initialization in progress. |
| - Get the fork generation counter now. */ |
| -6: movl (%rdi), %eax |
| - |
| -5: movl %eax, %edx |
| - |
| - testl $2, %eax |
| - jnz 4f |
| - |
| - andl $3, %edx |
| - orl __fork_generation(%rip), %edx |
| - orl $1, %edx |
| - |
| - LOCK |
| - cmpxchgl %edx, (%rdi) |
| - jnz 5b |
| - |
| - /* Check whether another thread already runs the initializer. */ |
| - testl $1, %eax |
| - jz 3f /* No -> do it. */ |
| - |
| - /* Check whether the initializer execution was interrupted |
| - by a fork. */ |
| - xorl %edx, %eax |
| - testl $0xfffffffc, %eax |
| - jnz 3f /* Different for generation -> run initializer. */ |
| - |
| - /* Somebody else got here first. Wait. */ |
| -#ifdef __ASSUME_PRIVATE_FUTEX |
| - movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi |
| -#else |
| -# if FUTEX_WAIT == 0 |
| - movl %fs:PRIVATE_FUTEX, %esi |
| -# else |
| - movl $FUTEX_WAIT, %esi |
| - orl %fs:PRIVATE_FUTEX, %esi |
| -# endif |
| -#endif |
| - movl $SYS_futex, %eax |
| - syscall |
| - jmp 6b |
| - |
| - /* Preserve the pointer to the control variable. */ |
| -3: pushq %rdi |
| - cfi_adjust_cfa_offset(8) |
| - pushq %rdi |
| - cfi_adjust_cfa_offset(8) |
| - |
| -.LcleanupSTART: |
| - callq *16(%rsp) |
| -.LcleanupEND: |
| - |
| - /* Get the control variable address back. */ |
| - popq %rdi |
| - cfi_adjust_cfa_offset(-8) |
| - |
| - /* Sucessful run of the initializer. Signal that we are done. */ |
| - LOCK |
| - incl (%rdi) |
| - |
| - addq $8, %rsp |
| - cfi_adjust_cfa_offset(-8) |
| - |
| - /* Wake up all other threads. */ |
| - movl $0x7fffffff, %edx |
| -#ifdef __ASSUME_PRIVATE_FUTEX |
| - movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi |
| -#else |
| - movl $FUTEX_WAKE, %esi |
| - orl %fs:PRIVATE_FUTEX, %esi |
| -#endif |
| - movl $SYS_futex, %eax |
| - syscall |
| - |
| -4: addq $8, %rsp |
| - cfi_adjust_cfa_offset(-8) |
| - xorl %eax, %eax |
| - retq |
| - .size __pthread_once,.-__pthread_once |
| - |
| - |
| -hidden_def (__pthread_once) |
| -strong_alias (__pthread_once, pthread_once) |
| - |
| - |
| - .type clear_once_control,@function |
| - .align 16 |
| -clear_once_control: |
| - cfi_adjust_cfa_offset(3 * 8) |
| - movq (%rsp), %rdi |
| - movq %rax, %r8 |
| - movl $0, (%rdi) |
| - |
| - movl $0x7fffffff, %edx |
| -#ifdef __ASSUME_PRIVATE_FUTEX |
| - movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi |
| -#else |
| - movl $FUTEX_WAKE, %esi |
| - orl %fs:PRIVATE_FUTEX, %esi |
| -#endif |
| - movl $SYS_futex, %eax |
| - syscall |
| - |
| - movq %r8, %rdi |
| -.LcallUR: |
| - call _Unwind_Resume@PLT |
| - hlt |
| -.LENDCODE: |
| - cfi_endproc |
| - .size clear_once_control,.-clear_once_control |
| - |
| - |
| - .section .gcc_except_table,"a",@progbits |
| -.LexceptSTART: |
| - .byte DW_EH_PE_omit # @LPStart format |
| - .byte DW_EH_PE_omit # @TType format |
| - .byte DW_EH_PE_uleb128 # call-site format |
| - .uleb128 .Lcstend-.Lcstbegin |
| -.Lcstbegin: |
| - .uleb128 .LcleanupSTART-.LSTARTCODE |
| - .uleb128 .LcleanupEND-.LcleanupSTART |
| - .uleb128 clear_once_control-.LSTARTCODE |
| - .uleb128 0 |
| - .uleb128 .LcallUR-.LSTARTCODE |
| - .uleb128 .LENDCODE-.LcallUR |
| - .uleb128 0 |
| - .uleb128 0 |
| -.Lcstend: |
| - |
| - |
| -#ifdef SHARED |
| - .hidden DW.ref.__gcc_personality_v0 |
| - .weak DW.ref.__gcc_personality_v0 |
| - .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits |
| - .align LP_SIZE |
| - .type DW.ref.__gcc_personality_v0, @object |
| - .size DW.ref.__gcc_personality_v0, LP_SIZE |
| -DW.ref.__gcc_personality_v0: |
| - ASM_ADDR __gcc_personality_v0 |
| -#endif |