|
|
50f89d |
commit 3d265911c2aac65d978f679101594f9071024874
|
|
|
50f89d |
Author: Andreas Schwab <schwab@suse.de>
|
|
|
50f89d |
Date: Mon Nov 12 11:11:40 2018 +0100
|
|
|
50f89d |
|
|
|
50f89d |
Reindent nptl/pthread_rwlock_common.c
|
|
|
50f89d |
|
|
|
50f89d |
diff --git a/nptl/pthread_rwlock_common.c b/nptl/pthread_rwlock_common.c
|
|
|
50f89d |
index a290d08332b802a5..5dd534271aed6b41 100644
|
|
|
50f89d |
--- a/nptl/pthread_rwlock_common.c
|
|
|
50f89d |
+++ b/nptl/pthread_rwlock_common.c
|
|
|
50f89d |
@@ -34,7 +34,7 @@
|
|
|
50f89d |
|
|
|
50f89d |
A thread is allowed to acquire a read lock recursively (i.e., have rdlock
|
|
|
50f89d |
critical sections that overlap in sequenced-before) unless the kind of the
|
|
|
50f89d |
- rwlock is set to PTHREAD_RWLOCK_PREFER_WRITERS_NONRECURSIVE_NP.
|
|
|
50f89d |
+ rwlock is set to PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP.
|
|
|
50f89d |
|
|
|
50f89d |
This lock is built so that workloads of mostly readers can be executed with
|
|
|
50f89d |
low runtime overheads. This matches that the default kind of the lock is
|
|
|
50f89d |
@@ -46,7 +46,7 @@
|
|
|
50f89d |
An uncontended write lock acquisition is as fast as for a normal
|
|
|
50f89d |
exclusive mutex but writer contention is somewhat more costly due to
|
|
|
50f89d |
keeping track of the exact number of writers. If the rwlock kind requests
|
|
|
50f89d |
- writers to be preferred (i.e., PTHREAD_RWLOCK_PREFER_WRITERS_NP or the
|
|
|
50f89d |
+ writers to be preferred (i.e., PTHREAD_RWLOCK_PREFER_WRITER_NP or the
|
|
|
50f89d |
no-recursive-readers variant of it), then writer--to--writer lock ownership
|
|
|
50f89d |
hand-over is fairly fast and bypasses lock acquisition attempts by readers.
|
|
|
50f89d |
The costs of lock ownership transfer between readers and writers vary. If
|
|
|
50f89d |
@@ -251,7 +251,7 @@ __pthread_rwlock_rdunlock (pthread_rwlock_t *rwlock)
|
|
|
50f89d |
the first reader's store to __wrphase_futex (or a later value) if
|
|
|
50f89d |
the writer observes that a write phase has been started. */
|
|
|
50f89d |
if (atomic_compare_exchange_weak_release (&rwlock->__data.__readers,
|
|
|
50f89d |
- &r, rnew))
|
|
|
50f89d |
+ &r, rnew))
|
|
|
50f89d |
break;
|
|
|
50f89d |
/* TODO Back-off. */
|
|
|
50f89d |
}
|
|
|
50f89d |
@@ -285,7 +285,7 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* Make sure we are not holding the rwlock as a writer. This is a deadlock
|
|
|
50f89d |
situation we recognize and report. */
|
|
|
50f89d |
if (__glibc_unlikely (atomic_load_relaxed (&rwlock->__data.__cur_writer)
|
|
|
50f89d |
- == THREAD_GETMEM (THREAD_SELF, tid)))
|
|
|
50f89d |
+ == THREAD_GETMEM (THREAD_SELF, tid)))
|
|
|
50f89d |
return EDEADLK;
|
|
|
50f89d |
|
|
|
50f89d |
/* If we prefer writers, recursive rdlock is disallowed, we are in a read
|
|
|
50f89d |
@@ -299,9 +299,9 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
if (rwlock->__data.__flags == PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
|
|
|
50f89d |
{
|
|
|
50f89d |
r = atomic_load_relaxed (&rwlock->__data.__readers);
|
|
|
50f89d |
- while (((r & PTHREAD_RWLOCK_WRPHASE) == 0)
|
|
|
50f89d |
- && ((r & PTHREAD_RWLOCK_WRLOCKED) != 0)
|
|
|
50f89d |
- && ((r >> PTHREAD_RWLOCK_READER_SHIFT) > 0))
|
|
|
50f89d |
+ while ((r & PTHREAD_RWLOCK_WRPHASE) == 0
|
|
|
50f89d |
+ && (r & PTHREAD_RWLOCK_WRLOCKED) != 0
|
|
|
50f89d |
+ && (r >> PTHREAD_RWLOCK_READER_SHIFT) > 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* TODO Spin first. */
|
|
|
50f89d |
/* Try setting the flag signaling that we are waiting without having
|
|
|
50f89d |
@@ -315,11 +315,11 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
__readers, and all threads set the flag under the same
|
|
|
50f89d |
conditions. */
|
|
|
50f89d |
while ((atomic_load_relaxed (&rwlock->__data.__readers)
|
|
|
50f89d |
- & PTHREAD_RWLOCK_RWAITING) != 0)
|
|
|
50f89d |
+ & PTHREAD_RWLOCK_RWAITING) != 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
int private = __pthread_rwlock_get_private (rwlock);
|
|
|
50f89d |
int err = futex_abstimed_wait (&rwlock->__data.__readers,
|
|
|
50f89d |
- r, abstime, private);
|
|
|
50f89d |
+ r, abstime, private);
|
|
|
50f89d |
/* We ignore EAGAIN and EINTR. On time-outs, we can just
|
|
|
50f89d |
return because we don't need to clean up anything. */
|
|
|
50f89d |
if (err == ETIMEDOUT)
|
|
|
50f89d |
@@ -338,8 +338,9 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
expected value for future operations. Acquire MO so we synchronize with
|
|
|
50f89d |
prior writers as well as the last reader of the previous read phase (see
|
|
|
50f89d |
below). */
|
|
|
50f89d |
- r = atomic_fetch_add_acquire (&rwlock->__data.__readers,
|
|
|
50f89d |
- (1 << PTHREAD_RWLOCK_READER_SHIFT)) + (1 << PTHREAD_RWLOCK_READER_SHIFT);
|
|
|
50f89d |
+ r = (atomic_fetch_add_acquire (&rwlock->__data.__readers,
|
|
|
50f89d |
+ (1 << PTHREAD_RWLOCK_READER_SHIFT))
|
|
|
50f89d |
+ + (1 << PTHREAD_RWLOCK_READER_SHIFT));
|
|
|
50f89d |
|
|
|
50f89d |
/* Check whether there is an overflow in the number of readers. We assume
|
|
|
50f89d |
that the total number of threads is less than half the maximum number
|
|
|
50f89d |
@@ -359,8 +360,9 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* Relaxed MO is okay because we just want to undo our registration and
|
|
|
50f89d |
cannot have changed the rwlock state substantially if the CAS
|
|
|
50f89d |
succeeds. */
|
|
|
50f89d |
- if (atomic_compare_exchange_weak_relaxed (&rwlock->__data.__readers, &r,
|
|
|
50f89d |
- r - (1 << PTHREAD_RWLOCK_READER_SHIFT)))
|
|
|
50f89d |
+ if (atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
+ (&rwlock->__data.__readers,
|
|
|
50f89d |
+ &r, r - (1 << PTHREAD_RWLOCK_READER_SHIFT)))
|
|
|
50f89d |
return EAGAIN;
|
|
|
50f89d |
}
|
|
|
50f89d |
|
|
|
50f89d |
@@ -378,15 +380,15 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* Otherwise, if we were in a write phase (states #6 or #8), we must wait
|
|
|
50f89d |
for explicit hand-over of the read phase; the only exception is if we
|
|
|
50f89d |
can start a read phase if there is no primary writer currently. */
|
|
|
50f89d |
- while (((r & PTHREAD_RWLOCK_WRPHASE) != 0)
|
|
|
50f89d |
- && ((r & PTHREAD_RWLOCK_WRLOCKED) == 0))
|
|
|
50f89d |
+ while ((r & PTHREAD_RWLOCK_WRPHASE) != 0
|
|
|
50f89d |
+ && (r & PTHREAD_RWLOCK_WRLOCKED) == 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
- /* Try to enter a read phase: If the CAS below succeeds, we have
|
|
|
50f89d |
+ /* Try to enter a read phase: If the CAS below succeeds, we have
|
|
|
50f89d |
ownership; if it fails, we will simply retry and reassess the
|
|
|
50f89d |
situation.
|
|
|
50f89d |
Acquire MO so we synchronize with prior writers. */
|
|
|
50f89d |
if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers, &r,
|
|
|
50f89d |
- r ^ PTHREAD_RWLOCK_WRPHASE))
|
|
|
50f89d |
+ r ^ PTHREAD_RWLOCK_WRPHASE))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* We started the read phase, so we are also responsible for
|
|
|
50f89d |
updating the write-phase futex. Relaxed MO is sufficient.
|
|
|
50f89d |
@@ -397,7 +399,7 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
(but we can pretend to do the setting and unsetting of WRLOCKED
|
|
|
50f89d |
atomically, and thus can skip this step). */
|
|
|
50f89d |
if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 0)
|
|
|
50f89d |
- & PTHREAD_RWLOCK_FUTEX_USED) != 0)
|
|
|
50f89d |
+ & PTHREAD_RWLOCK_FUTEX_USED) != 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
int private = __pthread_rwlock_get_private (rwlock);
|
|
|
50f89d |
futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private);
|
|
|
50f89d |
@@ -435,16 +437,17 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
for (;;)
|
|
|
50f89d |
{
|
|
|
50f89d |
while (((wpf = atomic_load_relaxed (&rwlock->__data.__wrphase_futex))
|
|
|
50f89d |
- | PTHREAD_RWLOCK_FUTEX_USED) == (1 | PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
+ | PTHREAD_RWLOCK_FUTEX_USED) == (1 | PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
{
|
|
|
50f89d |
int private = __pthread_rwlock_get_private (rwlock);
|
|
|
50f89d |
if (((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0)
|
|
|
50f89d |
- && !atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
+ && (!atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
(&rwlock->__data.__wrphase_futex,
|
|
|
50f89d |
- &wpf, wpf | PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
+ &wpf, wpf | PTHREAD_RWLOCK_FUTEX_USED)))
|
|
|
50f89d |
continue;
|
|
|
50f89d |
int err = futex_abstimed_wait (&rwlock->__data.__wrphase_futex,
|
|
|
50f89d |
- 1 | PTHREAD_RWLOCK_FUTEX_USED, abstime, private);
|
|
|
50f89d |
+ 1 | PTHREAD_RWLOCK_FUTEX_USED,
|
|
|
50f89d |
+ abstime, private);
|
|
|
50f89d |
if (err == ETIMEDOUT)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* If we timed out, we need to unregister. If no read phase
|
|
|
50f89d |
@@ -477,8 +480,8 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
in this case and thus make the spin-waiting we need
|
|
|
50f89d |
unnecessarily expensive. */
|
|
|
50f89d |
while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex)
|
|
|
50f89d |
- | PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
- == (1 | PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
+ | PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
+ == (1 | PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* TODO Back-off? */
|
|
|
50f89d |
}
|
|
|
50f89d |
@@ -495,7 +498,7 @@ __pthread_rwlock_rdlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
release of the writer, and so that we observe a recent value of
|
|
|
50f89d |
__wrphase_futex (see below). */
|
|
|
50f89d |
if ((atomic_load_acquire (&rwlock->__data.__readers)
|
|
|
50f89d |
- & PTHREAD_RWLOCK_WRPHASE) == 0)
|
|
|
50f89d |
+ & PTHREAD_RWLOCK_WRPHASE) == 0)
|
|
|
50f89d |
/* We are in a read phase now, so the least recent modification of
|
|
|
50f89d |
__wrphase_futex we can read from is the store by the writer
|
|
|
50f89d |
with value 1. Thus, only now we can assume that if we observe
|
|
|
50f89d |
@@ -516,8 +519,9 @@ __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock)
|
|
|
50f89d |
atomic_store_relaxed (&rwlock->__data.__cur_writer, 0);
|
|
|
50f89d |
/* Disable waiting by writers. We will wake up after we decided how to
|
|
|
50f89d |
proceed. */
|
|
|
50f89d |
- bool wake_writers = ((atomic_exchange_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers_futex, 0) & PTHREAD_RWLOCK_FUTEX_USED) != 0);
|
|
|
50f89d |
+ bool wake_writers
|
|
|
50f89d |
+ = ((atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0)
|
|
|
50f89d |
+ & PTHREAD_RWLOCK_FUTEX_USED) != 0);
|
|
|
50f89d |
|
|
|
50f89d |
if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP)
|
|
|
50f89d |
{
|
|
|
50f89d |
@@ -529,8 +533,8 @@ __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock)
|
|
|
50f89d |
synchronize with us and thus can take over our view of
|
|
|
50f89d |
__readers (including, for example, whether we are in a write
|
|
|
50f89d |
phase or not). */
|
|
|
50f89d |
- if (atomic_compare_exchange_weak_release (&rwlock->__data.__writers,
|
|
|
50f89d |
- &w, w | PTHREAD_RWLOCK_WRHANDOVER))
|
|
|
50f89d |
+ if (atomic_compare_exchange_weak_release
|
|
|
50f89d |
+ (&rwlock->__data.__writers, &w, w | PTHREAD_RWLOCK_WRHANDOVER))
|
|
|
50f89d |
/* Another writer will take over. */
|
|
|
50f89d |
goto done;
|
|
|
50f89d |
/* TODO Back-off. */
|
|
|
50f89d |
@@ -543,9 +547,10 @@ __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock)
|
|
|
50f89d |
unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers);
|
|
|
50f89d |
/* Release MO so that subsequent readers or writers synchronize with us. */
|
|
|
50f89d |
while (!atomic_compare_exchange_weak_release
|
|
|
50f89d |
- (&rwlock->__data.__readers, &r, (r ^ PTHREAD_RWLOCK_WRLOCKED)
|
|
|
50f89d |
- ^ ((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0 ? 0
|
|
|
50f89d |
- : PTHREAD_RWLOCK_WRPHASE)))
|
|
|
50f89d |
+ (&rwlock->__data.__readers, &r,
|
|
|
50f89d |
+ ((r ^ PTHREAD_RWLOCK_WRLOCKED)
|
|
|
50f89d |
+ ^ ((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0 ? 0
|
|
|
50f89d |
+ : PTHREAD_RWLOCK_WRPHASE))))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* TODO Back-off. */
|
|
|
50f89d |
}
|
|
|
50f89d |
@@ -574,7 +579,7 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* Make sure we are not holding the rwlock as a writer. This is a deadlock
|
|
|
50f89d |
situation we recognize and report. */
|
|
|
50f89d |
if (__glibc_unlikely (atomic_load_relaxed (&rwlock->__data.__cur_writer)
|
|
|
50f89d |
- == THREAD_GETMEM (THREAD_SELF, tid)))
|
|
|
50f89d |
+ == THREAD_GETMEM (THREAD_SELF, tid)))
|
|
|
50f89d |
return EDEADLK;
|
|
|
50f89d |
|
|
|
50f89d |
/* First we try to acquire the role of primary writer by setting WRLOCKED;
|
|
|
50f89d |
@@ -593,12 +598,12 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
this could be less scalable if readers arrive and leave frequently. */
|
|
|
50f89d |
bool may_share_futex_used_flag = false;
|
|
|
50f89d |
unsigned int r = atomic_fetch_or_acquire (&rwlock->__data.__readers,
|
|
|
50f89d |
- PTHREAD_RWLOCK_WRLOCKED);
|
|
|
50f89d |
+ PTHREAD_RWLOCK_WRLOCKED);
|
|
|
50f89d |
if (__glibc_unlikely ((r & PTHREAD_RWLOCK_WRLOCKED) != 0))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* There is another primary writer. */
|
|
|
50f89d |
- bool prefer_writer =
|
|
|
50f89d |
- (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP);
|
|
|
50f89d |
+ bool prefer_writer
|
|
|
50f89d |
+ = (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP);
|
|
|
50f89d |
if (prefer_writer)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* We register as a waiting writer, so that we can make use of
|
|
|
50f89d |
@@ -617,8 +622,7 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* Try to become the primary writer or retry. Acquire MO as in
|
|
|
50f89d |
the fetch_or above. */
|
|
|
50f89d |
if (atomic_compare_exchange_weak_acquire
|
|
|
50f89d |
- (&rwlock->__data.__readers, &r,
|
|
|
50f89d |
- r | PTHREAD_RWLOCK_WRLOCKED))
|
|
|
50f89d |
+ (&rwlock->__data.__readers, &r, r | PTHREAD_RWLOCK_WRLOCKED))
|
|
|
50f89d |
{
|
|
|
50f89d |
if (prefer_writer)
|
|
|
50f89d |
{
|
|
|
50f89d |
@@ -633,8 +637,7 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
__writers).
|
|
|
50f89d |
??? Perhaps this is not strictly necessary for
|
|
|
50f89d |
reasons we do not yet know of. */
|
|
|
50f89d |
- atomic_fetch_add_relaxed (&rwlock->__data.__writers,
|
|
|
50f89d |
- -1);
|
|
|
50f89d |
+ atomic_fetch_add_relaxed (&rwlock->__data.__writers, -1);
|
|
|
50f89d |
}
|
|
|
50f89d |
break;
|
|
|
50f89d |
}
|
|
|
50f89d |
@@ -646,8 +649,7 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
succeed, we own WRLOCKED. */
|
|
|
50f89d |
if (prefer_writer)
|
|
|
50f89d |
{
|
|
|
50f89d |
- unsigned int w = atomic_load_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers);
|
|
|
50f89d |
+ unsigned int w = atomic_load_relaxed (&rwlock->__data.__writers);
|
|
|
50f89d |
if ((w & PTHREAD_RWLOCK_WRHANDOVER) != 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* Acquire MO is required here so that we synchronize with
|
|
|
50f89d |
@@ -677,13 +679,13 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* We did not acquire WRLOCKED nor were able to use writer--writer
|
|
|
50f89d |
hand-over, so we block on __writers_futex. */
|
|
|
50f89d |
int private = __pthread_rwlock_get_private (rwlock);
|
|
|
50f89d |
- unsigned int wf = atomic_load_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers_futex);
|
|
|
50f89d |
+ unsigned int wf
|
|
|
50f89d |
+ = atomic_load_relaxed (&rwlock->__data.__writers_futex);
|
|
|
50f89d |
if (((wf & ~(unsigned int) PTHREAD_RWLOCK_FUTEX_USED) != 1)
|
|
|
50f89d |
|| ((wf != (1 | PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
- && !atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
+ && (!atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
(&rwlock->__data.__writers_futex, &wf,
|
|
|
50f89d |
- 1 | PTHREAD_RWLOCK_FUTEX_USED)))
|
|
|
50f89d |
+ 1 | PTHREAD_RWLOCK_FUTEX_USED))))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* If we cannot block on __writers_futex because there is no
|
|
|
50f89d |
primary writer, or we cannot set PTHREAD_RWLOCK_FUTEX_USED,
|
|
|
50f89d |
@@ -704,7 +706,8 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
in this group. */
|
|
|
50f89d |
may_share_futex_used_flag = true;
|
|
|
50f89d |
int err = futex_abstimed_wait (&rwlock->__data.__writers_futex,
|
|
|
50f89d |
- 1 | PTHREAD_RWLOCK_FUTEX_USED, abstime, private);
|
|
|
50f89d |
+ 1 | PTHREAD_RWLOCK_FUTEX_USED,
|
|
|
50f89d |
+ abstime, private);
|
|
|
50f89d |
if (err == ETIMEDOUT)
|
|
|
50f89d |
{
|
|
|
50f89d |
if (prefer_writer)
|
|
|
50f89d |
@@ -716,10 +719,10 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
that this happened before the timeout; see
|
|
|
50f89d |
pthread_rwlock_rdlock_full for the full reasoning.)
|
|
|
50f89d |
Also see the similar code above. */
|
|
|
50f89d |
- unsigned int w = atomic_load_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers);
|
|
|
50f89d |
+ unsigned int w
|
|
|
50f89d |
+ = atomic_load_relaxed (&rwlock->__data.__writers);
|
|
|
50f89d |
while (!atomic_compare_exchange_weak_acquire
|
|
|
50f89d |
- (&rwlock->__data.__writers, &w,
|
|
|
50f89d |
+ (&rwlock->__data.__writers, &w,
|
|
|
50f89d |
(w == PTHREAD_RWLOCK_WRHANDOVER + 1 ? 0 : w - 1)))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* TODO Back-off. */
|
|
|
50f89d |
@@ -751,7 +754,8 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
modifications of __readers ensures that this store happens after the
|
|
|
50f89d |
store of value 0 by the previous primary writer. */
|
|
|
50f89d |
atomic_store_relaxed (&rwlock->__data.__writers_futex,
|
|
|
50f89d |
- 1 | (may_share_futex_used_flag ? PTHREAD_RWLOCK_FUTEX_USED : 0));
|
|
|
50f89d |
+ 1 | (may_share_futex_used_flag
|
|
|
50f89d |
+ ? PTHREAD_RWLOCK_FUTEX_USED : 0));
|
|
|
50f89d |
|
|
|
50f89d |
/* If we are in a write phase, we have acquired the lock. */
|
|
|
50f89d |
if ((r & PTHREAD_RWLOCK_WRPHASE) != 0)
|
|
|
50f89d |
@@ -759,15 +763,15 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
|
|
|
50f89d |
/* If we are in a read phase and there are no readers, try to start a write
|
|
|
50f89d |
phase. */
|
|
|
50f89d |
- while (((r & PTHREAD_RWLOCK_WRPHASE) == 0)
|
|
|
50f89d |
- && ((r >> PTHREAD_RWLOCK_READER_SHIFT) == 0))
|
|
|
50f89d |
+ while ((r & PTHREAD_RWLOCK_WRPHASE) == 0
|
|
|
50f89d |
+ && (r >> PTHREAD_RWLOCK_READER_SHIFT) == 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* Acquire MO so that we synchronize with prior writers and do
|
|
|
50f89d |
not interfere with their updates to __writers_futex, as well
|
|
|
50f89d |
as regarding prior readers and their updates to __wrphase_futex,
|
|
|
50f89d |
respectively. */
|
|
|
50f89d |
if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers,
|
|
|
50f89d |
- &r, r | PTHREAD_RWLOCK_WRPHASE))
|
|
|
50f89d |
+ &r, r | PTHREAD_RWLOCK_WRPHASE))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* We have started a write phase, so need to enable readers to wait.
|
|
|
50f89d |
See the similar case in __pthread_rwlock_rdlock_full. Unlike in
|
|
|
50f89d |
@@ -792,24 +796,24 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
for (;;)
|
|
|
50f89d |
{
|
|
|
50f89d |
while (((wpf = atomic_load_relaxed (&rwlock->__data.__wrphase_futex))
|
|
|
50f89d |
- | PTHREAD_RWLOCK_FUTEX_USED) == PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
+ | PTHREAD_RWLOCK_FUTEX_USED) == PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
{
|
|
|
50f89d |
int private = __pthread_rwlock_get_private (rwlock);
|
|
|
50f89d |
- if (((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0)
|
|
|
50f89d |
- && !atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
+ if ((wpf & PTHREAD_RWLOCK_FUTEX_USED) == 0
|
|
|
50f89d |
+ && (!atomic_compare_exchange_weak_relaxed
|
|
|
50f89d |
(&rwlock->__data.__wrphase_futex, &wpf,
|
|
|
50f89d |
- PTHREAD_RWLOCK_FUTEX_USED))
|
|
|
50f89d |
+ PTHREAD_RWLOCK_FUTEX_USED)))
|
|
|
50f89d |
continue;
|
|
|
50f89d |
int err = futex_abstimed_wait (&rwlock->__data.__wrphase_futex,
|
|
|
50f89d |
- PTHREAD_RWLOCK_FUTEX_USED, abstime, private);
|
|
|
50f89d |
+ PTHREAD_RWLOCK_FUTEX_USED,
|
|
|
50f89d |
+ abstime, private);
|
|
|
50f89d |
if (err == ETIMEDOUT)
|
|
|
50f89d |
{
|
|
|
50f89d |
- if (rwlock->__data.__flags
|
|
|
50f89d |
- != PTHREAD_RWLOCK_PREFER_READER_NP)
|
|
|
50f89d |
+ if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* We try writer--writer hand-over. */
|
|
|
50f89d |
- unsigned int w = atomic_load_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers);
|
|
|
50f89d |
+ unsigned int w
|
|
|
50f89d |
+ = atomic_load_relaxed (&rwlock->__data.__writers);
|
|
|
50f89d |
if (w != 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* We are about to hand over WRLOCKED, so we must
|
|
|
50f89d |
@@ -823,13 +827,13 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
Release MO so that another writer that gets
|
|
|
50f89d |
WRLOCKED from us can take over our view of
|
|
|
50f89d |
__readers. */
|
|
|
50f89d |
- unsigned int wf = atomic_exchange_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers_futex, 0);
|
|
|
50f89d |
+ unsigned int wf
|
|
|
50f89d |
+ = atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0);
|
|
|
50f89d |
while (w != 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
if (atomic_compare_exchange_weak_release
|
|
|
50f89d |
(&rwlock->__data.__writers, &w,
|
|
|
50f89d |
- w | PTHREAD_RWLOCK_WRHANDOVER))
|
|
|
50f89d |
+ w | PTHREAD_RWLOCK_WRHANDOVER))
|
|
|
50f89d |
{
|
|
|
50f89d |
/* Wake other writers. */
|
|
|
50f89d |
if ((wf & PTHREAD_RWLOCK_FUTEX_USED) != 0)
|
|
|
50f89d |
@@ -844,8 +848,7 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
again. Make sure we don't loose the flag that
|
|
|
50f89d |
signals whether there are threads waiting on
|
|
|
50f89d |
this futex. */
|
|
|
50f89d |
- atomic_store_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers_futex, wf);
|
|
|
50f89d |
+ atomic_store_relaxed (&rwlock->__data.__writers_futex, wf);
|
|
|
50f89d |
}
|
|
|
50f89d |
}
|
|
|
50f89d |
/* If we timed out and we are not in a write phase, we can
|
|
|
50f89d |
@@ -857,8 +860,8 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* We are about to release WRLOCKED, so we must release
|
|
|
50f89d |
__writers_futex too; see the handling of
|
|
|
50f89d |
writer--writer hand-over above. */
|
|
|
50f89d |
- unsigned int wf = atomic_exchange_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__writers_futex, 0);
|
|
|
50f89d |
+ unsigned int wf
|
|
|
50f89d |
+ = atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0);
|
|
|
50f89d |
while ((r & PTHREAD_RWLOCK_WRPHASE) == 0)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* While we don't need to make anything from a
|
|
|
50f89d |
@@ -877,11 +880,11 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
/* Wake other writers. */
|
|
|
50f89d |
if ((wf & PTHREAD_RWLOCK_FUTEX_USED) != 0)
|
|
|
50f89d |
futex_wake (&rwlock->__data.__writers_futex,
|
|
|
50f89d |
- 1, private);
|
|
|
50f89d |
+ 1, private);
|
|
|
50f89d |
/* Wake waiting readers. */
|
|
|
50f89d |
if ((r & PTHREAD_RWLOCK_RWAITING) != 0)
|
|
|
50f89d |
futex_wake (&rwlock->__data.__readers,
|
|
|
50f89d |
- INT_MAX, private);
|
|
|
50f89d |
+ INT_MAX, private);
|
|
|
50f89d |
return ETIMEDOUT;
|
|
|
50f89d |
}
|
|
|
50f89d |
}
|
|
|
50f89d |
@@ -898,10 +901,9 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
atomic_thread_fence_acquire ();
|
|
|
50f89d |
/* We still need to wait for explicit hand-over, but we must
|
|
|
50f89d |
not use futex_wait anymore. */
|
|
|
50f89d |
- while ((atomic_load_relaxed
|
|
|
50f89d |
- (&rwlock->__data.__wrphase_futex)
|
|
|
50f89d |
- | PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
- == PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
+ while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex)
|
|
|
50f89d |
+ | PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
+ == PTHREAD_RWLOCK_FUTEX_USED)
|
|
|
50f89d |
{
|
|
|
50f89d |
/* TODO Back-off. */
|
|
|
50f89d |
}
|
|
|
50f89d |
@@ -915,12 +917,12 @@ __pthread_rwlock_wrlock_full (pthread_rwlock_t *rwlock,
|
|
|
50f89d |
if (ready)
|
|
|
50f89d |
break;
|
|
|
50f89d |
if ((atomic_load_acquire (&rwlock->__data.__readers)
|
|
|
50f89d |
- & PTHREAD_RWLOCK_WRPHASE) != 0)
|
|
|
50f89d |
+ & PTHREAD_RWLOCK_WRPHASE) != 0)
|
|
|
50f89d |
ready = true;
|
|
|
50f89d |
}
|
|
|
50f89d |
|
|
|
50f89d |
done:
|
|
|
50f89d |
atomic_store_relaxed (&rwlock->__data.__cur_writer,
|
|
|
50f89d |
- THREAD_GETMEM (THREAD_SELF, tid));
|
|
|
50f89d |
+ THREAD_GETMEM (THREAD_SELF, tid));
|
|
|
50f89d |
return 0;
|
|
|
50f89d |
}
|