Skip to content

Commit e398f38

Browse files
jartjart
committed
Make more improvements to threads and mappings
- NetBSD should now have faster synchronization - POSIX barriers may now be shared across processes - An edge case with memory map tracking has been fixed - Grand Central Dispatch is no longer used on MacOS ARM64 - POSIX mutexes in normal mode now use futexes across processes
1 parent 2187d6d commit e398f38

20 files changed

+558
-163
lines changed

‎libc/intrin/cp.c

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,9 @@
2626

2727
int begin_cancelation_point(void) {
2828
int state = 0;
29-
struct CosmoTib *tib;
30-
struct PosixThread *pt;
3129
if (__tls_enabled) {
32-
tib = __get_tls();
33-
if ((pt = (struct PosixThread *)tib->tib_pthread)) {
30+
struct PosixThread *pt;
31+
if ((pt = _pthread_self())) {
3432
state = pt->pt_flags & PT_INCANCEL;
3533
pt->pt_flags |= PT_INCANCEL;
3634
}
@@ -39,11 +37,9 @@ int begin_cancelation_point(void) {
3937
}
4038

4139
void end_cancelation_point(int state) {
42-
struct CosmoTib *tib;
43-
struct PosixThread *pt;
4440
if (__tls_enabled) {
45-
tib = __get_tls();
46-
if ((pt = (struct PosixThread *)tib->tib_pthread)) {
41+
struct PosixThread *pt;
42+
if ((pt = _pthread_self())) {
4743
pt->pt_flags &= ~PT_INCANCEL;
4844
pt->pt_flags |= state;
4945
}

‎libc/intrin/maps.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
#include "libc/thread/tls2.internal.h"
77
COSMOPOLITAN_C_START_
88

9+
#define MAPS_RETRY ((void *)-1)
10+
911
#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e)
1012

1113
struct Map {

‎libc/intrin/mmap.c

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ static int __muntrack(char *addr, size_t size, int pagesz,
120120
struct Map *map;
121121
struct Map *next;
122122
struct Map *floor;
123+
StartOver:
123124
floor = __maps_floor(addr);
124125
for (map = floor; map && map->addr <= addr + size; map = next) {
125126
next = __maps_next(map);
@@ -148,6 +149,8 @@ static int __muntrack(char *addr, size_t size, int pagesz,
148149
ASSERT(left > 0);
149150
struct Map *leftmap;
150151
if ((leftmap = __maps_alloc())) {
152+
if (leftmap == MAPS_RETRY)
153+
goto StartOver;
151154
map->addr += left;
152155
map->size = right;
153156
if (!(map->flags & MAP_ANONYMOUS))
@@ -167,6 +170,8 @@ static int __muntrack(char *addr, size_t size, int pagesz,
167170
size_t right = map_addr + map_size - addr;
168171
struct Map *rightmap;
169172
if ((rightmap = __maps_alloc())) {
173+
if (rightmap == MAPS_RETRY)
174+
goto StartOver;
170175
map->size = left;
171176
__maps.pages -= (right + pagesz - 1) / pagesz;
172177
rightmap->addr = addr;
@@ -184,8 +189,14 @@ static int __muntrack(char *addr, size_t size, int pagesz,
184189
size_t right = map_size - middle - left;
185190
struct Map *leftmap;
186191
if ((leftmap = __maps_alloc())) {
192+
if (leftmap == MAPS_RETRY)
193+
goto StartOver;
187194
struct Map *middlemap;
188195
if ((middlemap = __maps_alloc())) {
196+
if (middlemap == MAPS_RETRY) {
197+
__maps_free(leftmap);
198+
goto StartOver;
199+
}
189200
leftmap->addr = map_addr;
190201
leftmap->size = left;
191202
leftmap->off = map->off;
@@ -204,6 +215,7 @@ static int __muntrack(char *addr, size_t size, int pagesz,
204215
*deleted = middlemap;
205216
__maps_check();
206217
} else {
218+
__maps_free(leftmap);
207219
rc = -1;
208220
}
209221
} else {
@@ -304,12 +316,11 @@ struct Map *__maps_alloc(void) {
304316
map->flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK;
305317
map->hand = sys.maphandle;
306318
__maps_lock();
307-
__maps_insert(map++);
319+
__maps_insert(map);
308320
__maps_unlock();
309-
map->addr = MAP_FAILED;
310-
for (int i = 1; i < gransz / sizeof(struct Map) - 1; ++i)
321+
for (int i = 1; i < gransz / sizeof(struct Map); ++i)
311322
__maps_free(map + i);
312-
return map;
323+
return MAPS_RETRY;
313324
}
314325

315326
static int __munmap(char *addr, size_t size) {
@@ -396,33 +407,39 @@ void *__maps_pickaddr(size_t size) {
396407
static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
397408
int64_t off, int pagesz, int gransz) {
398409

410+
// allocate Map object
411+
struct Map *map;
412+
do {
413+
if (!(map = __maps_alloc()))
414+
return MAP_FAILED;
415+
} while (map == MAPS_RETRY);
416+
399417
// polyfill nuances of fixed mappings
400418
int sysflags = flags;
401419
bool noreplace = false;
402420
bool should_untrack = false;
403421
if (flags & MAP_FIXED_NOREPLACE) {
404-
if (flags & MAP_FIXED)
422+
if (flags & MAP_FIXED) {
423+
__maps_free(map);
405424
return (void *)einval();
425+
}
406426
sysflags &= ~MAP_FIXED_NOREPLACE;
407427
if (IsLinux()) {
408428
noreplace = true;
409429
sysflags |= MAP_FIXED_NOREPLACE_linux;
410430
} else if (IsFreebsd() || IsNetbsd()) {
411431
sysflags |= MAP_FIXED;
412-
if (__maps_overlaps(addr, size, pagesz))
432+
if (__maps_overlaps(addr, size, pagesz)) {
433+
__maps_free(map);
413434
return (void *)eexist();
435+
}
414436
} else {
415437
noreplace = true;
416438
}
417439
} else if (flags & MAP_FIXED) {
418440
should_untrack = true;
419441
}
420442

421-
// allocate Map object
422-
struct Map *map;
423-
if (!(map = __maps_alloc()))
424-
return MAP_FAILED;
425-
426443
// remove mapping we blew away
427444
if (IsWindows() && should_untrack)
428445
__munmap(addr, size);
@@ -572,23 +589,27 @@ static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size,
572589
return (void *)einval();
573590
}
574591

592+
// allocate object for tracking new mapping
593+
struct Map *map;
594+
do {
595+
if (!(map = __maps_alloc()))
596+
return (void *)enomem();
597+
} while (map == MAPS_RETRY);
598+
575599
// check old interval is fully contained within one mapping
576600
struct Map *old_map;
577601
if (!(old_map = __maps_floor(old_addr)) ||
578602
old_addr + old_size > old_map->addr + PGUP(old_map->size) ||
579-
old_addr < old_map->addr)
603+
old_addr < old_map->addr) {
604+
__maps_free(map);
580605
return (void *)efault();
606+
}
581607

582608
// save old properties
583609
int old_off = old_map->off;
584610
int old_prot = old_map->prot;
585611
int old_flags = old_map->flags;
586612

587-
// allocate object for tracking new mapping
588-
struct Map *map;
589-
if (!(map = __maps_alloc()))
590-
return (void *)enomem();
591-
592613
// netbsd mremap fixed returns enoent rather than unmapping old pages
593614
if (IsNetbsd() && (flags & MREMAP_FIXED))
594615
if (__munmap(new_addr, new_size)) {

‎libc/intrin/mprotect.c

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ int __mprotect(char *addr, size_t size, int prot) {
7575
return edeadlk();
7676
}
7777
struct Map *map, *floor;
78+
StartOver:
7879
floor = __maps_floor(addr);
7980
for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) {
8081
char *map_addr = map->addr;
@@ -93,10 +94,12 @@ int __mprotect(char *addr, size_t size, int prot) {
9394
}
9495
} else if (addr <= map_addr) {
9596
// change lefthand side of mapping
96-
size_t left = PGUP(addr + size - map_addr);
97+
size_t left = addr + size - map_addr;
9798
size_t right = map_size - left;
9899
struct Map *leftmap;
99100
if ((leftmap = __maps_alloc())) {
101+
if (leftmap == MAPS_RETRY)
102+
goto StartOver;
100103
if (!__mprotect_chunk(map_addr, left, prot, false)) {
101104
leftmap->addr = map_addr;
102105
leftmap->size = left;
@@ -127,6 +130,8 @@ int __mprotect(char *addr, size_t size, int prot) {
127130
size_t right = map_addr + map_size - addr;
128131
struct Map *leftmap;
129132
if ((leftmap = __maps_alloc())) {
133+
if (leftmap == MAPS_RETRY)
134+
goto StartOver;
130135
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
131136
leftmap->addr = map_addr;
132137
leftmap->size = left;
@@ -159,8 +164,14 @@ int __mprotect(char *addr, size_t size, int prot) {
159164
size_t right = map_size - middle - left;
160165
struct Map *leftmap;
161166
if ((leftmap = __maps_alloc())) {
167+
if (leftmap == MAPS_RETRY)
168+
goto StartOver;
162169
struct Map *midlmap;
163170
if ((midlmap = __maps_alloc())) {
171+
if (midlmap == MAPS_RETRY) {
172+
__maps_free(leftmap);
173+
goto StartOver;
174+
}
164175
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
165176
leftmap->addr = map_addr;
166177
leftmap->size = left;

‎libc/intrin/pthread_mutex_lock.c

Lines changed: 64 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -27,41 +27,47 @@
2727
#include "libc/runtime/internal.h"
2828
#include "libc/thread/lock.h"
2929
#include "libc/thread/thread.h"
30+
#include "third_party/nsync/futex.internal.h"
3031
#include "third_party/nsync/mu.h"
3132

32-
static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
33-
int me;
33+
static void pthread_mutex_lock_naive(pthread_mutex_t *mutex, uint64_t word) {
3434
int backoff = 0;
35-
uint64_t word, lock;
36-
37-
// get current state of lock
38-
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
39-
40-
#if PTHREAD_USE_NSYNC
41-
// use fancy nsync mutex if possible
42-
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && //
43-
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
44-
_weaken(nsync_mu_lock)) {
45-
_weaken(nsync_mu_lock)((nsync_mu *)mutex);
46-
return 0;
35+
uint64_t lock;
36+
for (;;) {
37+
word = MUTEX_UNLOCK(word);
38+
lock = MUTEX_LOCK(word);
39+
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
40+
memory_order_acquire,
41+
memory_order_relaxed))
42+
return;
43+
backoff = pthread_delay_np(mutex, backoff);
4744
}
48-
#endif
45+
}
4946

50-
// implement barebones normal mutexes
51-
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
52-
for (;;) {
53-
word = MUTEX_UNLOCK(word);
54-
lock = MUTEX_LOCK(word);
55-
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
56-
memory_order_acquire,
57-
memory_order_relaxed))
58-
return 0;
59-
backoff = pthread_delay_np(mutex, backoff);
60-
}
47+
// see "take 3" algorithm in "futexes are tricky" by ulrich drepper
48+
// slightly improved to attempt acquiring multiple times b4 syscall
49+
static void pthread_mutex_lock_drepper(atomic_int *futex, char pshare) {
50+
int word;
51+
for (int i = 0; i < 4; ++i) {
52+
word = 0;
53+
if (atomic_compare_exchange_strong_explicit(
54+
futex, &word, 1, memory_order_acquire, memory_order_acquire))
55+
return;
56+
pthread_pause_np();
6157
}
58+
if (word == 1)
59+
word = atomic_exchange_explicit(futex, 2, memory_order_acquire);
60+
while (word > 0) {
61+
_weaken(nsync_futex_wait_)(futex, 2, pshare, 0);
62+
word = atomic_exchange_explicit(futex, 2, memory_order_acquire);
63+
}
64+
}
6265

63-
// implement recursive mutexes
64-
me = gettid();
66+
static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,
67+
uint64_t word) {
68+
uint64_t lock;
69+
int backoff = 0;
70+
int me = gettid();
6571
for (;;) {
6672
if (MUTEX_OWNER(word) == me) {
6773
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
@@ -91,6 +97,36 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
9197
}
9298
}
9399

100+
static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
101+
uint64_t word;
102+
103+
// get current state of lock
104+
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
105+
106+
#if PTHREAD_USE_NSYNC
107+
// use superior mutexes if possible
108+
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && //
109+
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
110+
_weaken(nsync_mu_lock)) {
111+
_weaken(nsync_mu_lock)((nsync_mu *)mutex);
112+
return 0;
113+
}
114+
#endif
115+
116+
// handle normal mutexes
117+
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
118+
if (_weaken(nsync_futex_wait_)) {
119+
pthread_mutex_lock_drepper(&mutex->_futex, MUTEX_PSHARED(word));
120+
} else {
121+
pthread_mutex_lock_naive(mutex, word);
122+
}
123+
return 0;
124+
}
125+
126+
// handle recursive and error checking mutexes
127+
return pthread_mutex_lock_recursive(mutex, word);
128+
}
129+
94130
/**
95131
* Locks mutex.
96132
*

0 commit comments

Comments
 (0)