Skip to content

Commit 529cb48

Browse files
jartjart
committed
Improve dlopen() on Apple Silicon
- Introduce MAP_JIT which is zero on other platforms - Invent __jit_begin() and __jit_end() which wrap Apple's APIs - Runtime dispatch to sys_icache_invalidate() in __clear_cache()
1 parent 7a9e176 commit 529cb48

File tree

20 files changed

+120
-117
lines changed

20 files changed

+120
-117
lines changed

‎ape/ape-m1.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636

3737
#define pagesz 16384
3838
#define SYSLIB_MAGIC ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24)
39-
#define SYSLIB_VERSION 7
39+
#define SYSLIB_VERSION 8
4040

4141
struct Syslib {
4242
int magic;
@@ -891,6 +891,7 @@ int main(int argc, char **argv, char **envp) {
891891
M->lib.pthread_jit_write_protect_supported_np =
892892
pthread_jit_write_protect_supported_np;
893893
M->lib.pthread_jit_write_protect_np = pthread_jit_write_protect_np_workaround;
894+
M->lib.sys_icache_invalidate = sys_icache_invalidate;
894895
M->lib.pthread_create = pthread_create;
895896
M->lib.pthread_exit = pthread_exit;
896897
M->lib.pthread_kill = pthread_kill;

‎libc/calls/munmap-sysv.c

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "libc/intrin/describeflags.internal.h"
2222
#include "libc/intrin/directmap.internal.h"
2323
#include "libc/intrin/strace.internal.h"
24+
#include "libc/runtime/syslib.internal.h"
2425

2526
/**
2627
* Unmaps memory directly with system.
@@ -32,10 +33,12 @@
3233
*/
3334
int sys_munmap(void *p, size_t n) {
3435
int rc;
35-
if (!IsMetal()) {
36-
rc = __sys_munmap(p, n);
37-
} else {
36+
if (IsXnuSilicon()) {
37+
rc = _sysret(__syslib->__munmap(p, n));
38+
} else if (IsMetal()) {
3839
rc = sys_munmap_metal(p, n);
40+
} else {
41+
rc = __sys_munmap(p, n);
3942
}
4043
KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
4144
DescribeFrame((intptr_t)p >> 16), n, rc);

‎libc/calls/syscall-sysv.internal.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ i32 __sys_fcntl_cp(i32, i32, ...);
2020
i32 __sys_fstat(i32, void *);
2121
i32 __sys_fstatat(i32, const char *, void *, i32);
2222
i32 __sys_gettid(i64 *);
23+
i32 __sys_mprotect(void *, u64, i32);
2324
i32 __sys_munmap(void *, u64);
2425
i32 __sys_openat(i32, const char *, i32, u32);
2526
i32 __sys_openat_nc(i32, const char *, i32, u32);

‎libc/dlopen/dlopen.c

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -424,12 +424,12 @@ static char *dlerror_set(const char *str) {
424424
return dlerror_buf;
425425
}
426426

427-
static char *foreign_alloc_block(void) {
427+
static dontinline char *foreign_alloc_block(void) {
428428
char *p = 0;
429429
size_t sz = 65536;
430430
if (!IsWindows()) {
431431
p = __sys_mmap(0, sz, PROT_READ | PROT_WRITE | PROT_EXEC,
432-
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, 0);
432+
MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT, -1, 0, 0);
433433
if (p == MAP_FAILED) {
434434
p = 0;
435435
}
@@ -448,7 +448,7 @@ static char *foreign_alloc_block(void) {
448448
return p;
449449
}
450450

451-
static void *foreign_alloc(size_t n) {
451+
static dontinline void *foreign_alloc(size_t n) {
452452
void *res;
453453
static char *block;
454454
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@@ -520,11 +520,14 @@ static void *foreign_thunk_sysv(void *func) {
520520
*p++ = 0xff;
521521
*p++ = 0xe2;
522522
#elif defined(__aarch64__)
523-
if (!(p = code = foreign_alloc(36))) return 0; // 16 + 16 + 4 = 36
524-
p = movimm(p, 5, (uintptr_t)func);
525-
p = movimm(p, 10, (uintptr_t)foreign_tramp);
526-
*(uint32_t *)p = 0xd61f0140; // br x10
527-
__clear_cache(code, p + 4);
523+
__jit_begin();
524+
if ((p = code = foreign_alloc(36))) {
525+
p = movimm(p, 5, (uintptr_t)func);
526+
p = movimm(p, 10, (uintptr_t)foreign_tramp);
527+
*(uint32_t *)p = 0xd61f0140; // br x10
528+
__clear_cache(code, p + 4);
529+
}
530+
__jit_end();
528531
#else
529532
#error "unsupported architecture"
530533
#endif

‎libc/dlopen/dlopen.mk

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ LIBC_DLOPEN_A_DIRECTDEPS = \
3333
LIBC_RUNTIME \
3434
LIBC_SYSV \
3535
LIBC_SYSV_CALLS \
36-
LIBC_STR \
37-
THIRD_PARTY_COMPILER_RT
36+
LIBC_STR
3837

3938
LIBC_DLOPEN_A_DEPS := \
4039
$(call uniq,$(foreach x,$(LIBC_DLOPEN_A_DIRECTDEPS),$($(x))))

‎libc/intrin/directmap.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@
1717
│ PERFORMANCE OF THIS SOFTWARE. │
1818
╚─────────────────────────────────────────────────────────────────────────────*/
1919
#include "libc/calls/syscall-sysv.internal.h"
20+
#include "libc/dce.h"
2021
#include "libc/errno.h"
2122
#include "libc/intrin/describeflags.internal.h"
2223
#include "libc/intrin/directmap.internal.h"
2324
#include "libc/intrin/strace.internal.h"
2425
#include "libc/nt/runtime.h"
2526
#include "libc/runtime/memtrack.internal.h"
27+
#include "libc/runtime/syslib.internal.h"
2628

2729
/**
2830
* Obtains memory mapping directly from system.
@@ -37,7 +39,11 @@
3739
struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
3840
int64_t off) {
3941
struct DirectMap d;
40-
if (!IsWindows() && !IsMetal()) {
42+
if (IsXnuSilicon()) {
43+
long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
44+
d.maphandle = kNtInvalidHandleValue;
45+
d.addr = (void *)p;
46+
} else if (!IsWindows() && !IsMetal()) {
4147
d.addr = __sys_mmap(addr, size, prot, flags, fd, off, off);
4248
d.maphandle = kNtInvalidHandleValue;
4349
} else if (IsMetal()) {

‎libc/intrin/mprotect-sysv.c

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
2+
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
3+
╞══════════════════════════════════════════════════════════════════════════════╡
4+
│ Copyright 2023 Justine Alexandra Roberts Tunney │
5+
│ │
6+
│ Permission to use, copy, modify, and/or distribute this software for │
7+
│ any purpose with or without fee is hereby granted, provided that the │
8+
│ above copyright notice and this permission notice appear in all copies. │
9+
│ │
10+
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
11+
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
12+
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
13+
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
14+
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
15+
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
16+
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
17+
│ PERFORMANCE OF THIS SOFTWARE. │
18+
╚─────────────────────────────────────────────────────────────────────────────*/
19+
#include "libc/calls/syscall-sysv.internal.h"
20+
#include "libc/dce.h"
21+
#include "libc/runtime/syslib.internal.h"
22+
23+
int sys_mprotect(void *data, size_t size, int prot) {
24+
if (IsXnuSilicon()) {
25+
return _sysret(__syslib->__mprotect(data, size, prot));
26+
} else {
27+
return __sys_mprotect(data, size, prot);
28+
}
29+
}

‎libc/runtime/jit.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
2+
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
3+
╞══════════════════════════════════════════════════════════════════════════════╡
4+
│ Copyright 2023 Justine Alexandra Roberts Tunney │
5+
│ │
6+
│ Permission to use, copy, modify, and/or distribute this software for │
7+
│ any purpose with or without fee is hereby granted, provided that the │
8+
│ above copyright notice and this permission notice appear in all copies. │
9+
│ │
10+
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
11+
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
12+
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
13+
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
14+
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
15+
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
16+
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
17+
│ PERFORMANCE OF THIS SOFTWARE. │
18+
╚─────────────────────────────────────────────────────────────────────────────*/
19+
#include "libc/dce.h"
20+
#include "libc/runtime/runtime.h"
21+
#include "libc/runtime/syslib.internal.h"
22+
23+
void __jit_begin(void) {
24+
if (IsXnuSilicon()) {
25+
if (__syslib->__pthread_jit_write_protect_supported_np()) {
26+
__syslib->__pthread_jit_write_protect_np(false);
27+
}
28+
}
29+
}
30+
31+
void __jit_end(void) {
32+
if (IsXnuSilicon()) {
33+
if (__syslib->__pthread_jit_write_protect_supported_np()) {
34+
__syslib->__pthread_jit_write_protect_np(true);
35+
}
36+
}
37+
}

‎libc/runtime/runtime.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ bool32 _isheap(void *);
115115
/* code morphing */
116116
void __morph_begin(void);
117117
void __morph_end(void);
118+
void __jit_begin(void);
119+
void __jit_end(void);
118120
void __clear_cache(void *, void *);
119121
/* portability */
120122
int NtGetVersion(void) pureconst;

‎libc/runtime/runtime.mk

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ LIBC_RUNTIME_A_DIRECTDEPS = \
4545
LIBC_STR \
4646
LIBC_SYSV \
4747
LIBC_SYSV_CALLS \
48+
THIRD_PARTY_COMPILER_RT \
4849
THIRD_PARTY_NSYNC \
4950
THIRD_PARTY_PUFF \
5051
THIRD_PARTY_XED

0 commit comments

Comments
 (0)