Skip to content

Commit b69f3d2

Browse files
jartjart
committed
Optimize memory layout
1 parent 0305194 commit b69f3d2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+383
-347
lines changed

‎ape/ape.lds

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -714,8 +714,8 @@ ASSERT(!DEFINED(_start16) || REAL(_end) < 65536,
714714
ASSERT(IS2POW(ape_stack_memsz),
715715
"ape_stack_memsz must be a two power");
716716

717-
ASSERT(!(ape_stack_vaddr & (ape_stack_memsz - 1)),
718-
"ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000000000 - ape_stack_memsz);");
717+
ASSERT(ape_stack_vaddr % ape_stack_memsz == 0,
718+
"ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000040000 - ape_stack_memsz);");
719719

720720
ASSERT(ALIGNOF(.tdata) <= TLS_ALIGNMENT && ALIGNOF(.tbss) <= TLS_ALIGNMENT,
721721
"_Thread_local _Alignof can't exceed TLS_ALIGNMENT");

‎ape/relocations.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@
3636
*/
3737
#define REAL(x) ((x) - (IMAGE_BASE_VIRTUAL - IMAGE_BASE_REAL))
3838

39-
#if IMAGE_BASE_VIRTUAL % 0x200000 != 0
40-
#error "IMAGE_BASE_VIRTUAL must be 2mb aligned"
39+
#if IMAGE_BASE_VIRTUAL % 0x1000 != 0
40+
#error "IMAGE_BASE_VIRTUAL must be 4kb aligned"
4141
#endif
4242
#if IMAGE_BASE_PHYSICAL % 0x1000 != 0
4343
#error "IMAGE_BASE_PHYSICAL must be 4kb aligned"

‎libc/atomic.h

Lines changed: 24 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,8 @@
11
#ifndef COSMOPOLITAN_LIBC_ATOMIC_H_
22
#define COSMOPOLITAN_LIBC_ATOMIC_H_
3-
#include "libc/inttypes.h"
4-
#if !(__ASSEMBLER__ + __LINKER__ + 0)
5-
COSMOPOLITAN_C_START_
6-
7-
/**
8-
* @fileoverview C11 Atomic Types
9-
*
10-
* We supoprt C++ and old C compilers. It's recommended you use macros
11-
* like `_Atomic(int)` rather than `_Atomic int` or `atomic_int` since
12-
* we only define a portability macro for the syntax `_Atomic(T)`.
13-
*
14-
* @see libc/integral/c.inc
15-
* @see libc/intrin/atomic.h
16-
*/
173

184
#define atomic_bool _Atomic(_Bool)
19-
#define atomic_bool32 atomic_int32
5+
#define atomic_bool32 _Atomic(__INT32_TYPE__)
206
#define atomic_char _Atomic(char)
217
#define atomic_schar _Atomic(signed char)
228
#define atomic_uchar _Atomic(unsigned char)
@@ -28,29 +14,29 @@ COSMOPOLITAN_C_START_
2814
#define atomic_ulong _Atomic(unsigned long)
2915
#define atomic_llong _Atomic(long long)
3016
#define atomic_ullong _Atomic(unsigned long long)
31-
#define atomic_char16_t _Atomic(char16_t)
32-
#define atomic_char32_t _Atomic(char32_t)
33-
#define atomic_wchar_t _Atomic(wchar_t)
34-
#define atomic_intptr_t _Atomic(intptr_t)
35-
#define atomic_uintptr_t _Atomic(uintptr_t)
36-
#define atomic_size_t _Atomic(size_t)
37-
#define atomic_ptrdiff_t _Atomic(ptrdiff_t)
38-
#define atomic_int_fast8_t _Atomic(int_fast8_t)
39-
#define atomic_uint_fast8_t _Atomic(uint_fast8_t)
40-
#define atomic_int_fast16_t _Atomic(int_fast16_t)
41-
#define atomic_uint_fast16_t _Atomic(uint_fast16_t)
42-
#define atomic_int_fast32_t _Atomic(int_fast32_t)
43-
#define atomic_uint_fast32_t _Atomic(uint_fast32_t)
44-
#define atomic_int_fast64_t _Atomic(int_fast64_t)
45-
#define atomic_uint_fast64_t _Atomic(uint_fast64_t)
46-
#define atomic_int_least8_t _Atomic(int_least8_t)
47-
#define atomic_uint_least8_t _Atomic(uint_least8_t)
48-
#define atomic_int_least16_t _Atomic(int_least16_t)
49-
#define atomic_uint_least16_t _Atomic(uint_least16_t)
50-
#define atomic_int_least32_t _Atomic(int_least32_t)
51-
#define atomic_uint_least32_t _Atomic(uint_least32_t)
52-
#define atomic_int_least64_t _Atomic(int_least64_t)
53-
#define atomic_uint_least64_t _Atomic(uint_least64_t)
17+
#define atomic_char16_t _Atomic(__CHAR16_TYPE__)
18+
#define atomic_char32_t _Atomic(__CHAR32_TYPE__)
19+
#define atomic_wchar_t _Atomic(__WCHAR_TYPE__)
20+
#define atomic_intptr_t _Atomic(__INTPTR_TYPE__)
21+
#define atomic_uintptr_t _Atomic(__UINTPTR_TYPE__)
22+
#define atomic_size_t _Atomic(__SIZE_TYPE__)
23+
#define atomic_ptrdiff_t _Atomic(__PTRDIFF_TYPE__)
24+
#define atomic_int_fast8_t _Atomic(__INT_FAST8_TYPE__)
25+
#define atomic_uint_fast8_t _Atomic(__UINT_FAST8_TYPE__)
26+
#define atomic_int_fast16_t _Atomic(__INT_FAST16_TYPE__)
27+
#define atomic_uint_fast16_t _Atomic(__UINT_FAST16_TYPE__)
28+
#define atomic_int_fast32_t _Atomic(__INT_FAST32_TYPE__)
29+
#define atomic_uint_fast32_t _Atomic(__UINT_FAST32_TYPE__)
30+
#define atomic_int_fast64_t _Atomic(__INT_FAST64_TYPE__)
31+
#define atomic_uint_fast64_t _Atomic(__UINT_FAST64_TYPE__)
32+
#define atomic_int_least8_t _Atomic(__INT_LEAST8_TYPE__)
33+
#define atomic_uint_least8_t _Atomic(__UINT_LEAST8_TYPE__)
34+
#define atomic_int_least16_t _Atomic(__INT_LEAST16_TYPE__)
35+
#define atomic_uint_least16_t _Atomic(__UINT_LEAST16_TYPE__)
36+
#define atomic_int_least32_t _Atomic(__INT_LEAST32_TYPE__)
37+
#define atomic_uint_least32_t _Atomic(__UINT_LEAST32_TYPE__)
38+
#define atomic_int_least64_t _Atomic(__INT_LEAST64_TYPE__)
39+
#define atomic_uint_least64_t _Atomic(__UINT_LEAST64_TYPE__)
5440

5541
#ifdef __CLANG_ATOMIC_BOOL_LOCK_FREE
5642
#define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE
@@ -76,6 +62,4 @@ COSMOPOLITAN_C_START_
7662
#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE
7763
#endif
7864

79-
COSMOPOLITAN_C_END_
80-
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
8165
#endif /* COSMOPOLITAN_LIBC_ATOMIC_H_ */

‎libc/calls/munmap-sysv.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ int sys_munmap(void *p, size_t n) {
3838
} else {
3939
rc = sys_munmap_metal(p, n);
4040
}
41-
KERNTRACE("sys_munmap(%p%s, %'zu) → %d", p, DescribeFrame((intptr_t)p >> 16),
42-
n, rc);
41+
KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
42+
DescribeFrame((intptr_t)p >> 16), n, rc);
4343
return rc;
4444
}

‎libc/calls/weirdtypes.h

Lines changed: 2 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -37,34 +37,8 @@
3737
#define uid_t uint32_t
3838
#define rlim_t uint64_t /* int64_t on bsd */
3939

40-
typedef __INT_FAST8_TYPE__ int_fast8_t;
41-
typedef __UINT_FAST8_TYPE__ uint_fast8_t;
42-
typedef __INT_FAST16_TYPE__ int_fast16_t;
43-
typedef __UINT_FAST16_TYPE__ uint_fast16_t;
44-
typedef __INT_FAST32_TYPE__ int_fast32_t;
45-
typedef __UINT_FAST32_TYPE__ uint_fast32_t;
46-
typedef __INT_FAST64_TYPE__ int_fast64_t;
47-
typedef __UINT_FAST64_TYPE__ uint_fast64_t;
48-
49-
#define TIME_T_MAX __INT64_MAX__
50-
#define UINT_FAST64_MAX __UINT_FAST64_MAX__
51-
#define UINT_FAST8_MAX __UINT_FAST8_MAX__
52-
#define INT_FAST32_MAX __INT_FAST32_MAX__
53-
#define INT_FAST16_MAX __INT_FAST16_MAX__
54-
#define UINT_FAST32_MAX __UINT_FAST32_MAX__
55-
#define INT_FAST8_MAX __INT_FAST8_MAX__
56-
#define INT_FAST64_MAX __INT_FAST64_MAX__
57-
#define UINT_FAST16_MAX __UINT_FAST16_MAX__
58-
59-
#define TIME_T_MIN (-TIME_T_MAX - 1)
60-
#define UINT_FAST64_MIN (-UINT_FAST64_MAX - 1)
61-
#define UINT_FAST8_MIN (-UINT_FAST8_MAX - 1)
62-
#define INT_FAST32_MIN (-INT_FAST32_MAX - 1)
63-
#define INT_FAST16_MIN (-INT_FAST16_MAX - 1)
64-
#define UINT_FAST32_MIN (-UINT_FAST32_MAX - 1)
65-
#define INT_FAST8_MIN (-INT_FAST8_MAX - 1)
66-
#define INT_FAST64_MIN (-INT_FAST64_MAX - 1)
67-
#define UINT_FAST16_MIN (-UINT_FAST16_MAX - 1)
40+
#define TIME_T_MAX __INT64_MAX__
41+
#define TIME_T_MIN (-TIME_T_MAX - 1)
6842

6943
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
7044
#endif /* COSMOPOLITAN_LIBC_CALLS_WEIRDTYPES_H_ */

‎libc/intrin/describeframe.c

Lines changed: 42 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,32 +16,62 @@
1616
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
1717
│ PERFORMANCE OF THIS SOFTWARE. │
1818
╚─────────────────────────────────────────────────────────────────────────────*/
19+
#include "libc/dce.h"
1920
#include "libc/intrin/describeflags.internal.h"
2021
#include "libc/intrin/kprintf.h"
2122
#include "libc/macros.internal.h"
2223
#include "libc/runtime/memtrack.internal.h"
2324
#include "libc/runtime/runtime.h"
25+
#include "libc/runtime/winargs.internal.h"
2426

2527
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
2628
#define UNSHADOW(x) ((int64_t)(MAX(0, (x)-0x7fff8000)) << 3)
2729
#define FRAME(x) ((int)((x) >> 16))
2830

29-
const char *(DescribeFrame)(char buf[32], int x) {
30-
/* asan runtime depends on this function */
31-
char *p;
32-
if (IsShadowFrame(x)) {
33-
ksnprintf(buf, 32, " shadow=%.8x", FRAME(UNSHADOW(ADDR(x))));
34-
return buf;
35-
return " shadow ";
31+
static const char *GetFrameName(int x) {
32+
if (!x) {
33+
return "null";
34+
} else if (IsShadowFrame(x)) {
35+
return "shadow";
3636
} else if (IsAutoFrame(x)) {
37-
return " automap";
37+
return "automap";
3838
} else if (IsFixedFrame(x)) {
39-
return " fixed ";
39+
return "fixed";
4040
} else if (IsArenaFrame(x)) {
41-
return " arena ";
41+
return "arena";
4242
} else if (IsStaticStackFrame(x)) {
43-
return " stack ";
43+
return "stack";
44+
} else if (IsGfdsFrame(x)) {
45+
return "g_fds";
46+
} else if (IsZiposFrame(x)) {
47+
return "zipos";
48+
} else if (IsNsyncFrame(x)) {
49+
return "nsync";
50+
} else if (IsMemtrackFrame(x)) {
51+
return "memtrack";
52+
} else if (IsOldStackFrame(x)) {
53+
return "oldstack";
54+
} else if (IsWindows() &&
55+
(((GetStaticStackAddr(0) + GetStackSize()) >> 16) <= x &&
56+
x <= ((GetStaticStackAddr(0) + GetStackSize() +
57+
sizeof(struct WinArgs) - 1) >>
58+
16))) {
59+
return "winargs";
60+
} else if ((int)((intptr_t)_base >> 16) <= x &&
61+
x <= (int)(((intptr_t)_end - 1) >> 16)) {
62+
return "image";
63+
} else {
64+
return "unknown";
65+
}
66+
}
67+
68+
const char *(DescribeFrame)(char buf[32], int x) {
69+
char *p;
70+
if (IsShadowFrame(x)) {
71+
ksnprintf(buf, 64, "%s %s %.8x", GetFrameName(x),
72+
GetFrameName(FRAME(UNSHADOW(ADDR(x)))), FRAME(UNSHADOW(ADDR(x))));
73+
return buf;
4474
} else {
45-
return "";
75+
return GetFrameName(x);
4676
}
4777
}

‎libc/intrin/describemapping.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) {
4949
DescribeProt(p, prot);
5050
p[3] = DescribeMapType(flags);
5151
p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-';
52-
p[6] = (flags & MAP_FIXED) ? 'F' : '-';
53-
p[7] = 0;
52+
p[5] = (flags & MAP_FIXED) ? 'F' : '-';
53+
p[6] = 0;
5454
return p;
5555
}

‎libc/intrin/directmap.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,9 @@ struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
4646
} else {
4747
d = sys_mmap_nt(addr, size, prot, flags, fd, off);
4848
}
49-
KERNTRACE("sys_mmap(%.12p%s, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m", addr,
50-
DescribeFrame((intptr_t)addr >> 16), size, DescribeProtFlags(prot),
51-
DescribeMapFlags(flags), fd, off, d.addr, d.maphandle);
49+
KERNTRACE("sys_mmap(%.12p /* %s */, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m",
50+
addr, DescribeFrame((intptr_t)addr >> 16), size,
51+
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, d.addr,
52+
d.maphandle);
5253
return d;
5354
}

‎libc/intrin/isatleastwindows10.greg.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
╚─────────────────────────────────────────────────────────────────────────────*/
1919
#include "libc/assert.h"
2020
#include "libc/dce.h"
21-
#include "libc/nt/enum/version.h"
2221
#include "libc/nt/version.h"
2322

2423
/**

‎libc/intrin/memtrack.greg.c

Lines changed: 28 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,20 @@
3737
#include "libc/sysv/consts/prot.h"
3838
#include "libc/sysv/errfuns.h"
3939

40+
#if IsModeDbg()
41+
#define ASSERT_MEMTRACK() \
42+
if (!AreMemoryIntervalsOk(mm)) { \
43+
PrintMemoryIntervals(2, mm); \
44+
notpossible; \
45+
}
46+
#else
47+
#define ASSERT_MEMTRACK()
48+
#endif
49+
4050
static void *MoveMemoryIntervals(struct MemoryInterval *d,
4151
const struct MemoryInterval *s, int n) {
42-
// asan runtime depends on this function
4352
int i;
44-
assert(n >= 0);
53+
if (n < 0) unreachable;
4554
if (d > s) {
4655
for (i = n; i--;) {
4756
d[i] = s[i];
@@ -55,9 +64,8 @@ static void *MoveMemoryIntervals(struct MemoryInterval *d,
5564
}
5665

5766
static void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i, int n) {
58-
// asan runtime depends on this function
59-
assert(i >= 0);
60-
assert(i + n <= mm->i);
67+
if (i < 0) unreachable;
68+
if (i + n > mm->i) unreachable;
6169
MoveMemoryIntervals(mm->p + i, mm->p + i + n, mm->i - (i + n));
6270
mm->i -= n;
6371
}
@@ -96,19 +104,14 @@ static bool ExtendMemoryIntervals(struct MemoryIntervals *mm) {
96104
if (!dm.addr) return false;
97105
mm->n = (size + gran) / sizeof(*mm->p);
98106
}
99-
#if IsModeDbg()
100-
assert(AreMemoryIntervalsOk(mm));
101-
#endif
107+
ASSERT_MEMTRACK();
102108
return true;
103109
}
104110

105111
int CreateMemoryInterval(struct MemoryIntervals *mm, int i) {
106-
// asan runtime depends on this function
107-
int rc;
108-
rc = 0;
109-
assert(i >= 0);
110-
assert(i <= mm->i);
111-
assert(mm->n >= 0);
112+
if (i < 0) unreachable;
113+
if (i > mm->i) unreachable;
114+
if (mm->n < 0) unreachable;
112115
if (UNLIKELY(mm->i == mm->n) && !ExtendMemoryIntervals(mm)) return enomem();
113116
MoveMemoryIntervals(mm->p + i + 1, mm->p + i, mm->i++ - i);
114117
return 0;
@@ -126,12 +129,9 @@ static int PunchHole(struct MemoryIntervals *mm, int x, int y, int i) {
126129
int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
127130
void wf(struct MemoryIntervals *, int, int)) {
128131
unsigned l, r;
129-
#if IsModeDbg()
130-
assert(y >= x);
131-
assert(AreMemoryIntervalsOk(mm));
132-
#endif
132+
ASSERT_MEMTRACK();
133+
if (y < x) unreachable;
133134
if (!mm->i) return 0;
134-
135135
// binary search for the lefthand side
136136
l = FindMemoryInterval(mm, x);
137137
if (l == mm->i) return 0;
@@ -140,8 +140,8 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
140140
// binary search for the righthand side
141141
r = FindMemoryInterval(mm, y);
142142
if (r == mm->i || (r > l && y < mm->p[r].x)) --r;
143-
assert(r >= l);
144-
assert(x <= mm->p[r].y);
143+
if (r < l) unreachable;
144+
if (x > mm->p[r].y) unreachable;
145145

146146
// remove the middle of an existing map
147147
//
@@ -162,11 +162,11 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
162162
// ----|mmmm|----------------- after
163163
//
164164
if (x > mm->p[l].x && x <= mm->p[l].y) {
165-
assert(y >= mm->p[l].y);
165+
if (y < mm->p[l].y) unreachable;
166166
if (IsWindows()) return einval();
167167
mm->p[l].size -= (size_t)(mm->p[l].y - (x - 1)) * FRAMESIZE;
168168
mm->p[l].y = x - 1;
169-
assert(mm->p[l].x <= mm->p[l].y);
169+
if (mm->p[l].x > mm->p[l].y) unreachable;
170170
++l;
171171
}
172172

@@ -177,11 +177,11 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
177177
// ---------------|mm|-------- after
178178
//
179179
if (y >= mm->p[r].x && y < mm->p[r].y) {
180-
assert(x <= mm->p[r].x);
180+
if (x > mm->p[r].x) unreachable;
181181
if (IsWindows()) return einval();
182182
mm->p[r].size -= (size_t)((y + 1) - mm->p[r].x) * FRAMESIZE;
183183
mm->p[r].x = y + 1;
184-
assert(mm->p[r].x <= mm->p[r].y);
184+
if (mm->p[r].x > mm->p[r].y) unreachable;
185185
--r;
186186
}
187187

@@ -197,16 +197,9 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
197197
int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
198198
int prot, int flags, bool readonlyfile, bool iscow,
199199
long offset, long size) {
200-
// asan runtime depends on this function
201200
unsigned i;
202-
#if IsModeDbg()
203-
assert(y >= x);
204-
if (!AreMemoryIntervalsOk(mm)) {
205-
PrintMemoryIntervals(2, mm);
206-
}
207-
assert(AreMemoryIntervalsOk(mm));
208-
#endif
209-
201+
ASSERT_MEMTRACK();
202+
if (y < x) unreachable;
210203
i = FindMemoryInterval(mm, x);
211204

212205
// try to extend the righthand side of the lefthand entry
@@ -249,5 +242,6 @@ int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
249242
mm->p[i].iscow = iscow;
250243
mm->p[i].readonlyfile = readonlyfile;
251244
}
245+
252246
return 0;
253247
}

0 commit comments

Comments
 (0)