2525#include "libc/dce.h"
2626#include "libc/errno.h"
2727#include "libc/intrin/atomic.h"
28+ #include "libc/intrin/describebacktrace.internal.h"
2829#include "libc/intrin/describeflags.internal.h"
2930#include "libc/intrin/directmap.internal.h"
3031#include "libc/intrin/dll.h"
32+ #include "libc/intrin/kprintf.h"
3133#include "libc/intrin/maps.h"
3234#include "libc/intrin/strace.internal.h"
3335#include "libc/intrin/weaken.h"
4446#include "libc/sysv/errfuns.h"
4547#include "libc/thread/thread.h"
4648
49+ #define MMDEBUG 0 // this code is too slow for openbsd/windows
4750#define WINBASE 0x100080040000 // TODO: Can we support Windows Vista again?
4851#define WINMAXX 0x200080000000
4952
5053#define MAP_FIXED_NOREPLACE_linux 0x100000
5154
5255#define PGUP (x ) (((x) + granularity - 1) & -granularity)
5356
57+ #if MMDEBUG
58+ #define ASSERT (x ) (void)0
59+ #else
60+ #define ASSERT (x ) \
61+ do { \
62+ if (!(x)) { \
63+ char bt[160]; \
64+ struct StackFrame *bp = __builtin_frame_address(0); \
65+ kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \
66+ kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp)); \
67+ __print_maps(); \
68+ _Exit(99); \
69+ } \
70+ } while (0)
71+ #endif
72+
5473static atomic_ulong rollo ;
5574
75+ static bool overlaps_existing_map (const char * addr , size_t size ) {
76+ int granularity = __granularity ();
77+ for (struct Map * map = __maps .maps ; map ; map = map -> next )
78+ if (MAX (addr , map -> addr ) <
79+ MIN (addr + PGUP (size ), map -> addr + PGUP (map -> size )))
80+ return true;
81+ return false;
82+ }
83+
84+ void __maps_check (void ) {
85+ #if MMDEBUG
86+ size_t maps = 0 ;
87+ size_t pages = 0 ;
88+ int granularity = getauxval (AT_PAGESZ );
89+ for (struct Map * map = __maps .maps ; map ; map = map -> next ) {
90+ ASSERT (map -> addr != MAP_FAILED );
91+ ASSERT (map -> size );
92+ pages += PGUP (map -> size ) / granularity ;
93+ maps += 1 ;
94+ }
95+ ASSERT (maps = __maps .count );
96+ ASSERT (pages == __maps .pages );
97+ for (struct Map * m1 = __maps .maps ; m1 ; m1 = m1 -> next )
98+ for (struct Map * m2 = m1 -> next ; m2 ; m2 = m2 -> next )
99+ ASSERT (MAX (m1 -> addr , m2 -> addr ) >=
100+ MIN (m1 -> addr + PGUP (m1 -> size ), m2 -> addr + PGUP (m2 -> size )));
101+ #endif
102+ }
103+
56104void __maps_free (struct Map * map ) {
105+ map -> next = 0 ;
106+ map -> size = 0 ;
107+ map -> addr = MAP_FAILED ;
108+ ASSERT (dll_is_alone (& map -> elem ));
57109 dll_make_last (& __maps .free , & map -> elem );
58110}
59111
60112void __maps_insert (struct Map * map ) {
61113 struct Map * last = __maps .maps ;
62- if (last && //
114+ int granularity = getauxval (AT_PAGESZ );
115+ __maps .pages += PGUP (map -> size ) / granularity ;
116+ if (last && !IsWindows () && //
63117 map -> addr == last -> addr + last -> size && //
118+ (map -> flags & MAP_ANONYMOUS ) && //
64119 map -> flags == last -> flags && //
65- map -> prot == last -> prot && //
66- map -> off == last -> off && //
67- map -> h == last -> h && //
68- map -> off == -1 ) {
120+ map -> prot == last -> prot ) {
121+ last -> size += map -> size ;
122+ dll_remove (& __maps .used , & last -> elem );
123+ dll_make_first (& __maps .used , & last -> elem );
124+ __maps_free (map );
125+ } else if (last && !IsWindows () && //
126+ map -> addr + map -> size == last -> addr && //
127+ (map -> flags & MAP_ANONYMOUS ) && //
128+ map -> flags == last -> flags && //
129+ map -> prot == last -> prot ) {
130+ last -> addr -= map -> size ;
69131 last -> size += map -> size ;
70132 dll_remove (& __maps .used , & last -> elem );
71133 dll_make_first (& __maps .used , & last -> elem );
@@ -74,7 +136,9 @@ void __maps_insert(struct Map *map) {
74136 dll_make_first (& __maps .used , & map -> elem );
75137 map -> next = __maps .maps ;
76138 __maps .maps = map ;
139+ ++ __maps .count ;
77140 }
141+ __maps_check ();
78142}
79143
80144struct Map * __maps_alloc (void ) {
@@ -104,21 +168,14 @@ struct Map *__maps_alloc(void) {
104168 return map ;
105169}
106170
107- static bool __overlaps_existing_map (const char * addr , size_t size ) {
108- for (struct Map * map = __maps .maps ; map ; map = map -> next ) {
109- if (MAX (addr , map -> addr ) < MIN (addr + size , map -> addr + map -> size ))
110- return true;
111- }
112- return false;
113- }
114-
115171static int __munmap_chunk (void * addr , size_t size ) {
116172 return sys_munmap (addr , size );
117173}
118174
119175static int __munmap (char * addr , size_t size , bool untrack_only ) {
120176
121177 // validate arguments
178+ int pagesz = getauxval (AT_PAGESZ );
122179 int granularity = __granularity ();
123180 if (((uintptr_t )addr & (granularity - 1 )) || //
124181 !size || (uintptr_t )addr + size < size )
@@ -127,7 +184,6 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
127184 // untrack and delete mapping
128185 int rc = 0 ;
129186 __maps_lock ();
130- // we can't call strace, kprintf, or nothing
131187StartOver :;
132188 struct Map * map = __maps .maps ;
133189 _Atomic(struct Map * ) * prev = & __maps .maps ;
@@ -141,12 +197,16 @@ StartOver:;
141197 // remove mapping completely
142198 dll_remove (& __maps .used , & map -> elem );
143199 * prev = next ;
144- map -> size = 0 ;
145- map -> addr = MAP_FAILED ;
200+ __maps . pages -= ( map_size + pagesz - 1 ) / pagesz ;
201+ __maps . count -= 1 ;
146202 if (untrack_only ) {
147203 __maps_free (map );
204+ __maps_check ();
148205 } else {
206+ __maps_unlock ();
149207 if (!IsWindows ()) {
208+ ASSERT (addr <= map_addr );
209+ ASSERT (map_addr + PGUP (map_size ) <= addr + PGUP (size ));
150210 if (__munmap_chunk (map_addr , map_size ))
151211 rc = -1 ;
152212 } else {
@@ -155,7 +215,9 @@ StartOver:;
155215 if (!CloseHandle (map -> h ))
156216 rc = -1 ;
157217 }
218+ __maps_lock ();
158219 __maps_free (map );
220+ __maps_check ();
159221 goto StartOver ;
160222 }
161223 map = next ;
@@ -167,25 +229,39 @@ StartOver:;
167229 rc = einval ();
168230 } else if (addr <= map_addr ) {
169231 // shave off lefthand side of mapping
170- size_t left = addr + size - map_addr ;
171- size_t right = map_addr + map_size - (addr + size );
232+ ASSERT (addr + size < map_addr + PGUP (map_size ));
233+ size_t left = PGUP (addr + size - map_addr );
234+ size_t right = map_size - left ;
235+ ASSERT (right > 0 );
236+ ASSERT (left > 0 );
172237 map -> addr += left ;
173238 map -> size = right ;
174239 if (map -> off != -1 )
175240 map -> off += left ;
241+ __maps .pages -= (left + pagesz - 1 ) / pagesz ;
242+ __maps_check ();
176243 if (!untrack_only ) {
244+ __maps_unlock ();
245+ ASSERT (addr <= map_addr );
246+ ASSERT (map_addr + PGUP (left ) <= addr + PGUP (size ));
177247 if (__munmap_chunk (map_addr , left ) == -1 )
178248 rc = -1 ;
249+ __maps_lock ();
179250 goto StartOver ;
180251 }
181252 } else if (addr + PGUP (size ) >= map_addr + PGUP (map_size )) {
182253 // shave off righthand side of mapping
183254 size_t left = addr - map_addr ;
184255 size_t right = map_addr + map_size - addr ;
185256 map -> size = left ;
257+ __maps .pages -= (right + pagesz - 1 ) / pagesz ;
258+ __maps_check ();
186259 if (!untrack_only ) {
260+ __maps_unlock ();
261+ ASSERT (PGUP (right ) <= PGUP (size ));
187262 if (__munmap_chunk (addr , right ) == -1 )
188263 rc = -1 ;
264+ __maps_lock ();
189265 goto StartOver ;
190266 }
191267 } else {
@@ -207,9 +283,14 @@ StartOver:;
207283 map -> off += left + middle ;
208284 dll_make_first (& __maps .used , & leftmap -> elem );
209285 * prev = leftmap ;
286+ __maps .pages -= (middle + pagesz - 1 ) / pagesz ;
287+ __maps .count += 1 ;
288+ __maps_check ();
210289 if (!untrack_only ) {
290+ __maps_unlock ();
211291 if (__munmap_chunk (addr , size ) == -1 )
212292 rc = -1 ;
293+ __maps_lock ();
213294 goto StartOver ;
214295 }
215296 } else {
@@ -241,7 +322,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
241322 sysflags |= MAP_FIXED_NOREPLACE_linux ;
242323 } else if (IsFreebsd () || IsNetbsd ()) {
243324 sysflags |= MAP_FIXED ;
244- if (__overlaps_existing_map (addr , size ))
325+ if (overlaps_existing_map (addr , size ))
245326 return (void * )eexist ();
246327 } else {
247328 noreplace = true;
@@ -296,7 +377,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
296377
297378 // untrack mapping we blew away
298379 if (should_untrack )
299- __munmap (addr , size , true);
380+ __munmap (res . addr , size , true);
300381
301382 // track Map object
302383 map -> addr = res .addr ;
@@ -346,7 +427,7 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
346427
347428 // so we create an separate map for each granule in the mapping
348429 if (!(flags & MAP_FIXED )) {
349- while (__overlaps_existing_map (addr , size )) {
430+ while (overlaps_existing_map (addr , size )) {
350431 if (flags & MAP_FIXED_NOREPLACE )
351432 return (void * )eexist ();
352433 addr += granularity ;
0 commit comments