Write more runtime tests and fix bugs

This change adds tests for the new memory manager code particularly with
its windows support. Function call tracing now works reliably on Silicon
since our function hooker was missing new Apple self-modifying code APIs

Many tests that were disabled a long time ago on aarch64 are reactivated
by this change, now that arm support is on equal terms with x86. There's
been a lot of places where ftrace could cause deadlocks, which have been
hunted down across all platforms thanks to new tests. A bug in Windows's
kill() function has been identified.

Author: jart

libc/calls/ntspawn.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/proc/ntspawn.h"
+#include "libc/calls/state.internal.h"
 #include "libc/calls/struct/sigset.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/intrin/strace.h"

libc/intrin/fds.c

@@ -190,7 +190,9 @@ textstartup void __init_fds(int argc, char **argv, char **envp) {
                 map->prot = PROT_READ | PROT_WRITE;
                 map->flags = MAP_SHARED | MAP_ANONYMOUS;
                 map->hand = shand;
+                __maps_lock();
                 __maps_insert(map);
+                __maps_unlock();
               }
             }
           }

libc/intrin/maps.c

@@ -112,6 +112,13 @@ void __maps_init(void) {
 }
 
 bool __maps_held(void) {
+  return !__tls_enabled || (__get_tls()->tib_flags & TIB_FLAG_VFORKED) ||
+         MUTEX_OWNER(
+             atomic_load_explicit(&__maps.lock.word, memory_order_relaxed)) ==
+             atomic_load_explicit(&__get_tls()->tib_ptid, memory_order_relaxed);
+}
+
+bool __maps_reentrant(void) {
   return __tls_enabled && !(__get_tls()->tib_flags & TIB_FLAG_VFORKED) &&
          MUTEX_OWNER(
              atomic_load_explicit(&__maps.lock.word, memory_order_relaxed)) ==

libc/intrin/maps.h

@@ -84,6 +84,7 @@ void __maps_init(void);
 void __maps_lock(void);
 void __maps_check(void);
 void __maps_unlock(void);
+bool __maps_reentrant(void);
 void *__maps_randaddr(void);
 void __maps_add(struct Map *);
 void __maps_free(struct Map *);
@@ -103,28 +104,28 @@ forceinline optimizespeed int __maps_search(const void *key,
   return (addr > map->addr) - (addr < map->addr);
 }
 
-dontinstrument static inline struct Map *__maps_next(struct Map *map) {
+static inline struct Map *__maps_next(struct Map *map) {
   struct Tree *node;
   if ((node = tree_next(&map->tree)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
-dontinstrument static inline struct Map *__maps_prev(struct Map *map) {
+static inline struct Map *__maps_prev(struct Map *map) {
   struct Tree *node;
   if ((node = tree_prev(&map->tree)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
-dontinstrument static inline struct Map *__maps_first(void) {
+static inline struct Map *__maps_first(void) {
   struct Tree *node;
   if ((node = tree_first(__maps.maps)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
-dontinstrument static inline struct Map *__maps_last(void) {
+static inline struct Map *__maps_last(void) {
   struct Tree *node;
   if ((node = tree_last(__maps.maps)))
     return MAP_TREE_CONTAINER(node);

libc/intrin/mmap.c

@@ -49,7 +49,7 @@
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 
-#define MMDEBUG  1
+#define MMDEBUG  0
 #define MAX_SIZE 0x0ff800000000ul
 
 #define MAP_FIXED_NOREPLACE_linux 0x100000
@@ -94,8 +94,11 @@ privileged optimizespeed struct Map *__maps_floor(const char *addr) {
 }
 
 static bool __maps_overlaps(const char *addr, size_t size) {
-  struct Map *map, *floor = __maps_floor(addr);
-  for (map = floor; map && map->addr <= addr + size; map = __maps_next(map))
+  struct Map *map;
+  ASSERT(__maps_held());
+  if (!(map = __maps_floor(addr)))
+    map = __maps_first();
+  for (; map && map->addr <= addr + size; map = __maps_next(map))
     if (MAX(addr, map->addr) <
         MIN(addr + PGUP(size), map->addr + PGUP(map->size)))
       return true;
@@ -105,30 +108,33 @@ static bool __maps_overlaps(const char *addr, size_t size) {
 // returns true if all fragments of all allocations which overlap
 // [addr,addr+size) are completely contained by [addr,addr+size).
 textwindows static bool __maps_envelops(const char *addr, size_t size) {
-  struct Map *map, *next;
+  struct Map *map;
   size = PGUP(size);
+  ASSERT(__maps_held());
   if (!(map = __maps_floor(addr)))
-    if (!(map = __maps_first()))
-      return true;
-  do {
-    if (MAX(addr, map->addr) >= MIN(addr + size, map->addr + map->size))
-      break;  // didn't overlap mapping
-    if (!__maps_isalloc(map))
-      return false;  // didn't include first fragment of alloc
-    if (addr > map->addr)
-      return false;  // excluded leading pages of first fragment
-    // set map to last fragment in allocation
-    for (; (next = __maps_next(map)) && !__maps_isalloc(next); map = next)
-      // fragments within an allocation must be perfectly contiguous
-      ASSERT(map->addr + map->size == next->addr);
-    if (addr + size < map->addr + PGUP(map->size))
-      return false;  // excluded trailing pages of allocation
-  } while ((map = next));
+    map = __maps_first();
+  while (map && map->addr <= addr + size) {
+    if (MAX(addr, map->addr) < MIN(addr + size, map->addr + map->size)) {
+      if (!__maps_isalloc(map))
+        return false;  // didn't include first fragment of alloc
+      if (addr > map->addr)
+        return false;    // excluded leading pages of first fragment
+      struct Map *next;  // set map to last fragment in allocation
+      for (; (next = __maps_next(map)) && !__maps_isalloc(next); map = next)
+        ASSERT(map->addr + map->size == next->addr);  // contiguous
+      if (addr + size < map->addr + PGUP(map->size))
+        return false;  // excluded trailing pages of allocation
+      map = next;
+    } else {
+      map = __maps_next(map);
+    }
+  }
   return true;
 }
 
 void __maps_check(void) {
 #if MMDEBUG
+  ASSERT(__maps_held());
   size_t maps = 0;
   size_t pages = 0;
   static unsigned mono;
@@ -152,20 +158,37 @@ void __maps_check(void) {
 #endif
 }
 
+#if MMDEBUG
+static void __maps_ok(void) {
+  ASSERT(!__maps_reentrant());
+  __maps_lock();
+  __maps_check();
+  __maps_unlock();
+}
+__attribute__((__constructor__)) static void __maps_ctor(void) {
+  atexit(__maps_ok);
+  __maps_ok();
+}
+__attribute__((__destructor__)) static void __maps_dtor(void) {
+  __maps_ok();
+}
+#endif
+
 static int __muntrack(char *addr, size_t size, struct Map **deleted,
                       struct Map **untracked, struct Map temp[2]) {
   int rc = 0;
   size_t ti = 0;
   struct Map *map;
   struct Map *next;
   size = PGUP(size);
+  ASSERT(__maps_held());
   if (!(map = __maps_floor(addr)))
     map = __maps_first();
   for (; map && map->addr <= addr + size; map = next) {
     next = __maps_next(map);
     char *map_addr = map->addr;
     size_t map_size = map->size;
-    if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))))
+    if (MAX(addr, map_addr) >= MIN(addr + size, map_addr + PGUP(map_size)))
       continue;
     if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
       if (map->hand == MAPS_RESERVATION)
@@ -350,6 +373,7 @@ static bool __maps_mergeable(const struct Map *x, const struct Map *y) {
 void __maps_insert(struct Map *map) {
   struct Map *left, *right;
   ASSERT(map->size);
+  ASSERT(__maps_held());
   ASSERT(!__maps_overlaps(map->addr, map->size));
   __maps.pages += (map->size + __pagesize - 1) / __pagesize;
 
@@ -460,7 +484,7 @@ static int __munmap(char *addr, size_t size) {
     return einval();
 
   // test for signal handler tragedy
-  if (__maps_held())
+  if (__maps_reentrant())
     return edeadlk();
 
   // lock the memory manager
@@ -469,8 +493,10 @@ static int __munmap(char *addr, size_t size) {
 
   // on windows we can only unmap whole allocations
   if (IsWindows())
-    if (!__maps_envelops(addr, size))
+    if (!__maps_envelops(addr, size)) {
+      __maps_unlock();
       return enotsup();
+    }
 
   // untrack mappings
   int rc;
@@ -500,6 +526,7 @@ void *__maps_randaddr(void) {
 }
 
 static void *__maps_pickaddr(size_t size) {
+  ASSERT(__maps_held());
   char *addr = 0;
   struct Map *map, *prev;
   size = GRUP(size);
@@ -569,11 +596,15 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
       noreplace = true;
       sysflags |= MAP_FIXED_NOREPLACE_linux;
     } else if (IsFreebsd() || IsNetbsd()) {
+      // todo: insert a reservation like windows
       sysflags |= MAP_FIXED;
+      __maps_lock();
       if (__maps_overlaps(addr, size)) {
+        __maps_unlock();
         __maps_free(map);
         return (void *)eexist();
       }
+      __maps_unlock();
     } else {
       noreplace = true;
     }
@@ -729,7 +760,7 @@ static void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
     return (void *)enomem();
 
   // test for signal handler reentry
-  if (__maps_held())
+  if (__maps_reentrant())
     return (void *)edeadlk();
 
   // create memory mappping
@@ -874,7 +905,7 @@ static void *__mremap(char *old_addr, size_t old_size, size_t new_size,
       return (void *)enomem();
 
   // test for signal handler reentry
-  if (__maps_held())
+  if (__maps_reentrant())
     return (void *)edeadlk();
 
   // lock the memory manager

libc/intrin/mprotect.c

@@ -67,16 +67,17 @@ int __mprotect(char *addr, size_t size, int prot) {
   size = (size + pagesz - 1) & -pagesz;
 
   // test for signal handler reentry
-  if (__maps_held())
+  if (__maps_reentrant())
     return edeadlk();
 
   // change mappings
   int rc = 0;
   bool found = false;
   __maps_lock();
-  struct Map *map, *floor;
-  floor = __maps_floor(addr);
-  for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) {
+  struct Map *map;
+  if (!(map = __maps_floor(addr)))
+    map = __maps_first();
+  for (; map && map->addr <= addr + size; map = __maps_next(map)) {
     char *map_addr = map->addr;
     size_t map_size = map->size;
     char *beg = MAX(addr, map_addr);
@@ -85,7 +86,7 @@ int __mprotect(char *addr, size_t size, int prot) {
       continue;
     found = true;
     if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
-      // change protection of entire mapping
+      // change protection status of pages
       if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
         map->prot = prot;
       } else {

libc/intrin/msync-nt.c

@@ -31,32 +31,29 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
 
   if ((uintptr_t)addr & (__pagesize - 1))
     return einval();
-  if (__maps_held())
+  if (__maps_reentrant())
     return edeadlk();
 
   int rc = 0;
   __maps_lock();
-  struct Map *map, *next;
+  struct Map *next, *map;
   if (!(map = __maps_floor(addr)))
-    if (!(map = __maps_first()))
-      return true;
-  for (; map; map = next) {
+    map = __maps_first();
+  for (; map && map->addr <= addr + size; map = next) {
     next = __maps_next(map);
     if (!__maps_isalloc(map))
       continue;
     if (map->flags & MAP_ANONYMOUS)
       continue;
     if (MAX(addr, map->addr) >= MIN(addr + size, map->addr + map->size))
-      break;  // didn't overlap mapping
+      continue;  // didn't overlap mapping
 
     // get true size of win32 allocation
     size_t allocsize = map->size;
-    for (struct Map *map2 = next; map2; map2 = __maps_next(map2)) {
-      if (!__maps_isalloc(map2) && map->addr + allocsize == map2->addr) {
-        allocsize += map2->size;
-      } else {
-        break;
-      }
+    while (next && !__maps_isalloc(next) &&
+           next->addr + allocsize == next->addr) {
+      allocsize += next->size;
+      next = __maps_next(next);
     }
 
     // perform the flush

libc/log/oncrash_arm64.c

@@ -47,6 +47,7 @@
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/stack.h"
 #include "libc/runtime/symbols.internal.h"
+#include "libc/runtime/syslib.internal.h"
 #include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/auxv.h"

libc/log/showcrashreports.c

@@ -82,11 +82,7 @@ void ShowCrashReports(void) {
   ss.ss_sp = crashstack;
   unassert(!sigaltstack(&ss, 0));
   InstallCrashHandler(SIGQUIT, 0);
-#ifdef __x86_64__
   InstallCrashHandler(SIGTRAP, 0);
-#else
-  InstallCrashHandler(SIGTRAP, 0);
-#endif
   InstallCrashHandler(SIGFPE, 0);
   InstallCrashHandler(SIGILL, 0);
   InstallCrashHandler(SIGBUS, 0);

libc/proc/fork-nt.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/internal.h"
 #include "libc/calls/sig.internal.h"
+#include "libc/calls/state.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/errno.h"
 #include "libc/intrin/directmap.h"