Make fork() go 30% faster

This change makes fork() go nearly as fast as sys_fork() on UNIX. As for
Windows this change shaves about 4-5ms off fork() + wait() latency. This
is accomplished by using WriteProcessMemory() from the parent process to
setup the address space of a suspended process; it is better than a pipe

Author: jart

Makefile

@@ -135,7 +135,7 @@ ARCH = aarch64
 HOSTS ?= pi pi5 studio freebsdarm
 else
 ARCH = x86_64
-HOSTS ?= freebsd rhel7 xnu openbsd netbsd win10
+HOSTS ?= freebsd rhel7 xnu openbsd netbsd win10 luna
 endif
 
 ZIPOBJ_FLAGS += -a$(ARCH)

libc/intrin/describemapping.c

@@ -17,19 +17,21 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/describeflags.h"
+#include "libc/intrin/maps.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
 
 static char DescribeMapType(int flags) {
   switch (flags & MAP_TYPE) {
     case MAP_FILE:
+      if (flags & MAP_NOFORK)
+        return 'i';  // executable image
       return '-';
     case MAP_PRIVATE:
       if (flags & MAP_NOFORK)
-        return 'P';
-      else
-        return 'p';
+        return 'w';  // windows memory
+      return 'p';
     case MAP_SHARED:
       return 's';
     default:

libc/intrin/dlopen.c

@@ -19,7 +19,7 @@
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 
-pthread_mutex_t __dlopen_lock_obj = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t __dlopen_lock_obj = PTHREAD_MUTEX_INITIALIZER;
 
 void __dlopen_lock(void) {
   _pthread_mutex_lock(&__dlopen_lock_obj);
@@ -28,3 +28,7 @@ void __dlopen_lock(void) {
 void __dlopen_unlock(void) {
   _pthread_mutex_unlock(&__dlopen_lock_obj);
 }
+
+void __dlopen_wipe(void) {
+  _pthread_mutex_wipe_np(&__dlopen_lock_obj);
+}

libc/intrin/localtime_lock.c

@@ -19,7 +19,7 @@
 #include "libc/thread/posixthread.internal.h"
 #include "third_party/tz/lock.h"
 
-pthread_mutex_t __localtime_lock_obj = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t __localtime_lock_obj = PTHREAD_MUTEX_INITIALIZER;
 
 void __localtime_lock(void) {
   _pthread_mutex_lock(&__localtime_lock_obj);
@@ -28,3 +28,7 @@ void __localtime_lock(void) {
 void __localtime_unlock(void) {
   _pthread_mutex_unlock(&__localtime_lock_obj);
 }
+
+void __localtime_wipe(void) {
+  _pthread_mutex_wipe_np(&__localtime_lock_obj);
+}

libc/intrin/maps.c

@@ -30,6 +30,7 @@
 #include "libc/nexgen32e/rdtsc.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/stack.h"
+#include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
 #include "libc/thread/lock.h"
 #include "libc/thread/tls.h"
@@ -40,10 +41,6 @@ __static_yoink("_init_maps");
 
 #define ABI privileged optimizespeed
 
-// take great care if you enable this
-// especially if you're using --ftrace too
-#define DEBUG_MAPS_LOCK 0
-
 struct Maps __maps;
 
 void __maps_add(struct Map *map) {
@@ -61,14 +58,18 @@ void __maps_stack(char *stackaddr, int pagesz, int guardsize, size_t stacksize,
   __maps.stack.addr = stackaddr + guardsize;
   __maps.stack.size = stacksize - guardsize;
   __maps.stack.prot = stackprot;
-  __maps.stack.hand = -1;
+  __maps.stack.hand = MAPS_SUBREGION;
+  __maps.stack.flags = MAP_PRIVATE | MAP_ANONYMOUS;
   __maps_adder(&__maps.stack, pagesz);
   if (guardsize) {
     __maps.guard.addr = stackaddr;
     __maps.guard.size = guardsize;
-    __maps.guard.prot = PROT_NONE;
+    __maps.guard.prot = PROT_NONE | PROT_GUARD;
     __maps.guard.hand = stackhand;
+    __maps.guard.flags = MAP_PRIVATE | MAP_ANONYMOUS;
     __maps_adder(&__maps.guard, pagesz);
+  } else {
+    __maps.stack.hand = stackhand;
   }
 }
 
@@ -102,28 +103,13 @@ void __maps_init(void) {
   }
 
   // record .text and .data mappings
-  static struct Map text, data;
-  text.addr = (char *)__executable_start;
-  text.size = _etext - __executable_start;
-  text.prot = PROT_READ | PROT_EXEC;
+  __maps_track((char *)__executable_start, _etext - __executable_start,
+               PROT_READ | PROT_EXEC, MAP_NOFORK);
   uintptr_t ds = ((uintptr_t)_etext + pagesz - 1) & -pagesz;
-  if (ds < (uintptr_t)_end) {
-    data.addr = (char *)ds;
-    data.size = (uintptr_t)_end - ds;
-    data.prot = PROT_READ | PROT_WRITE;
-    __maps_adder(&data, pagesz);
-  }
-  __maps_adder(&text, pagesz);
-}
-
-#if DEBUG_MAPS_LOCK
-privileged static void __maps_panic(const char *msg) {
-  // it's only safe to pass a format string. if we use directives such
-  // as %s, %t etc. then kprintf() will recursively call __maps_lock()
-  kprintf(msg);
-  DebugBreak();
+  if (ds < (uintptr_t)_end)
+    __maps_track((char *)ds, (uintptr_t)_end - ds, PROT_READ | PROT_WRITE,
+                 MAP_NOFORK);
 }
-#endif
 
 bool __maps_held(void) {
   return __tls_enabled && !(__get_tls()->tib_flags & TIB_FLAG_VFORKED) &&
@@ -143,7 +129,12 @@ ABI void __maps_lock(void) {
   if (tib->tib_flags & TIB_FLAG_VFORKED)
     return;
   me = atomic_load_explicit(&tib->tib_ptid, memory_order_relaxed);
-  if (me <= 0)
+  word = 0;
+  lock = MUTEX_LOCK(word);
+  lock = MUTEX_SET_OWNER(lock, me);
+  if (atomic_compare_exchange_strong_explicit(&__maps.lock.word, &word, lock,
+                                              memory_order_acquire,
+                                              memory_order_relaxed))
     return;
   word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
   for (;;) {
@@ -154,24 +145,13 @@ ABI void __maps_lock(void) {
         return;
       continue;
     }
-#if DEBUG_MAPS_LOCK
-    if (__deadlock_tracked(&__maps.lock) == 1)
-      __maps_panic("error: maps lock already held\n");
-    if (__deadlock_check(&__maps.lock, 1))
-      __maps_panic("error: maps lock is cyclic\n");
-#endif
     word = 0;
     lock = MUTEX_LOCK(word);
     lock = MUTEX_SET_OWNER(lock, me);
     if (atomic_compare_exchange_weak_explicit(&__maps.lock.word, &word, lock,
                                               memory_order_acquire,
-                                              memory_order_relaxed)) {
-#if DEBUG_MAPS_LOCK
-      __deadlock_track(&__maps.lock, 0);
-      __deadlock_record(&__maps.lock, 0);
-#endif
+                                              memory_order_relaxed))
       return;
-    }
     for (;;) {
       word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
       if (MUTEX_OWNER(word) == me)
@@ -183,7 +163,6 @@ ABI void __maps_lock(void) {
 }
 
 ABI void __maps_unlock(void) {
-  int me;
   uint64_t word;
   struct CosmoTib *tib;
   if (!__tls_enabled)
@@ -192,28 +171,16 @@ ABI void __maps_unlock(void) {
     return;
   if (tib->tib_flags & TIB_FLAG_VFORKED)
     return;
-  me = atomic_load_explicit(&tib->tib_ptid, memory_order_relaxed);
-  if (me <= 0)
-    return;
   word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
-#if DEBUG_MAPS_LOCK
-  if (__deadlock_tracked(&__maps.lock) == 0)
-    __maps_panic("error: maps lock not owned by caller\n");
-#endif
   for (;;) {
-    if (MUTEX_DEPTH(word)) {
+    if (MUTEX_DEPTH(word))
       if (atomic_compare_exchange_weak_explicit(
               &__maps.lock.word, &word, MUTEX_DEC_DEPTH(word),
               memory_order_relaxed, memory_order_relaxed))
         break;
-    }
     if (atomic_compare_exchange_weak_explicit(&__maps.lock.word, &word, 0,
                                               memory_order_release,
-                                              memory_order_relaxed)) {
-#if DEBUG_MAPS_LOCK
-      __deadlock_untrack(&__maps.lock);
-#endif
+                                              memory_order_relaxed))
       break;
-    }
   }
 }

libc/intrin/maps.h

@@ -5,16 +5,37 @@
 #include "libc/runtime/runtime.h"
 COSMOPOLITAN_C_START_
 
+/* size of dynamic memory that is used internally by your memory manager */
+#define MAPS_SIZE 65536
+
+/* when map->hand is MAPS_RESERVATION it means mmap() is transactionally
+   reserving address space it is in the process of requesting from win32 */
+#define MAPS_RESERVATION -2
+
+/* when map->hand is MAPS_SUBREGION it means that an allocation has been
+   broken into multiple fragments by mprotect(). the first fragment must
+   be set to MAPS_VIRTUAL or your CreateFileMapping() handle. your frags
+   must be perfectly contiguous in memory and should have the same flags */
+#define MAPS_SUBREGION -3
+
+/* indicates an allocation was created by VirtualAlloc() and so munmap()
+   must call VirtualFree() when destroying it. use it on the hand field. */
+#define MAPS_VIRTUAL -4
+
+/* if this is used on MAP_PRIVATE memory, then it's assumed to be memory
+   that win32 allocated, e.g. a CreateThread() stack. if this is used on
+   MAP_FILE memory, then it's assumed to be part of the executable image */
+#define MAP_NOFORK 0x10000000
+
 #define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e)
 
 struct Map {
   char *addr;        /* granule aligned */
   size_t size;       /* must be nonzero */
   int64_t off;       /* ignore for anon */
   int flags;         /* memory map flag */
-  char prot;         /* memory protects */
+  short prot;        /* memory protects */
   bool iscow;        /* windows nt only */
-  bool precious;     /* windows nt only */
   bool readonlyfile; /* windows nt only */
   unsigned visited;  /* checks and fork */
   intptr_t hand;     /* windows nt only */
@@ -29,11 +50,17 @@ struct MapLock {
   _Atomic(uint64_t) word;
 };
 
+struct MapSlab {
+  struct MapSlab *next;
+  struct Map maps[(MAPS_SIZE - sizeof(struct MapSlab *)) / sizeof(struct Map)];
+};
+
 struct Maps {
   uint128_t rand;
   struct Tree *maps;
   struct MapLock lock;
   _Atomic(uintptr_t) freed;
+  _Atomic(struct MapSlab *) slabs;
   size_t count;
   size_t pages;
   struct Map stack;
@@ -76,33 +103,37 @@ forceinline optimizespeed int __maps_search(const void *key,
   return (addr > map->addr) - (addr < map->addr);
 }
 
-static inline struct Map *__maps_next(struct Map *map) {
+dontinstrument static inline struct Map *__maps_next(struct Map *map) {
   struct Tree *node;
   if ((node = tree_next(&map->tree)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
-static inline struct Map *__maps_prev(struct Map *map) {
+dontinstrument static inline struct Map *__maps_prev(struct Map *map) {
   struct Tree *node;
   if ((node = tree_prev(&map->tree)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
-static inline struct Map *__maps_first(void) {
+dontinstrument static inline struct Map *__maps_first(void) {
   struct Tree *node;
   if ((node = tree_first(__maps.maps)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
-static inline struct Map *__maps_last(void) {
+dontinstrument static inline struct Map *__maps_last(void) {
   struct Tree *node;
   if ((node = tree_last(__maps.maps)))
     return MAP_TREE_CONTAINER(node);
   return 0;
 }
 
+static inline bool __maps_isalloc(struct Map *map) {
+  return map->hand != MAPS_SUBREGION;
+}
+
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_MAPS_H_ */