Make cli commands yoinkable into cosmo shell

This lets our system() and popen() commands function sort of like
BusyBox and ToyBox. By default the Cosmopolitan Shell is lightweight.
But if you use STATIC_YOINK then you can pull the individual commands
you want into the linkage, and they'll be included in a single binary.
For example the demo binary embeds `tr` and `sed` and ends up ~140kb.

Author: jart

Makefile

@@ -127,6 +127,7 @@ include ape/ape.mk				# │
 include libc/fmt/fmt.mk				# │
 include libc/vga/vga.mk				#─┘
 include libc/calls/calls.mk			#─┐
+include third_party/getopt/getopt.mk		# │
 include libc/runtime/runtime.mk			# ├──SYSTEMS RUNTIME
 include libc/crt/crt.mk				# │  You can issue system calls
 include third_party/nsync/nsync.mk		# │
@@ -150,7 +151,6 @@ include dsp/mpeg/mpeg.mk			# │
 include dsp/dsp.mk				# │
 include third_party/zlib/gz/gz.mk		# │
 include third_party/musl/musl.mk		# │
-include third_party/getopt/getopt.mk		# │
 include libc/libc.mk				#─┘
 include libc/sock/sock.mk			#─┐
 include dsp/tty/tty.mk				# ├──ONLINE RUNTIME
@@ -171,6 +171,7 @@ include third_party/maxmind/maxmind.mk
 include net/finger/finger.mk
 include third_party/double-conversion/test/test.mk
 include third_party/lua/lua.mk
+include third_party/tr/tr.mk
 include third_party/sed/sed.mk
 include third_party/awk/awk.mk
 include third_party/make/make.mk

examples/examples.mk

@@ -49,12 +49,12 @@ EXAMPLES_DIRECTDEPS =								\
 	LIBC_LOG								\
 	LIBC_MEM								\
 	LIBC_NEXGEN32E								\
+	LIBC_NT_ADVAPI32							\
 	LIBC_NT_IPHLPAPI							\
 	LIBC_NT_KERNEL32							\
 	LIBC_NT_NTDLL								\
 	LIBC_NT_USER32								\
 	LIBC_NT_WS2_32								\
-	LIBC_NT_ADVAPI32							\
 	LIBC_RUNTIME								\
 	LIBC_SOCK								\
 	LIBC_STDIO								\
@@ -71,6 +71,7 @@ EXAMPLES_DIRECTDEPS =								\
 	LIBC_ZIPOS								\
 	NET_HTTP								\
 	NET_HTTPS								\
+	THIRD_PARTY_AWK								\
 	THIRD_PARTY_COMPILER_RT							\
 	THIRD_PARTY_DLMALLOC							\
 	THIRD_PARTY_DOUBLECONVERSION						\
@@ -84,7 +85,9 @@ EXAMPLES_DIRECTDEPS =								\
 	THIRD_PARTY_NSYNC							\
 	THIRD_PARTY_NSYNC_MEM							\
 	THIRD_PARTY_QUICKJS							\
+	THIRD_PARTY_SED								\
 	THIRD_PARTY_STB								\
+	THIRD_PARTY_TR								\
 	THIRD_PARTY_XED								\
 	THIRD_PARTY_ZLIB							\
 	TOOL_BUILD_LIB								\

examples/system.c

@@ -7,13 +7,20 @@
 │   • http://creativecommons.org/publicdomain/zero/1.0/            │
 ╚─────────────────────────────────────────────────────────────────*/
 #endif
-#include "libc/dce.h"
+#include "libc/runtime/runtime.h"
 #include "libc/stdio/stdio.h"
 
+/**
+ * @fileoverview Cosmopolitan Command Interpreter Demo
+ * Yes this works on Windows.
+ */
+
+STATIC_YOINK("_tr");
+STATIC_YOINK("_sed");
+
 int main(int argc, char *argv[]) {
-  if (IsWindows()) {
-    system("notepad");
-  } else {
-    system("ls | tr a-z A-Z");
-  }
+  system("x=world\n"
+         "echo hello $x |\n"
+         "  tr a-z A-Z |\n"
+         "  sed 's/\\(.\\)/\\1 /g'");
 }

libc/log/err.c

@@ -94,12 +94,12 @@ verrc(int eval, int code, const char *fmt, va_list ap)
 {
 	if (err_file == NULL)
 		err_set_file(NULL);
-	fprintf(err_file, "%s: ", program_invocation_name);
+	(fprintf)(err_file, "%s: ", program_invocation_name);
 	if (fmt != NULL) {
-		vfprintf(err_file, fmt, ap);
-		fprintf(err_file, ": ");
+		(vfprintf)(err_file, fmt, ap);
+		(fprintf)(err_file, ": ");
 	}
-	fprintf(err_file, "%s\n", _strerdoc(code));
+	(fprintf)(err_file, "%s\n", _strerdoc(code));
 	if (err_exit)
 		err_exit(eval);
 	exit(eval);
@@ -119,10 +119,10 @@ verrx(int eval, const char *fmt, va_list ap)
 {
 	if (err_file == NULL)
 		err_set_file(NULL);
-	fprintf(err_file, "%s: ", program_invocation_name);
+	(fprintf)(err_file, "%s: ", program_invocation_name);
 	if (fmt != NULL)
-		vfprintf(err_file, fmt, ap);
-	fprintf(err_file, "\n");
+		(vfprintf)(err_file, fmt, ap);
+	(fprintf)(err_file, "\n");
 	if (err_exit)
 		err_exit(eval);
 	exit(eval);
@@ -160,12 +160,12 @@ vwarnc(int code, const char *fmt, va_list ap)
 	saved_errno = errno;
 	if (err_file == NULL)
 		err_set_file(NULL);
-	fprintf(err_file, "%s: ", program_invocation_name);
+	(fprintf)(err_file, "%s: ", program_invocation_name);
 	if (fmt != NULL) {
-		vfprintf(err_file, fmt, ap);
-		fprintf(err_file, ": ");
+		(vfprintf)(err_file, fmt, ap);
+		(fprintf)(err_file, ": ");
 	}
-	fprintf(err_file, "%s\n", strerror(code));
+	(fprintf)(err_file, "%s\n", strerror(code));
 	errno = saved_errno;
 }
 
@@ -186,9 +186,9 @@ vwarnx(const char *fmt, va_list ap)
 	saved_errno = errno;
 	if (err_file == NULL)
 		err_set_file(NULL);
-	fprintf(err_file, "%s: ", program_invocation_name);
+	(fprintf)(err_file, "%s: ", program_invocation_name);
 	if (fmt != NULL)
-		vfprintf(err_file, fmt, ap);
-	fprintf(err_file, "\n");
+		(vfprintf)(err_file, fmt, ap);
+	(fprintf)(err_file, "\n");
 	errno = saved_errno;
 }

libc/mem/mem.h

@@ -52,7 +52,6 @@ struct mallinfo {
 struct mallinfo mallinfo(void);
 
 void malloc_stats(void);
-bool32 mallopt(int, int);
 size_t malloc_footprint(void);
 size_t malloc_max_footprint(void);
 size_t malloc_footprint_limit(void);

libc/runtime/cocmd.c

@@ -26,13 +26,17 @@
 #include "libc/fmt/itoa.h"
 #include "libc/fmt/magnumstrs.internal.h"
 #include "libc/intrin/bits.h"
+#include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/sysv/consts/s.h"
 #include "libc/sysv/consts/sig.h"
 #include "libc/sysv/consts/timer.h"
+#include "third_party/awk/cmd.h"
+#include "third_party/sed/cmd.h"
+#include "third_party/tr/cmd.h"
 
 /**
  * @fileoverview Cosmopolitan Command Interpreter
@@ -48,6 +52,7 @@
 #define STATE_QUOTED_VAR 4
 #define STATE_WHITESPACE 5
 
+#define TOMBSTONE ((char *)-1)
 #define READ24(s) READ32LE(s "\0")
 
 struct Env {
@@ -62,6 +67,7 @@ static int envi;
 static int vari;
 static size_t n;
 static char *cmd;
+static char *assign;
 static char var[32];
 static int lastchild;
 static int exitstatus;
@@ -175,7 +181,14 @@ static void Append(int c) {
 static char *Finish(void) {
   char *s = r;
   Append(0);
-  return r = q, s;
+  r = q;
+  if (!assign) {
+    return s;
+  } else {
+    PutEnv(envs, s);
+    assign = 0;
+    return TOMBSTONE;
+  }
 }
 
 static int True(void) {
@@ -347,6 +360,19 @@ static int Test(void) {
   return 1;
 }
 
+static int Fake(int main(int, char **)) {
+  int exitstatus, ws, pid;
+  if ((pid = fork()) == -1) SysExit(21, "vfork", prog);
+  if (!pid) {
+    // TODO(jart): Maybe nuke stdio state somehow?
+    environ = envs;
+    exit(main(n, args));
+  }
+  if (waitpid(pid, &ws, 0) == -1) SysExit(22, "waitpid", prog);
+  exitstatus = WIFEXITED(ws) ? WEXITSTATUS(ws) : 128 + WTERMSIG(ws);
+  return n = 0, exitstatus;
+}
+
 static int TryBuiltin(void) {
   if (!n) return 0;
   if (!strcmp(args[0], "exit")) Exit();
@@ -362,6 +388,9 @@ static int TryBuiltin(void) {
   if (!strcmp(args[0], "false")) return False();
   if (!strcmp(args[0], "usleep")) return Usleep();
   if (!strcmp(args[0], "toupper")) return Toupper();
+  if (_weaken(_tr) && !strcmp(args[0], "tr")) return Fake(_weaken(_tr));
+  if (_weaken(_sed) && !strcmp(args[0], "sed")) return Fake(_weaken(_sed));
+  if (_weaken(_awk) && !strcmp(args[0], "awk")) return Fake(_weaken(_awk));
   return -1;
 }
 
@@ -477,6 +506,9 @@ static char *Tokenize(void) {
         } else if (*p == '\\') {
           if (!p[1]) UnsupportedSyntax(*p);
           Append(*++p);
+        } else if (*p == '=') {
+          if (!n && q > r) assign = r;
+          Append(*p);
         } else if (*p == '|') {
           if (q > r) {
             return Finish();
@@ -492,7 +524,7 @@ static char *Tokenize(void) {
             Pipe();
             t = STATE_WHITESPACE;
           }
-        } else if (*p == ';') {
+        } else if (*p == ';' || *p == '\n') {
           if (q > r) {
             return Finish();
           } else {
@@ -595,6 +627,7 @@ int _cocmd(int argc, char **argv, char **envp) {
     unsupported[i] = true;
   }
   unsupported['\t'] = false;
+  unsupported['\n'] = false;
   unsupported[0177] = true;
   unsupported['~'] = true;
   unsupported['`'] = true;
@@ -633,6 +666,7 @@ int _cocmd(int argc, char **argv, char **envp) {
   n = 0;
   r = q = argbuf;
   while ((arg = Tokenize())) {
+    if (arg == TOMBSTONE) continue;
     if (n + 1 < ARRAYLEN(args)) {
       if (isdigit(arg[0]) && arg[1] == '>' && arg[2] == '&' &&
           isdigit(arg[3])) {

test/libc/mem/malloc_test.c

@@ -27,6 +27,7 @@
 #include "libc/mem/gc.h"
 #include "libc/mem/gc.internal.h"
 #include "libc/mem/mem.h"
+#include "libc/runtime/internal.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/sysconf.h"
@@ -143,10 +144,12 @@ void MallocFree(void) {
 }
 
 BENCH(bulk_free, bench) {
-  EZBENCH2("free(malloc(16))", donothing, MallocFree());
   EZBENCH2("free() bulk", BulkFreeBenchSetup(), FreeBulk());
   EZBENCH2("bulk_free()", BulkFreeBenchSetup(),
            bulk_free(bulk, ARRAYLEN(bulk)));
+  EZBENCH2("free(malloc(16)) ST", donothing, MallocFree());
+  __enable_threads();
+  EZBENCH2("free(malloc(16)) MT", donothing, MallocFree());
 }
 
 #define ITERATIONS 10000

third_party/awk/awk.mk

@@ -3,20 +3,14 @@
 
 PKGS += THIRD_PARTY_AWK
 
-THIRD_PARTY_AWK_SRCS = $(THIRD_PARTY_AWK_A_SRCS)
-THIRD_PARTY_AWK_HDRS = $(THIRD_PARTY_AWK_A_HDRS)
-THIRD_PARTY_AWK_INCS = $(THIRD_PARTY_AWK_A_INCS)
-THIRD_PARTY_AWK_BINS = $(THIRD_PARTY_AWK_COMS) $(THIRD_PARTY_AWK_COMS:%=%.dbg)
-THIRD_PARTY_AWK_COMS = o/$(MODE)/third_party/awk/awk.com
-
 THIRD_PARTY_AWK_ARTIFACTS += THIRD_PARTY_AWK_A
 THIRD_PARTY_AWK = $(THIRD_PARTY_AWK_A_DEPS) $(THIRD_PARTY_AWK_A)
 THIRD_PARTY_AWK_A = o/$(MODE)/third_party/awk/awk.a
-THIRD_PARTY_AWK_A_FILES := $(wildcard third_party/awk/*)
-THIRD_PARTY_AWK_A_HDRS = $(filter %.h,$(THIRD_PARTY_AWK_A_FILES))
-THIRD_PARTY_AWK_A_INCS = $(filter %.inc,$(THIRD_PARTY_AWK_A_FILES))
-THIRD_PARTY_AWK_A_SRCS = $(filter %.c,$(THIRD_PARTY_AWK_A_FILES))
-THIRD_PARTY_AWK_A_OBJS = $(THIRD_PARTY_AWK_A_SRCS:%.c=o/$(MODE)/%.o)
+THIRD_PARTY_AWK_FILES := $(wildcard third_party/awk/*)
+THIRD_PARTY_AWK_HDRS = $(filter %.h,$(THIRD_PARTY_AWK_FILES))
+THIRD_PARTY_AWK_INCS = $(filter %.inc,$(THIRD_PARTY_AWK_FILES))
+THIRD_PARTY_AWK_SRCS = $(filter %.c,$(THIRD_PARTY_AWK_FILES))
+THIRD_PARTY_AWK_OBJS = $(THIRD_PARTY_AWK_SRCS:%.c=o/$(MODE)/%.o)
 
 THIRD_PARTY_AWK_A_DIRECTDEPS =				\
 	LIBC_FMT					\
@@ -36,22 +30,22 @@ THIRD_PARTY_AWK_A_DIRECTDEPS =				\
 THIRD_PARTY_AWK_A_DEPS :=				\
 	$(call uniq,$(foreach x,$(THIRD_PARTY_AWK_A_DIRECTDEPS),$($(x))))
 
-THIRD_PARTY_AWK_A_CHECKS =				\
+THIRD_PARTY_AWK_CHECKS =				\
 	$(THIRD_PARTY_AWK_A).pkg			\
-	$(THIRD_PARTY_AWK_A_HDRS:%=o/$(MODE)/%.ok)
+	$(THIRD_PARTY_AWK_HDRS:%=o/$(MODE)/%.ok)
 
 $(THIRD_PARTY_AWK_A):					\
 		third_party/awk/			\
 		$(THIRD_PARTY_AWK_A).pkg		\
-		$(THIRD_PARTY_AWK_A_OBJS)
+		$(THIRD_PARTY_AWK_OBJS)
 
 $(THIRD_PARTY_AWK_A).pkg:				\
-		$(THIRD_PARTY_AWK_A_OBJS)		\
+		$(THIRD_PARTY_AWK_OBJS)		\
 		$(foreach x,$(THIRD_PARTY_AWK_A_DIRECTDEPS),$($(x)_A).pkg)
 
 o/$(MODE)/third_party/awk/awk.com.dbg:			\
 		$(THIRD_PARTY_AWK)			\
-		o/$(MODE)/third_party/awk/main.o	\
+		o/$(MODE)/third_party/awk/cmd.o		\
 		o/$(MODE)/third_party/awk/README.zip.o	\
 		$(CRT)					\
 		$(APE_NO_MODIFY_SELF)
@@ -61,10 +55,9 @@ o/$(MODE)/third_party/awk/README.zip.o:			\
 		ZIPOBJ_FLAGS =				\
 			-B
 
-THIRD_PARTY_AWK_LIBS = $(foreach x,$(THIRD_PARTY_AWK_ARTIFACTS),$($(x)))
-THIRD_PARTY_AWK_SRCS = $(foreach x,$(THIRD_PARTY_AWK_ARTIFACTS),$($(x)_SRCS))
-THIRD_PARTY_AWK_CHECKS = $(foreach x,$(THIRD_PARTY_AWK_ARTIFACTS),$($(x)_CHECKS))
-THIRD_PARTY_AWK_OBJS = $(foreach x,$(THIRD_PARTY_AWK_ARTIFACTS),$($(x)_OBJS))
+THIRD_PARTY_AWK_BINS = $(THIRD_PARTY_AWK_COMS) $(THIRD_PARTY_AWK_COMS:%=%.dbg)
+THIRD_PARTY_AWK_COMS = o/$(MODE)/third_party/awk/awk.com
+THIRD_PARTY_AWK_LIBS = $(THIRD_PARTY_AWK_A)
 $(THIRD_PARTY_AWK_OBJS): $(BUILD_FILES) third_party/awk/awk.mk
 
 .PHONY: o/$(MODE)/third_party/awk

third_party/awk/awkgram.tab.c

@@ -1,13 +1,5 @@
 // clang-format off
-#include "libc/mem/alg.h"
-#include "libc/fmt/conv.h"
 #include "libc/mem/mem.h"
-#include "libc/stdio/rand.h"
-#include "libc/runtime/runtime.h"
-#include "libc/stdio/temp.h"
-#include "libc/sysv/consts/exit.h"
-#include "third_party/gdtoa/gdtoa.h"
-#include "libc/mem/alg.h"
 #include "libc/str/str.h"
 #define YYBYACC 1
 #define YYMAJOR 1
@@ -19,13 +11,6 @@
 #define YYRECOVERING() (yyerrflag!=0)
 #define YYPREFIX "yy"
 #line 26 "awkgram.y"
-#include "libc/calls/calls.h"
-#include "libc/fmt/fmt.h"
-#include "libc/stdio/lock.internal.h"
-#include "libc/stdio/stdio.h"
-#include "libc/stdio/temp.h"
-#include "libc/mem/alg.h"
-#include "libc/str/str.h"
 #include "third_party/awk/awk.h"
 
 void checkdup(Node *list, Cell *item);