Chore: reorganize libutf and libfmt into base

I found the split to be arbitrary. Better to include the functionality
in the standard library. I also split the headers to allow for more
granular inclusion (but the library is still monolithic). The only
ugliness is the circular dependency introduced with libutf's generated
functions. We put explicit prereqs with the necessary object files
instead.

14 files changed, 378 insertions, 381 deletions

diff --git a/include/base.h b/include/base.h
index df580c3..8152cb7 100644
--- a/include/base.h
+++ b/include/base.h
@@ -23,385 +23,21 @@
 
 typedef wchar_t wchar;
 
-// ----------------------------------------------------------------------------
-// dynamic array
-
-typedef struct BufHdr
-{
-    vlong len;
-    vlong cap;
-    byte  buf[];
-} BufHdr;
-
-#define bufhdr(b)        ((BufHdr*)((uint8*)(b)-offsetof(BufHdr, buf)))
-#define buflen(b)        ((b) ? (bufhdr(b)->len) : 0)
-#define bufcap(b)        ((b) ? (bufhdr(b)->cap) : 0)
-#define bufend(b)        ((b) + buflen(b))
-#define bufsize(b)       ((b) ? (buflen(b) * sizeof((b)[0])) : 0)
-
-#define buffree(b)       ((b) ? (free(bufhdr(b)), (b) = nil) : 0)
-#define buffit(b, n)     ((n) <= bufcap(b) ? 0 : ((b) = ·bufgrow((b), (n), sizeof(*(b)))))
-
-#define bufpush(b, ...)  (buffit((b), 1 + buflen(b)), (b)[bufhdr(b)->len++] = (__VA_ARGS__))
-#define bufaddn(b, n)    (buffit(b, buflen(b)+n), bufhdr(b)->len += n, b+bufhdr(b)->len-n)
-
-#define bufpop(b)         ((b)[--bufhdr(b)->len])
-#define bufdel(b, i)      bufdeln((b), (i), 1)
-#define bufdeln(b, i, n)  (memmove((b)+(i), (b)+(i)+(n), sizeof(*(b))*(bufhdr(b)->len-(n)-(i)), bufhdr(b)->len -= (n))
-#define bufdelswap(b, i)  ((b)[i] = bufend(b)[-1], bufhdr(b)->len-=1)
-
-void* ·bufgrow(void*, vlong, vlong);
-
-// -----------------------------------------------------------------------------
-// memory allocation
-
-/* allocator interface */
-typedef struct mem·Allocator {
-    void *(*alloc)(void *heap, uint n, ulong size);
-    void  (*free)(void *heap, void *ptr);
-} mem·Allocator;
-
-extern mem·Allocator sys·Memory;
-
-typedef struct mem·Reallocator {
-    void *(*alloc)(void *iface, uint n, ulong size);
-    void *(*realloc)(void *iface, void *ptr, uint n, ulong size);
-    void  (*free)(void *iface, void *ptr);
-} mem·Reallocator;
-
-extern mem·Reallocator sys·FullMemory;
-
-/* simple memory arena */
-typedef struct mem·Arena mem·Arena;
-
-mem·Arena *mem·makearena(mem·Allocator from, void*);
-void      *mem·arenaalloc(mem·Arena *A, uint n, ulong size); 
-void       mem·freearena(mem·Arena *A);
-
-extern mem·Allocator mem·ArenaAllocator;
-
-/* generalized memxxx functions */
-void       memset64(void *dst, uint64 val, uintptr size);
-
-// -----------------------------------------------------------------------------
-// coroutines
-
-typedef struct Coro Coro;
-
-Coro*   coro·make(uintptr stk, uintptr (*func)(Coro*, uintptr));
-uintptr coro·yield(Coro *c, uintptr arg);
-int     coro·free(Coro *c);
-
-// -----------------------------------------------------------------------------
-// strings
-
-typedef byte* string;
-
-/* augmented string functions */
-string  str·makecap(const char *s, vlong len, vlong cap);
-string  str·makelen(const char *s, vlong len);
-string  str·make(const char *s);
-string  str·makef(const char *fmt, ...);
-void    str·free(string s);
-int     str·len(const string s);
-int     str·cap(const string s);
-void    str·clear(string *s);
-void    str·grow(string *s, vlong delta);
-void    str·fit(string *s);
-int     str·appendlen(string *s, vlong len, const char *b);
-int     str·append(string *s, const char *b);
-int     str·appendf(string *s, const char *fmt, ...);
-int     str·appendbyte(string *s, const char b);
-bool    str·equals(const string s, const string t);
-int     str·find(string s, const char* substr);
-void    str·lower(string s);
-void    str·upper(string s);
-int     str·read(string s, int size, int n, void *buf);
-void    str·replace(string s, const byte* from, const byte* to);
-string* str·split(string s, const byte* tok);
-string  str·join(vlong len, byte** fields, const byte* sep);
-
-/* raw C string functions */
-char    *str·copyn(char *dst, char *src, int n);
-
-// -----------------------------------------------------------------------------
-// i/o
-
-enum SeekPos
-{
-    seek·cur = SEEK_CUR,
-    seek·set = SEEK_SET,
-    seek·end = SEEK_END
-};
-
-typedef struct io·Reader
-{
-    int (*read)(void*, int sz, int n, void *buf);
-} io·Reader;
-extern io·Reader sys·Reader;
-
-typedef struct io·Peeker
-{
-    byte  (*get)(void*);
-    int   (*unget)(void*, byte);
-} io·Peeker;
-extern io·Peeker sys·Peeker;
-
-typedef struct io·Seeker
-{
-    int        (*seek)(void *skr, long off, enum SeekPos whence);
-    long       (*tell)(void *skr);
-} io·Seeker;
-extern io·Seeker sys·Seeker;
-
-typedef struct io·SeekReader
-{
-    io·Seeker;
-    io·Reader;
-} io·SeekReader;
-extern io·SeekReader sys·SeekReader;
-
-typedef struct io·PeekReader
-{
-    io·Reader;
-    io·Peeker;
-} io·PeekReader;
-extern io·PeekReader sys·PeekReader;
-
-typedef struct io·Writer
-{
-    int (*write)(void*, int sz, int n, void *buf);
-} io·Writer;
-extern io·Writer sys·Writer;
-
-typedef struct io·Putter
-{
-    int (*put) (void*, byte);
-    int (*puts)(void*, string);
-} io·Putter;
-extern io·Putter sys·Putter;
-
-typedef struct io·PutWriter
-{
-    io·Writer;
-    io·Putter;
-} io·PutWriter;
-extern io·PutWriter sys·PutWriter;
-
-typedef struct io·ReadWriter
-{
-    io·Reader;
-    io·Writer;
-} io·ReadWriter;
-extern io·ReadWriter sys·ReadWriter;
-
-/* XXX: change casing */
-enum
-{
-    ReadOK  = R_OK,
-    WriteOK = W_OK,
-    ExecOK  = X_OK,
-};
-
-/* XXX(deprecated): file handling */
-
-typedef FILE io·Stream;
-typedef struct stat io·Stat;
-
-io·Stream *io·open(byte *name, byte *mode);
-int        io·fd(io·Stream *s);
-int        io·stat(io·Stream *s, io·Stat *buf);
-int        io·close(io·Stream *s);
-byte       io·getbyte(io·Stream *s);
-int        io·ungetbyte(io·Stream *s, byte c);
-int        io·read(io·Stream *s, int sz, int n, void *buf);
-int        io·readln(io·Stream *s, int n, byte *buf);
-int        io·putbyte(io·Stream *s, byte c);
-int        io·putstring(io·Stream *s, string str);
-int        io·write(io·Stream *s, int sz, int n, void *buf);
-int        io·flush(io·Stream *s);
-int        io·seek(io·Stream *s, long off, enum SeekPos whence);
-long       io·tell(io·Stream *s);
-
-/* basic os helpers */
-
-int     os·exists(byte *path, int flag);
-byte   *os·dirname(byte *path);
-byte   *os·basename(byte *path);
-int     os·sep(void);
-
-/* io interfaces */
-/* buffered i/o */
-typedef struct io·Buffer io·Buffer;
-
-enum
-{
-    bufio·size   = 2*4096,
-    bufio·ungets = 8,
-    bufio·eof    = -1,
-    bufio·err    = -2,
-
-    bufio·nil = 1 << 0,
-    bufio·rdr = 1 << 1,
-    bufio·wtr = 1 << 2,
-    bufio·end = 1 << 3,
-};
-
-struct io·Buffer
-{
-    int state;
-    int runesize;
-    void *h;
-    union {
-        io·Reader rdr;
-        io·Writer wtr;
-    };
-    vlong size;
-    byte *beg, *pos, *end;
-    byte  buf[bufio·size + bufio·ungets];
-};
-
-int   bufio·initreader(io·Buffer *buf, io·Reader rdr, void *h);
-void  bufio·finireader(io·Buffer *buf);
-int   bufio·getbyte(io·Buffer *buf);
-int   bufio·ungetbyte(io·Buffer *buf, byte c);
-int   bufio·read(io·Buffer *buf, int sz, int n, void *out);
-
-// -----------------------------------------------------------------------------
-// memory mapped files
-
-typedef struct mmap·Reader
-{
-    vlong len;
-    union{
-        char  *b;
-        uchar *u;
-    };
-} mmap·Reader;
-
-mmap·Reader mmap·open(byte *name);
-int         mmap·close(mmap·Reader rdr);
-
-// -----------------------------------------------------------------------------
-// filesystem
-
-#define iota(x) 1 << (x)
-enum
-{
-    fs·preorder = iota(0),
-    fs·nolinks  = iota(1),
-    fs·verbose  = iota(2),
-};
-#undef iota
-
-typedef struct fs·Walker  fs·Walker;
-typedef struct fs·History fs·History;
-
-struct fs·Walker
-{
-    int fd, lev, max, err;
-    uchar  flags : 4;
-    fs·History *hist;
-    struct {
-        void  *data;
-        int  (*func)(void *data, char *relp, char *absp, io·Stat* info);
-    };
-    char *base, *end, path[4096];
-};
-
-int  fs·init(fs·Walker *, char *path);
-void fs·fini(fs·Walker *);
-void fs·walk(fs·Walker *);
-
-// -----------------------------------------------------------------------------
-// libflate
-// NOTE: Experimental!
-
-typedef struct flate·Reader flate·Reader;
-typedef struct flate·Writer flate·Writer;
-
-flate·Reader *flate·openreader(io·Reader rdr, void* r, mem·Allocator mem, void* m);
-int           flate·read(flate·Reader *rdr, int sz, int n, void *buf);
-int           flate·closereader(flate·Reader *rdr);
-
-flate·Writer *flate·openwriter(io·Writer wtr, void* w, mem·Allocator mem, void* m);
-int           flate·write(flate·Writer *wtr, int sz, int n, void *buf);
-int           flate·closewriter(flate·Writer *wtr);
-
-// -----------------------------------------------------------------------------
-// libgz
-
-typedef void gz·Stream;
-
-/* interfaces */
-extern io·Reader     gz·Reader;
-extern io·Peeker     gz·Peeker;
-extern io·Seeker     gz·Seeker;
-extern io·SeekReader gz·SeekReader;
-extern io·PeekReader gz·PeekReader;
-
-extern io·Writer     gz·Writer;
-extern io·Putter     gz·Putter;
-extern io·PutWriter  gz·PutWriter;
-extern io·ReadWriter gz·ReadWriter;
-
-gz·Stream *gz·open(byte *path, byte *mode);
-int        gz·close(gz·Stream* s);
-int        gz·read(gz·Stream *s, int sz, int n, void* buf);
-int        gz·readln(gz·Stream *s, int n, byte *buf);
-byte       gz·getbyte(gz·Stream *s);
-int        gz·ungetbyte(gz·Stream *s, byte c);
-int        gz·write(gz·Stream *s, int sz, int n, void* buf);
-int        gz·putbyte(gz·Stream *s, byte str);
-int        gz·putstring(gz·Stream *s, byte *str);
-int        gz·printf(gz·Stream *s, byte *fmt, ...);
-int        gz·flush(gz·Stream *s);
-int        gz·seek(gz·Stream *s, long off, enum SeekPos whence);
-long       gz·tell(gz·Stream *s);
-
-// -----------------------------------------------------------------------------
-// libjson
-// NOTE: Experimental!
-
-// -----------------------------------------------------------------------------
-// error handling functions
-
-void exits(char *s);
-void errorf(byte* fmt, ...);
-void verrorf(byte* fmt, va_list args);
-void panicf(byte *fmt, ...);
-void vpanicf(byte *fmt, va_list args);
-
-// -----------------------------------------------------------------------------
-// sorting
-
-void sort·int(uintptr n, int arr[]);
-void sort·int8(uintptr n, int8 arr[]);
-void sort·int16(uintptr n, int16 arr[]);
-void sort·int32(uintptr n, int32 arr[]);
-void sort·int64(uintptr n, int64 arr[]);
-
-void sort·uint(uintptr n, uint arr[]);
-void sort·uint8(uintptr n, uint8 arr[]);
-void sort·uint16(uintptr n, uint16 arr[]);
-void sort·uint32(uintptr n, uint32 arr[]);
-void sort·uint64(uintptr n, uint64 arr[]);
-
-void sort·float(uintptr n, float arr[]);
-void sort·double(uintptr n, double arr[]);
-
-void sort·string(uintptr n, byte* arr[]);
-
-// -----------------------------------------------------------------------------
-// fast random number generation
-
-int    rng·init(uint64 seed);
-double rng·random(void);
-double rng·exponential(double lambda);
-bool   rng·bernoulli(double f);
-double rng·normal(void);
-uint64 rng·randi(int max);
-uint64 rng·poisson(double mean);
+/* must be included first */
+#include <base/mem.h>
+#include <base/string.h>
+#include <base/error.h>
+#include <base/io.h>
+
+#include <base/fmt.h>
+#include <base/utf.h>
+#include <base/coro.h>
+#include <base/mmap.h>
+#include <base/fs.h>
+#include <base/flate.h>
+#include <base/gz.h>
+#include <base/sort.h>
+#include <base/rng.h>
 
 // -----------------------------------------------------------------------------
 // variable arguments
@@ -420,7 +56,6 @@ uint64 rng·poisson(double mean);
 #endif
 
 extern char *argv0;
-int arg·bytetorune(uint32*, byte *);
 #define	ARGBEGIN	for((argv0?0:(argv0=*argv)),argv++,argc--;            \
 			    argv[0] && argv[0][0]=='-' && argv[0][1];                 \
 			    argc--, argv++) {                                         \
@@ -431,7 +66,7 @@ int arg·bytetorune(uint32*, byte *);
 					argc--; argv++; break;                                \
 				}                                                         \
 				_argc = 0;                                                \
-				while(*_args && (_args += arg·bytetorune(&_argc, _args)))\
+				while(*_args && (_args += utf8·decode(_args, &_argc)))\
 				switch(_argc)
 #define	ARGEND		SET(_argt);USED(_argt);USED(_argc);USED(_args);}USED(argv);USED(argc);
 #define	ARGF()		(_argt=_args, _args="",\
diff --git a/include/base/coro.h b/include/base/coro.h
new file mode 100644
index 0000000..e8f23b1
--- /dev/null
+++ b/include/base/coro.h
@@ -0,0 +1,7 @@
+#pragma once
+
+typedef struct Coro Coro;
+
+Coro*   coro·make(uintptr stk, uintptr (*func)(Coro*, uintptr));
+uintptr coro·yield(Coro *c, uintptr arg);
+int     coro·free(Coro *c);
diff --git a/include/base/error.h b/include/base/error.h
new file mode 100644
index 0000000..57c1a77
--- /dev/null
+++ b/include/base/error.h
@@ -0,0 +1,7 @@
+#pragma once
+
+void exits(char *s);
+void errorf(byte* fmt, ...);
+void verrorf(byte* fmt, va_list args);
+void panicf(byte *fmt, ...);
+void vpanicf(byte *fmt, va_list args);
diff --git a/include/base/flate.h b/include/base/flate.h
new file mode 100644
index 0000000..670c8e7
--- /dev/null
+++ b/include/base/flate.h
@@ -0,0 +1,16 @@
+#pragma once
+
+// -----------------------------------------------------------------------------
+// libflate
+// NOTE: Experimental!
+
+typedef struct flate·Reader flate·Reader;
+typedef struct flate·Writer flate·Writer;
+
+flate·Reader *flate·openreader(io·Reader rdr, void* r, mem·Allocator mem, void* m);
+int           flate·read(flate·Reader *rdr, int sz, int n, void *buf);
+int           flate·closereader(flate·Reader *rdr);
+
+flate·Writer *flate·openwriter(io·Writer wtr, void* w, mem·Allocator mem, void* m);
+int           flate·write(flate·Writer *wtr, int sz, int n, void *buf);
+int           flate·closewriter(flate·Writer *wtr);
diff --git a/include/libfmt.h b/include/base/fmt.h
index 4c187fd..4c187fd 100644
--- a/include/libfmt.h
+++ b/include/base/fmt.h
diff --git a/include/base/fs.h b/include/base/fs.h
new file mode 100644
index 0000000..6852a84
--- /dev/null
+++ b/include/base/fs.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#define iota(x) 1 << (x)
+enum
+{
+    fs·preorder = iota(0),
+    fs·nolinks  = iota(1),
+    fs·verbose  = iota(2),
+};
+#undef iota
+
+typedef struct fs·Walker  fs·Walker;
+typedef struct fs·History fs·History;
+
+struct fs·Walker
+{
+    int fd, lev, max, err;
+    uchar  flags : 4;
+    fs·History *hist;
+    struct {
+        void  *data;
+        int  (*func)(void *data, char *relp, char *absp, io·Stat* info);
+    };
+    char *base, *end, path[4096];
+};
+
+int  fs·init(fs·Walker *, char *path);
+void fs·fini(fs·Walker *);
+void fs·walk(fs·Walker *);
diff --git a/include/base/gz.h b/include/base/gz.h
new file mode 100644
index 0000000..3b3a830
--- /dev/null
+++ b/include/base/gz.h
@@ -0,0 +1,29 @@
+#pragma once
+
+typedef void gz·Stream;
+
+/* interfaces */
+extern io·Reader     gz·Reader;
+extern io·Peeker     gz·Peeker;
+extern io·Seeker     gz·Seeker;
+extern io·SeekReader gz·SeekReader;
+extern io·PeekReader gz·PeekReader;
+
+extern io·Writer     gz·Writer;
+extern io·Putter     gz·Putter;
+extern io·PutWriter  gz·PutWriter;
+extern io·ReadWriter gz·ReadWriter;
+
+gz·Stream *gz·open(byte *path, byte *mode);
+int        gz·close(gz·Stream* s);
+int        gz·read(gz·Stream *s, int sz, int n, void* buf);
+int        gz·readln(gz·Stream *s, int n, byte *buf);
+byte       gz·getbyte(gz·Stream *s);
+int        gz·ungetbyte(gz·Stream *s, byte c);
+int        gz·write(gz·Stream *s, int sz, int n, void* buf);
+int        gz·putbyte(gz·Stream *s, byte str);
+int        gz·putstring(gz·Stream *s, byte *str);
+int        gz·printf(gz·Stream *s, byte *fmt, ...);
+int        gz·flush(gz·Stream *s);
+int        gz·seek(gz·Stream *s, long off, enum SeekPos whence);
+long       gz·tell(gz·Stream *s);
diff --git a/include/base/io.h b/include/base/io.h
new file mode 100644
index 0000000..f8c0bcc
--- /dev/null
+++ b/include/base/io.h
@@ -0,0 +1,141 @@
+#pragma once
+
+enum SeekPos
+{
+    seek·cur = SEEK_CUR,
+    seek·set = SEEK_SET,
+    seek·end = SEEK_END
+};
+
+typedef struct io·Reader
+{
+    int (*read)(void*, int sz, int n, void *buf);
+} io·Reader;
+extern io·Reader sys·Reader;
+
+typedef struct io·Peeker
+{
+    byte  (*get)(void*);
+    int   (*unget)(void*, byte);
+} io·Peeker;
+extern io·Peeker sys·Peeker;
+
+typedef struct io·Seeker
+{
+    int        (*seek)(void *skr, long off, enum SeekPos whence);
+    long       (*tell)(void *skr);
+} io·Seeker;
+extern io·Seeker sys·Seeker;
+
+typedef struct io·SeekReader
+{
+    io·Seeker;
+    io·Reader;
+} io·SeekReader;
+extern io·SeekReader sys·SeekReader;
+
+typedef struct io·PeekReader
+{
+    io·Reader;
+    io·Peeker;
+} io·PeekReader;
+extern io·PeekReader sys·PeekReader;
+
+typedef struct io·Writer
+{
+    int (*write)(void*, int sz, int n, void *buf);
+} io·Writer;
+extern io·Writer sys·Writer;
+
+typedef struct io·Putter
+{
+    int (*put) (void*, byte);
+    int (*puts)(void*, string);
+} io·Putter;
+extern io·Putter sys·Putter;
+
+typedef struct io·PutWriter
+{
+    io·Writer;
+    io·Putter;
+} io·PutWriter;
+extern io·PutWriter sys·PutWriter;
+
+typedef struct io·ReadWriter
+{
+    io·Reader;
+    io·Writer;
+} io·ReadWriter;
+extern io·ReadWriter sys·ReadWriter;
+
+/* XXX: change casing */
+enum
+{
+    ReadOK  = R_OK,
+    WriteOK = W_OK,
+    ExecOK  = X_OK,
+};
+
+/* XXX(deprecated): file handling */
+
+typedef FILE io·Stream;
+typedef struct stat io·Stat;
+
+io·Stream *io·open(byte *name, byte *mode);
+int        io·fd(io·Stream *s);
+int        io·stat(io·Stream *s, io·Stat *buf);
+int        io·close(io·Stream *s);
+byte       io·getbyte(io·Stream *s);
+int        io·ungetbyte(io·Stream *s, byte c);
+int        io·read(io·Stream *s, int sz, int n, void *buf);
+int        io·readln(io·Stream *s, int n, byte *buf);
+int        io·putbyte(io·Stream *s, byte c);
+int        io·putstring(io·Stream *s, string str);
+int        io·write(io·Stream *s, int sz, int n, void *buf);
+int        io·flush(io·Stream *s);
+int        io·seek(io·Stream *s, long off, enum SeekPos whence);
+long       io·tell(io·Stream *s);
+
+/* basic os helpers */
+
+int     os·exists(byte *path, int flag);
+byte   *os·dirname(byte *path);
+byte   *os·basename(byte *path);
+int     os·sep(void);
+
+/* io interfaces */
+/* buffered i/o */
+typedef struct io·Buffer io·Buffer;
+
+enum
+{
+    bufio·size   = 2*4096,
+    bufio·ungets = 8,
+    bufio·eof    = -1,
+    bufio·err    = -2,
+
+    bufio·nil = 1 << 0,
+    bufio·rdr = 1 << 1,
+    bufio·wtr = 1 << 2,
+    bufio·end = 1 << 3,
+};
+
+struct io·Buffer
+{
+    int state;
+    int runesize;
+    void *h;
+    union {
+        io·Reader rdr;
+        io·Writer wtr;
+    };
+    vlong size;
+    byte *beg, *pos, *end;
+    byte  buf[bufio·size + bufio·ungets];
+};
+
+int   bufio·initreader(io·Buffer *buf, io·Reader rdr, void *h);
+void  bufio·finireader(io·Buffer *buf);
+int   bufio·getbyte(io·Buffer *buf);
+int   bufio·ungetbyte(io·Buffer *buf, byte c);
+int   bufio·read(io·Buffer *buf, int sz, int n, void *out);
diff --git a/include/base/mem.h b/include/base/mem.h
new file mode 100644
index 0000000..3f1b6e1
--- /dev/null
+++ b/include/base/mem.h
@@ -0,0 +1,63 @@
+#pragma once
+
+// ----------------------------------------------------------------------------
+// dynamic array
+
+typedef struct BufHdr
+{
+    vlong len;
+    vlong cap;
+    byte  buf[];
+} BufHdr;
+
+#define bufhdr(b)        ((BufHdr*)((uint8*)(b)-offsetof(BufHdr, buf)))
+#define buflen(b)        ((b) ? (bufhdr(b)->len) : 0)
+#define bufcap(b)        ((b) ? (bufhdr(b)->cap) : 0)
+#define bufend(b)        ((b) + buflen(b))
+#define bufsize(b)       ((b) ? (buflen(b) * sizeof((b)[0])) : 0)
+
+#define buffree(b)       ((b) ? (free(bufhdr(b)), (b) = nil) : 0)
+#define buffit(b, n)     ((n) <= bufcap(b) ? 0 : ((b) = ·bufgrow((b), (n), sizeof(*(b)))))
+
+#define bufpush(b, ...)  (buffit((b), 1 + buflen(b)), (b)[bufhdr(b)->len++] = (__VA_ARGS__))
+#define bufaddn(b, n)    (buffit(b, buflen(b)+n), bufhdr(b)->len += n, b+bufhdr(b)->len-n)
+
+#define bufpop(b)         ((b)[--bufhdr(b)->len])
+#define bufdel(b, i)      bufdeln((b), (i), 1)
+#define bufdeln(b, i, n)  (memmove((b)+(i), (b)+(i)+(n), sizeof(*(b))*(bufhdr(b)->len-(n)-(i)), bufhdr(b)->len -= (n))
+#define bufdelswap(b, i)  ((b)[i] = bufend(b)[-1], bufhdr(b)->len-=1)
+
+void* ·bufgrow(void*, vlong, vlong);
+
+// -----------------------------------------------------------------------------
+// memory allocation
+
+/* allocator interface */
+typedef struct mem·Allocator {
+    void *(*alloc)(void *heap, uint n, ulong size);
+    void  (*free)(void *heap, void *ptr);
+} mem·Allocator;
+
+extern mem·Allocator sys·Memory;
+
+typedef struct mem·Reallocator {
+    void *(*alloc)(void *iface, uint n, ulong size);
+    void *(*realloc)(void *iface, void *ptr, uint n, ulong size);
+    void  (*free)(void *iface, void *ptr);
+} mem·Reallocator;
+
+extern mem·Reallocator sys·FullMemory;
+
+/* simple memory arena */
+typedef struct mem·Arena mem·Arena;
+
+mem·Arena *mem·makearena(mem·Allocator from, void*);
+void      *mem·arenaalloc(mem·Arena *A, uint n, ulong size); 
+void       mem·freearena(mem·Arena *A);
+
+extern mem·Allocator mem·ArenaAllocator;
+
+/* generalized memxxx functions */
+void       memset64(void *dst, uint64 val, uintptr size);
+
+
diff --git a/include/base/mmap.h b/include/base/mmap.h
new file mode 100644
index 0000000..9c4bd5f
--- /dev/null
+++ b/include/base/mmap.h
@@ -0,0 +1,13 @@
+#pragma once
+
+typedef struct mmap·Reader
+{
+    vlong len;
+    union{
+        char  *b;
+        uchar *u;
+    };
+} mmap·Reader;
+
+mmap·Reader mmap·open(byte *name);
+int         mmap·close(mmap·Reader rdr);
diff --git a/include/base/rng.h b/include/base/rng.h
new file mode 100644
index 0000000..ee41509
--- /dev/null
+++ b/include/base/rng.h
@@ -0,0 +1,9 @@
+#pragma once
+
+int    rng·init(uint64 seed);
+double rng·random(void);
+double rng·exponential(double lambda);
+bool   rng·bernoulli(double f);
+double rng·normal(void);
+uint64 rng·randi(int max);
+uint64 rng·poisson(double mean);
diff --git a/include/base/sort.h b/include/base/sort.h
new file mode 100644
index 0000000..e9a0e87
--- /dev/null
+++ b/include/base/sort.h
@@ -0,0 +1,18 @@
+#pragma once
+
+void sort·int(uintptr n, int arr[]);
+void sort·int8(uintptr n, int8 arr[]);
+void sort·int16(uintptr n, int16 arr[]);
+void sort·int32(uintptr n, int32 arr[]);
+void sort·int64(uintptr n, int64 arr[]);
+
+void sort·uint(uintptr n, uint arr[]);
+void sort·uint8(uintptr n, uint8 arr[]);
+void sort·uint16(uintptr n, uint16 arr[]);
+void sort·uint32(uintptr n, uint32 arr[]);
+void sort·uint64(uintptr n, uint64 arr[]);
+
+void sort·float(uintptr n, float arr[]);
+void sort·double(uintptr n, double arr[]);
+
+void sort·string(uintptr n, byte* arr[]);
diff --git a/include/base/string.h b/include/base/string.h
new file mode 100644
index 0000000..08466fa
--- /dev/null
+++ b/include/base/string.h
@@ -0,0 +1,30 @@
+#pragma once
+
+typedef byte* string;
+
+/* augmented string functions */
+string  str·makecap(const char *s, vlong len, vlong cap);
+string  str·makelen(const char *s, vlong len);
+string  str·make(const char *s);
+string  str·makef(const char *fmt, ...);
+void    str·free(string s);
+int     str·len(const string s);
+int     str·cap(const string s);
+void    str·clear(string *s);
+void    str·grow(string *s, vlong delta);
+void    str·fit(string *s);
+int     str·appendlen(string *s, vlong len, const char *b);
+int     str·append(string *s, const char *b);
+int     str·appendf(string *s, const char *fmt, ...);
+int     str·appendbyte(string *s, const char b);
+bool    str·equals(const string s, const string t);
+int     str·find(string s, const char* substr);
+void    str·lower(string s);
+void    str·upper(string s);
+int     str·read(string s, int size, int n, void *buf);
+void    str·replace(string s, const byte* from, const byte* to);
+string* str·split(string s, const byte* tok);
+string  str·join(vlong len, byte** fields, const byte* sep);
+
+/* raw C string functions */
+char    *str·copyn(char *dst, char *src, int n);
diff --git a/include/libutf.h b/include/base/utf.h
index 846296c..846296c 100644
--- a/include/libutf.h
+++ b/include/base/utf.h