Merge pull request #21 from billywhizz/main

0.0.12-pre

.github/workflows/push.yml

@@ -24,8 +24,8 @@ jobs:
         if: ${{ matrix.platform == 'x64' }}
         run: |
           make check
-          brew install libffi lz4 zstd
-          ./lo test/build.js
+          brew install libffi lz4 zstd jsonschema
+          LO_PREFIX=/usr/local/opt ./lo test/build.js
       - name: upload runtime artifact
         uses: actions/upload-artifact@v3
         with:

.gitignore

@@ -16,3 +16,7 @@ deps
 notes.md
 lib/inflate/em_inflate.c
 lib/inflate/em_inflate.h
+lib/**/*.cc
+!lib/core/core.cc
+!lib/inflate/inflate.cc
+!lib/mach/mach.cc

Makefile

@@ -5,7 +5,7 @@ CCARGS=-std=c++17 -c -fno-omit-frame-pointer -fno-rtti -fno-exceptions
 CARGS=-c -fno-omit-frame-pointer
 WARN=-Werror -Wpedantic -Wall -Wextra -Wno-unused-parameter
 OPT=-O3
-VERSION=0.0.11-pre
+VERSION=0.0.12-pre
 V8_VERSION=1.0.0
 RUNTIME=lo
 LO_HOME=$(shell pwd)
@@ -25,6 +25,7 @@ else
 			LARGS+=-s
     else ifeq ($(UNAME_S),Darwin)
 			os=mac
+			BINDINGS+=mach.o
 			LARGS+=-s -w
 			ifeq ($(ARCH),arm64)
 				LARGS+=-arch arm64
@@ -84,6 +85,9 @@ ${RUNTIME}.exe: v8/include v8/v8_monolith.lib main.js ## link the runtime for wi
 	cl /EHsc /std:c++17 /DRUNTIME='"${RUNTIME}"' /DVERSION='"${VERSION}"' /I./v8 /I./v8/include /c ${RUNTIME}.cc
 	cl v8/v8_monolith.lib ${RUNTIME}.obj main.obj winmm.lib dbghelp.lib advapi32.lib /link /out:${TARGET}.exe
 
+mach.o: lib/mach/mach.cc ## build the core binding
+	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o mach.o lib/mach/mach.cc
+
 core.o: lib/core/core.cc ## build the core binding
 	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o core.o lib/core/core.cc
 

TODO.md

@@ -68,6 +68,11 @@
 - [ ] **todo**: fix build.js for libssl so it works for compiling openssl from scratch
 - [ ] **todo**: add a safe_wrap method to main.js so we can wrap pointers with a check for maxsafeinteger
 - [ ] **todo**: when i do ```lo build binding <new_binding>``` generate binding definition with all cases covered and lots of comments, or not...
+- [ ] **todo**: figure out a nice way to write modules that work on different platforms. can we pre-process the js in some way to only include code for that platform in the binary?
+- [ ] **bug**:  when i do ```lo build``` after downloading runtime it fails because em_inflate source files are not in the binary and the build script is not available in lib/inflate. how do we resolve this?
+- [ ] **question**: how do we model constants like RTLD_DEFAULT which seem to be pointers to things?
+- [ ] **bug**:  on macos, when i do a ```lo build``` it generates same main.h for main.h main_win.h and main_mac.h
+- [ ] **todo**: we need a way to ignore auto-generated files for git, but also to be able to re-generate them to check in when we need to.
 
 ## features
 

builtins.S

@@ -88,6 +88,16 @@ __binary_lib_curl_api_js_start:
         .incbin "lib/curl/api.js"
         .global __binary_lib_curl_api_js_end
 __binary_lib_curl_api_js_end:
+.global __binary_lib_duckdb_api_js_start
+__binary_lib_duckdb_api_js_start:
+        .incbin "lib/duckdb/api.js"
+        .global __binary_lib_duckdb_api_js_end
+__binary_lib_duckdb_api_js_end:
+.global __binary_lib_duckdb_build_js_start
+__binary_lib_duckdb_build_js_start:
+        .incbin "lib/duckdb/build.js"
+        .global __binary_lib_duckdb_build_js_end
+__binary_lib_duckdb_build_js_end:
 .global __binary_lib_encode_api_js_start
 __binary_lib_encode_api_js_start:
         .incbin "lib/encode/api.js"
@@ -103,6 +113,11 @@ __binary_lib_inflate_api_js_start:
         .incbin "lib/inflate/api.js"
         .global __binary_lib_inflate_api_js_end
 __binary_lib_inflate_api_js_end:
+.global __binary_lib_inflate_build_js_start
+__binary_lib_inflate_build_js_start:
+        .incbin "lib/inflate/build.js"
+        .global __binary_lib_inflate_build_js_end
+__binary_lib_inflate_build_js_end:
 .global __binary_lib_libffi_api_js_start
 __binary_lib_libffi_api_js_start:
         .incbin "lib/libffi/api.js"
@@ -123,6 +138,11 @@ __binary_lib_mbedtls_api_js_start:
         .incbin "lib/mbedtls/api.js"
         .global __binary_lib_mbedtls_api_js_end
 __binary_lib_mbedtls_api_js_end:
+.global __binary_lib_mbedtls_build_js_start
+__binary_lib_mbedtls_build_js_start:
+        .incbin "lib/mbedtls/build.js"
+        .global __binary_lib_mbedtls_build_js_end
+__binary_lib_mbedtls_build_js_end:
 .global __binary_lib_net_api_js_start
 __binary_lib_net_api_js_start:
         .incbin "lib/net/api.js"
@@ -133,6 +153,11 @@ __binary_lib_pico_api_js_start:
         .incbin "lib/pico/api.js"
         .global __binary_lib_pico_api_js_end
 __binary_lib_pico_api_js_end:
+.global __binary_lib_pico_build_js_start
+__binary_lib_pico_build_js_start:
+        .incbin "lib/pico/build.js"
+        .global __binary_lib_pico_build_js_end
+__binary_lib_pico_build_js_end:
 .global __binary_lib_pthread_api_js_start
 __binary_lib_pthread_api_js_start:
         .incbin "lib/pthread/api.js"
@@ -158,18 +183,23 @@ __binary_lib_tcc_api_js_start:
         .incbin "lib/tcc/api.js"
         .global __binary_lib_tcc_api_js_end
 __binary_lib_tcc_api_js_end:
+.global __binary_lib_tcc_build_js_start
+__binary_lib_tcc_build_js_start:
+        .incbin "lib/tcc/build.js"
+        .global __binary_lib_tcc_build_js_end
+__binary_lib_tcc_build_js_end:
 .global __binary_lib_wireguard_api_js_start
 __binary_lib_wireguard_api_js_start:
         .incbin "lib/wireguard/api.js"
         .global __binary_lib_wireguard_api_js_end
 __binary_lib_wireguard_api_js_end:
+.global __binary_lib_wireguard_build_js_start
+__binary_lib_wireguard_build_js_start:
+        .incbin "lib/wireguard/build.js"
+        .global __binary_lib_wireguard_build_js_end
+__binary_lib_wireguard_build_js_end:
 .global __binary_lib_zlib_api_js_start
 __binary_lib_zlib_api_js_start:
         .incbin "lib/zlib/api.js"
         .global __binary_lib_zlib_api_js_end
 __binary_lib_zlib_api_js_end:
-.global __binary_lib_duckdb_api_js_start
-__binary_lib_duckdb_api_js_start:
-        .incbin "lib/duckdb/api.js"
-        .global __binary_lib_duckdb_api_js_end
-__binary_lib_duckdb_api_js_end:

builtins_linux.S

@@ -88,6 +88,16 @@ _binary_lib_curl_api_js_start:
         .incbin "lib/curl/api.js"
         .global _binary_lib_curl_api_js_end
 _binary_lib_curl_api_js_end:
+.global _binary_lib_duckdb_api_js_start
+_binary_lib_duckdb_api_js_start:
+        .incbin "lib/duckdb/api.js"
+        .global _binary_lib_duckdb_api_js_end
+_binary_lib_duckdb_api_js_end:
+.global _binary_lib_duckdb_build_js_start
+_binary_lib_duckdb_build_js_start:
+        .incbin "lib/duckdb/build.js"
+        .global _binary_lib_duckdb_build_js_end
+_binary_lib_duckdb_build_js_end:
 .global _binary_lib_encode_api_js_start
 _binary_lib_encode_api_js_start:
         .incbin "lib/encode/api.js"
@@ -103,6 +113,11 @@ _binary_lib_inflate_api_js_start:
         .incbin "lib/inflate/api.js"
         .global _binary_lib_inflate_api_js_end
 _binary_lib_inflate_api_js_end:
+.global _binary_lib_inflate_build_js_start
+_binary_lib_inflate_build_js_start:
+        .incbin "lib/inflate/build.js"
+        .global _binary_lib_inflate_build_js_end
+_binary_lib_inflate_build_js_end:
 .global _binary_lib_libffi_api_js_start
 _binary_lib_libffi_api_js_start:
         .incbin "lib/libffi/api.js"
@@ -123,6 +138,11 @@ _binary_lib_mbedtls_api_js_start:
         .incbin "lib/mbedtls/api.js"
         .global _binary_lib_mbedtls_api_js_end
 _binary_lib_mbedtls_api_js_end:
+.global _binary_lib_mbedtls_build_js_start
+_binary_lib_mbedtls_build_js_start:
+        .incbin "lib/mbedtls/build.js"
+        .global _binary_lib_mbedtls_build_js_end
+_binary_lib_mbedtls_build_js_end:
 .global _binary_lib_net_api_js_start
 _binary_lib_net_api_js_start:
         .incbin "lib/net/api.js"
@@ -133,6 +153,11 @@ _binary_lib_pico_api_js_start:
         .incbin "lib/pico/api.js"
         .global _binary_lib_pico_api_js_end
 _binary_lib_pico_api_js_end:
+.global _binary_lib_pico_build_js_start
+_binary_lib_pico_build_js_start:
+        .incbin "lib/pico/build.js"
+        .global _binary_lib_pico_build_js_end
+_binary_lib_pico_build_js_end:
 .global _binary_lib_pthread_api_js_start
 _binary_lib_pthread_api_js_start:
         .incbin "lib/pthread/api.js"
@@ -158,18 +183,23 @@ _binary_lib_tcc_api_js_start:
         .incbin "lib/tcc/api.js"
         .global _binary_lib_tcc_api_js_end
 _binary_lib_tcc_api_js_end:
+.global _binary_lib_tcc_build_js_start
+_binary_lib_tcc_build_js_start:
+        .incbin "lib/tcc/build.js"
+        .global _binary_lib_tcc_build_js_end
+_binary_lib_tcc_build_js_end:
 .global _binary_lib_wireguard_api_js_start
 _binary_lib_wireguard_api_js_start:
         .incbin "lib/wireguard/api.js"
         .global _binary_lib_wireguard_api_js_end
 _binary_lib_wireguard_api_js_end:
+.global _binary_lib_wireguard_build_js_start
+_binary_lib_wireguard_build_js_start:
+        .incbin "lib/wireguard/build.js"
+        .global _binary_lib_wireguard_build_js_end
+_binary_lib_wireguard_build_js_end:
 .global _binary_lib_zlib_api_js_start
 _binary_lib_zlib_api_js_start:
         .incbin "lib/zlib/api.js"
         .global _binary_lib_zlib_api_js_end
 _binary_lib_zlib_api_js_end:
-.global _binary_lib_duckdb_api_js_start
-_binary_lib_duckdb_api_js_start:
-        .incbin "lib/duckdb/api.js"
-        .global _binary_lib_duckdb_api_js_end
-_binary_lib_duckdb_api_js_end:

lib/build.js

@@ -170,9 +170,14 @@ function create_header (libs = [], bindings = [], opts) {
   config.os = 'win'
   writeFile(`${LO_HOME}/main_win.h`, encoder.encode(headerFile([...libs, 
     ...bindings.map(n => `lib/${n}/${n}.a`)], opts)))
-  config.os = os
+  config.os = 'mac'
+  // todo: this is a hack to get mach in core runtime on macos for now
+  writeFile(`${LO_HOME}/main_mac.h`, encoder.encode(headerFile([...libs, 
+    ...[...bindings.includes('mach') ? [] : ['mach'], ...bindings].map(n => `lib/${n}/${n}.a`)], opts)))
+  config.os = 'linux'
   writeFile(`${LO_HOME}/main.h`, encoder.encode(headerFile([...libs, 
     ...bindings.map(n => `lib/${n}/${n}.a`)], opts)))
+  config.os = os
 }
 
 async function build_runtime ({ libs = lo.builtins(), bindings = lo.libraries(), 
@@ -241,7 +246,7 @@ const encoder = new TextEncoder()
 const status = new Int32Array(2)
 
 // todo: clean up api so we can pass a config in and run builds through api
-const VERSION = getenv('VERSION') || '"0.0.11pre"'
+const VERSION = getenv('VERSION') || '"0.0.12pre"'
 const RUNTIME = getenv('RUNTIME') || '"lo"'
 const TARGET = getenv('TARGET') || 'lo'
 const LINK_TYPE = (getenv('LINK_TYPE') || '-rdynamic').split(' ')
@@ -328,26 +333,33 @@ const runtimes = {
       'lo.h',
       'lib/core/api.js',
       'lib/curl/api.js',
+      'lib/duckdb/api.js',
+      'lib/duckdb/build.js',
       'lib/encode/api.js',
       'lib/epoll/api.js',
       'lib/inflate/api.js',
+      'lib/inflate/build.js',
       'lib/libffi/api.js',
       'lib/libssl/api.js',
       'lib/lz4/api.js',
       'lib/mbedtls/api.js',
+      'lib/mbedtls/build.js',
       'lib/net/api.js',
       'lib/pico/api.js',
+      'lib/pico/build.js',
       'lib/pthread/api.js',
       'lib/seccomp/api.js',
       'lib/sqlite/api.js',
       'lib/system/api.js',
       'lib/tcc/api.js',
+      'lib/tcc/build.js',
       'lib/wireguard/api.js',
+      'lib/wireguard/build.js',
       'lib/zlib/api.js',
-      'lib/duckdb/api.js'
     ]
   },
   full: {
+    // todo: change this so we define the platforms in the api.js and we just filter above
     bindings: (os === 'linux' ? [
       'core', 
       'curl',
@@ -475,6 +487,10 @@ const runtimes = {
   }
 }
 
+if (os === 'mac') {
+  Object.keys(runtimes).forEach(name => runtimes[name].bindings.push('mach'))
+}
+
 async function build (args) {
   let verbose = false
   if (args.includes('-v')) {

lib/core/api.js

@@ -552,8 +552,23 @@ const constants = {
   O_TRUNC: 'i32', STDIN: 0, STDOUT: 1, STDERR: 2, O_CLOEXEC: 'i32',
   RUSAGE_SELF: 'i32', SEEK_SET: 'i32', SEEK_CUR: 'i32',
   SEEK_END: 'i32', S_IRWXO: 'i32', F_OK: 'i32', S_IFMT: 'i32', S_IFDIR: 'i32',
-  S_IFREG: 'i32', NAME_MAX: 'u32', O_RDWR: 'i32', O_DIRECTORY: 'i32'
+  S_IFREG: 'i32', NAME_MAX: 'u32', O_RDWR: 'i32', O_DIRECTORY: 'i32',
+  F_SETFL: 'i32', O_NONBLOCK: 'i32',
+  RTLD_NOW: 'i32', RTLD_LAZY: 'i32', EAGAIN: 'i32'
 }
+
+if (globalThis.lo) {
+  if (lo.core.os === 'mac') {
+    constants.RTLD_MAIN_ONLY = 'u64'
+    constants.RTLD_DEFAULT = 'u64'
+    constants.RTLD_SELF = 'u64'
+    constants.RTLD_NEXT = 'u64'
+  } else if (lo.core.os === 'linux') {
+// todo: we need a way to inject defines before the includes
+//    preamble = `#define _GNU_SOURCE\n${preamble}`
+  }
+}
+
 // list of headers to include
 const includes = [
   'unistd.h', 'sys/stat.h', 'fcntl.h', 'dirent.h', 'dlfcn.h', 'sys/mman.h',

lib/core/core.cc

@@ -1690,38 +1690,43 @@ void Init(Isolate* isolate, Local<ObjectTemplate> target) {
   SET_METHOD(isolate, module, "isolate_start", isolate_startSlow);
   SET_FAST_METHOD(isolate, module, "strnlen", &pFstrnlen, strnlenSlow);
 
-  SET_VALUE(isolate, module, "S_IFBLK", Integer::New(isolate, S_IFBLK));
-  SET_VALUE(isolate, module, "S_IFCHR", Integer::New(isolate, S_IFCHR));
-  SET_VALUE(isolate, module, "S_IFIFO", Integer::New(isolate, S_IFIFO));
-  SET_VALUE(isolate, module, "S_IRUSR", Integer::New(isolate, S_IRUSR));
-  SET_VALUE(isolate, module, "S_IWUSR", Integer::New(isolate, S_IWUSR));
-  SET_VALUE(isolate, module, "S_IRGRP", Integer::New(isolate, S_IRGRP));
-  SET_VALUE(isolate, module, "S_IWGRP", Integer::New(isolate, S_IWGRP));
-  SET_VALUE(isolate, module, "S_IROTH", Integer::New(isolate, S_IROTH));
-  SET_VALUE(isolate, module, "S_IWOTH", Integer::New(isolate, S_IWOTH));
-  SET_VALUE(isolate, module, "O_RDONLY", Integer::New(isolate, O_RDONLY));
-  SET_VALUE(isolate, module, "O_WRONLY", Integer::New(isolate, O_WRONLY));
-  SET_VALUE(isolate, module, "O_CREAT", Integer::New(isolate, O_CREAT));
-  SET_VALUE(isolate, module, "S_IRWXU", Integer::New(isolate, S_IRWXU));
-  SET_VALUE(isolate, module, "S_IRWXG", Integer::New(isolate, S_IRWXG));
-  SET_VALUE(isolate, module, "S_IXOTH", Integer::New(isolate, S_IXOTH));
-  SET_VALUE(isolate, module, "O_TRUNC", Integer::New(isolate, O_TRUNC));
-  SET_VALUE(isolate, module, "STDIN", Integer::New(isolate, 0));
-  SET_VALUE(isolate, module, "STDOUT", Integer::New(isolate, 1));
-  SET_VALUE(isolate, module, "STDERR", Integer::New(isolate, 2));
-  SET_VALUE(isolate, module, "O_CLOEXEC", Integer::New(isolate, O_CLOEXEC));
-  SET_VALUE(isolate, module, "RUSAGE_SELF", Integer::New(isolate, RUSAGE_SELF));
-  SET_VALUE(isolate, module, "SEEK_SET", Integer::New(isolate, SEEK_SET));
-  SET_VALUE(isolate, module, "SEEK_CUR", Integer::New(isolate, SEEK_CUR));
-  SET_VALUE(isolate, module, "SEEK_END", Integer::New(isolate, SEEK_END));
-  SET_VALUE(isolate, module, "S_IRWXO", Integer::New(isolate, S_IRWXO));
-  SET_VALUE(isolate, module, "F_OK", Integer::New(isolate, F_OK));
-  SET_VALUE(isolate, module, "S_IFMT", Integer::New(isolate, S_IFMT));
-  SET_VALUE(isolate, module, "S_IFDIR", Integer::New(isolate, S_IFDIR));
-  SET_VALUE(isolate, module, "S_IFREG", Integer::New(isolate, S_IFREG));
-  SET_VALUE(isolate, module, "NAME_MAX", Integer::New(isolate, NAME_MAX));
-  SET_VALUE(isolate, module, "O_RDWR", Integer::New(isolate, O_RDWR));
-  SET_VALUE(isolate, module, "O_DIRECTORY", Integer::New(isolate, O_DIRECTORY));
+  SET_VALUE(isolate, module, "S_IFBLK", Integer::New(isolate, (int32_t)S_IFBLK));
+  SET_VALUE(isolate, module, "S_IFCHR", Integer::New(isolate, (int32_t)S_IFCHR));
+  SET_VALUE(isolate, module, "S_IFIFO", Integer::New(isolate, (int32_t)S_IFIFO));
+  SET_VALUE(isolate, module, "S_IRUSR", Integer::New(isolate, (int32_t)S_IRUSR));
+  SET_VALUE(isolate, module, "S_IWUSR", Integer::New(isolate, (int32_t)S_IWUSR));
+  SET_VALUE(isolate, module, "S_IRGRP", Integer::New(isolate, (int32_t)S_IRGRP));
+  SET_VALUE(isolate, module, "S_IWGRP", Integer::New(isolate, (int32_t)S_IWGRP));
+  SET_VALUE(isolate, module, "S_IROTH", Integer::New(isolate, (int32_t)S_IROTH));
+  SET_VALUE(isolate, module, "S_IWOTH", Integer::New(isolate, (int32_t)S_IWOTH));
+  SET_VALUE(isolate, module, "O_RDONLY", Integer::New(isolate, (int32_t)O_RDONLY));
+  SET_VALUE(isolate, module, "O_WRONLY", Integer::New(isolate, (int32_t)O_WRONLY));
+  SET_VALUE(isolate, module, "O_CREAT", Integer::New(isolate, (int32_t)O_CREAT));
+  SET_VALUE(isolate, module, "S_IRWXU", Integer::New(isolate, (int32_t)S_IRWXU));
+  SET_VALUE(isolate, module, "S_IRWXG", Integer::New(isolate, (int32_t)S_IRWXG));
+  SET_VALUE(isolate, module, "S_IXOTH", Integer::New(isolate, (int32_t)S_IXOTH));
+  SET_VALUE(isolate, module, "O_TRUNC", Integer::New(isolate, (int32_t)O_TRUNC));
+  SET_VALUE(isolate, module, "STDIN", Number::New(isolate, (int64_t)0));
+  SET_VALUE(isolate, module, "STDOUT", Number::New(isolate, (int64_t)1));
+  SET_VALUE(isolate, module, "STDERR", Number::New(isolate, (int64_t)2));
+  SET_VALUE(isolate, module, "O_CLOEXEC", Integer::New(isolate, (int32_t)O_CLOEXEC));
+  SET_VALUE(isolate, module, "RUSAGE_SELF", Integer::New(isolate, (int32_t)RUSAGE_SELF));
+  SET_VALUE(isolate, module, "SEEK_SET", Integer::New(isolate, (int32_t)SEEK_SET));
+  SET_VALUE(isolate, module, "SEEK_CUR", Integer::New(isolate, (int32_t)SEEK_CUR));
+  SET_VALUE(isolate, module, "SEEK_END", Integer::New(isolate, (int32_t)SEEK_END));
+  SET_VALUE(isolate, module, "S_IRWXO", Integer::New(isolate, (int32_t)S_IRWXO));
+  SET_VALUE(isolate, module, "F_OK", Integer::New(isolate, (int32_t)F_OK));
+  SET_VALUE(isolate, module, "S_IFMT", Integer::New(isolate, (int32_t)S_IFMT));
+  SET_VALUE(isolate, module, "S_IFDIR", Integer::New(isolate, (int32_t)S_IFDIR));
+  SET_VALUE(isolate, module, "S_IFREG", Integer::New(isolate, (int32_t)S_IFREG));
+  SET_VALUE(isolate, module, "NAME_MAX", Integer::New(isolate, (uint32_t)NAME_MAX));
+  SET_VALUE(isolate, module, "O_RDWR", Integer::New(isolate, (int32_t)O_RDWR));
+  SET_VALUE(isolate, module, "O_DIRECTORY", Integer::New(isolate, (int32_t)O_DIRECTORY));
+  SET_VALUE(isolate, module, "F_SETFL", Integer::New(isolate, (int32_t)F_SETFL));
+  SET_VALUE(isolate, module, "O_NONBLOCK", Integer::New(isolate, (int32_t)O_NONBLOCK));
+  SET_VALUE(isolate, module, "RTLD_NOW", Integer::New(isolate, (int32_t)RTLD_NOW));
+  SET_VALUE(isolate, module, "RTLD_LAZY", Integer::New(isolate, (int32_t)RTLD_LAZY));
+  SET_VALUE(isolate, module, "EAGAIN", Integer::New(isolate, (int32_t)EAGAIN));
 
 
   SET_MODULE(isolate, target, "core", module);