Build System & Binding Definitions

.github/workflows/push.yml

@@ -39,6 +39,9 @@ jobs:
         platform: [x64]
     steps:
       - uses: actions/checkout@v3
+      - name: install libcurl
+        run: |
+          sudo apt-get install -qy libcurl4-openssl-dev
       - name: compile
         run: |
           make lo
@@ -52,7 +55,7 @@ jobs:
           name: lo-linux-x64
 
   build-linux-arm:
-#    if: ${{ false }}  # disable
+    if: ${{ false }}  # disable
     name: linux-arm64
     runs-on: ubuntu-22.04
     strategy:
@@ -61,6 +64,9 @@ jobs:
         platform: [x64]
     steps:
       - uses: actions/checkout@v3
+      - name: install libcurl
+        run: |
+          sudo apt-get install -qy libcurl4-openssl-dev
       - name: install arm64 tools
         run: |
           sudo apt-get install -qy binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
@@ -74,7 +80,7 @@ jobs:
           name: lo-linux-arm64
 
   build-windows:
-#    if: ${{ false }}  # disable
+    if: ${{ false }}  # disable
     name: windows
     runs-on: windows-latest
     strategy:

.gitignore

@@ -10,4 +10,5 @@ v8
 *.so
 lo
 .vscode
-scratch
+scratch
+deps

Makefile

@@ -1,18 +1,20 @@
 C=clang
 CC=clang++
 LARGS=-rdynamic
-CCARGS=-std=c++17 -c
-CARGS=-c
+CCARGS=-std=c++17 -c -fno-omit-frame-pointer -fno-rtti -fno-exceptions
+CARGS=-c -fno-omit-frame-pointer
 WARN=-Werror -Wpedantic -Wall -Wextra -Wno-unused-parameter
 OPT=-O3
 VERSION=0.0.4-pre
-V8_VERSION=12.0
+V8_VERSION=1.0.0
 RUNTIME=lo
 LO_HOME=$(shell pwd)
-BINDINGS=lib/core/core.a
+BINDINGS=core.o curl.o inflate.a
 ARCH=x64
 os=linux
 TARGET=${RUNTIME}
+LIBS=-lcurl
+V8_FLAGS=-DV8_COMPRESS_POINTERS -DV8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=0
 
 ifeq ($(OS),Windows_NT)
 	os=win
@@ -31,52 +33,63 @@ else
     endif
 endif
 
-v8/include:
+.PHONY: help clean
+
+help:
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9\/_\.-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+v8/include: ## download the v8 headers
 	curl -L -o v8-include.tar.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/include.tar.gz
 	tar -xvf v8-include.tar.gz
 ifneq ($(os),win)
 	rm -f v8-include.tar.gz
 endif
 
-v8/libv8_monolith.a:
-	curl -L -o v8/libv8_monolith.a.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/libv8_monolith-${os}-${ARCH}.a.tar.gz
+v8/libv8_monolith.a: ## download the v8 static libary for linux/macos
+	curl -L -o v8/libv8_monolith.a.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/libv8_monolith-${os}-${ARCH}.a.gz
 	gzip -d v8/libv8_monolith.a.gz
 	rm -f v8/libv8_monolith.a.gz
 
-v8/v8_monolith.lib:
-	curl -L -o v8/v8_monolith.lib.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/libv8_monolith-${os}-${ARCH}.lib.tar.gz
+v8/v8_monolith.lib: ## download the v8 static library for windows
+	curl -L -o v8/v8_monolith.lib.gz https://github.com/just-js/v8/releases/download/${V8_VERSION}/libv8_monolith-${os}-${ARCH}.lib.gz
 	gzip -d v8/v8_monolith.lib.gz
 
-${RUNTIME}: v8/include v8/libv8_monolith.a main.js ${BINDINGS}
-	@echo building ${RUNTIME} for ${os} on ${ARCH}
+main.o: ## compile the main.cc object file
+	$(CC) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' -I./v8 -I./v8/include ${WARN} ${V8_FLAGS} main.cc
+
+builtins.o: ## link all source files and assets into an object file
 ifeq (${os},linux)
 	$(C) ${CARGS} builtins_linux.S -o builtins.o
 else
 	$(C) ${CARGS} builtins.S -o builtins.o
 endif
-	$(CC) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' -I./v8 -I./v8/include ${WARN} main.cc
-	$(CC) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' -I./v8 -I./v8/include ${WARN} ${RUNTIME}.cc
-	$(CC) $(LARGS) ${OPT} main.o ${RUNTIME}.o builtins.o ${BINDINGS} v8/libv8_monolith.a -o ${TARGET}
 
-${RUNTIME}.exe: v8/include v8/v8_monolith.lib main.js
+${RUNTIME}.o: ## compile runtime into an object file 
+	$(CC) ${CCARGS} ${OPT} -DRUNTIME='"${RUNTIME}"' -DVERSION='"${VERSION}"' ${V8_FLAGS} -I./v8 -I./v8/include ${WARN} ${RUNTIME}.cc
+
+${RUNTIME}: v8/include v8/libv8_monolith.a main.js ${BINDINGS} builtins.o main.o ${RUNTIME}.o ## link the runtime for linux/macos
+	@echo building ${RUNTIME} for ${os} on ${ARCH}
+	$(CC) $(LARGS) ${OPT} main.o ${RUNTIME}.o builtins.o ${BINDINGS} v8/libv8_monolith.a ${LIBS} -o ${TARGET}
+
+${RUNTIME}.exe: v8/include v8/v8_monolith.lib main.js ## link the runtime for windows
 	cl /EHsc /std:c++17 /DRUNTIME='"${RUNTIME}"' /DVERSION='"${VERSION}"' /I./v8 /I./v8/include /c main.cc
 	cl /EHsc /std:c++17 /DRUNTIME='"${RUNTIME}"' /DVERSION='"${VERSION}"' /I./v8 /I./v8/include /c ${RUNTIME}.cc
 	cl v8/v8_monolith.lib ${RUNTIME}.obj main.obj winmm.lib dbghelp.lib advapi32.lib /link /out:${TARGET}.exe
 
-check:
-	./${RUNTIME} test/runtime.js
-
-lib/core/core.a: lib/core/api.js
-	$(MAKE) BINDING=core staticlib
+core.o: lib/core/core.cc ## build the core binding
+	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o core.o lib/core/core.cc
 
-lib/${BINDING}/${BINDING}.a: lib/${BINDING}/api.js
-	$(MAKE) BINDING=${BINDING} staticlib
+curl.o: lib/curl/curl.cc ## build the curl binding
+	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include $(WARN) ${V8_FLAGS} -o curl.o lib/curl/curl.cc
 
-staticlib: v8/include v8/libv8_monolith.a lib/${BINDING}/api.js
-	ARCH="${ARCH}" os="${os}" LARGS="${LARGS}" WARN="${WARN}" LO_HOME="${LO_HOME}" CCARGS="${CCARGS}" OPT="${OPT}" $(MAKE) -C lib/${BINDING}/ ${BINDING}.a
+inflate.a: lib/inflate/inflate.cc ## build the curl binding
+	$(C) -fPIC $(CARGS) $(OPT) -I. -I./v8 -I./v8/include -Ilib/inflate -o em_inflate.o lib/inflate/em_inflate.c
+	$(CC) -fPIC $(CCARGS) $(OPT) -I. -I./v8 -I./v8/include -Ilib/inflate $(WARN) ${V8_FLAGS} -o inflate.o lib/inflate/inflate.cc
+	ar crsT inflate.a inflate.o em_inflate.o
 
-sharedlib: v8/include v8/libv8_monolith.a lib/${BINDING}/${BINDING}.a
-	ARCH="${ARCH}" os="${os}" LARGS="${LARGS}" WARN="${WARN}" LO_HOME="${LO_HOME}" CCARGS="${CCARGS}" OPT="${OPT}" $(MAKE) -C lib/${BINDING}/ ${BINDING}.so
+check: ## run the runtime sanity tests
+	./${RUNTIME} test/dump.js
+	./${RUNTIME} test/runtime.js
 
 docs:
 	rm -fr docs
@@ -85,6 +98,10 @@ docs:
 	mv docs-$(VERSION) docs
 	rm -f docs.tar.gz
 
+install:
+	mkdir -p ${HOME}/.lo/bin
+	cp lo ${HOME}/.lo/bin/
+
 clean:
 ifeq ($(os),win)
 	@del /q *.obj > NUL 2>&1
@@ -93,6 +110,9 @@ ifeq ($(os),win)
 	@del /q ${RUNTIME}.lib > NUL 2>&1
 else
 	rm -f *.o
+	rm -f *.a
+	rm -f lib/**/*.a
+	rm -f lib/**/*.so
 	rm -f ${RUNTIME}
 endif
 

TODO.md

@@ -37,21 +37,41 @@
 - [ ] **todo**:  can we unload bindings and modules? have a look into this
 - [ ] **bug**:   if i embed the binding definition and change it then the lib won't rebuild as it uses embedded one
 - [ ] **todo**:  change binding defs so we can have multiple entries with same name but with different options for arch and platform
+- [ ] **todo**:  pass a flag to gen to tell it what os/arch we want to generate for
+- [ ] **bug**:   when rebuilding after changing bindings defs, they don't get re-generated as the ones on disk are not re-loaded.
+- [ ] **todo**:  clean up lib/gen.js. it's a real mess
+- [ ] **todo**:  have an embed cache separate from lib and require caches so we always load them from disk. hmmm... we need a nice way to handle this
+- [ ] **todo**:  module resolution is really broken
+- [ ] **todo**:  setTimeout, clearTimeout, setInterval, clearInterval
+- [ ] **todo**:  performance.now()
+- [ ] **todo**:  in assert, strip the assert line from the stack trace on the error
+- [ ] **question**: should we have something like __dirname on each module?
+- [ ] **todo**:  make lib/proc exec() async and used pidfd_open to monitor process on event loop
+- [ ] **todo**:  freeze core apis/intrinsics
+- [ ] **question**: how do we handle compiling dependencies of bindings cross platform if we don't depend on make?
+                    we could just write the build script as js?
+- [ ] **question**: how do we align structs in memory?
+- [ ] **todo**:  ability to chain cli args together
+- [ ] **todo**:  repl - doesn't really need async for now - ```lo repl```
+- [ ] **todo**:  rename lo.library and lo.libraries to lo.binding and lo.bindings
+
 
 ## features
 
+- [ ] **commands**: ability to host command scripts so i can run ```lo cmd blah``` and it will run blah.js from cmd direction in current dir or $HOME/.lo/cmd
 - [ ] **fetch**: a robust and fast fetch implementation
-- [ ] **WebSocket**: a robust and fast websocket implementation - client and server
 - [ ] **serve**: a robust and fast http serve implementation with Request and Response
 - [ ] **ffi**: a robust and fast ffi implementation
 - [ ] **spawn**: a robust and fast process spawning and control implementation
-- [ ] **Worker**: a robust and fast Web Worker implementation
 - [ ] **resources**: a solution for tracking handles, pointers and file descriptors
 - [ ] **hot loading**: look at ability to easily swap code out on the fly
 - [ ] **v8 api**: create a simple c api around v8 (like rusty_v8) so we can use it in bindings and compile/link bindings with tcc (maybe - i think so - the bindings libraries can be plain c)
+- [ ] **tracing**: a system for hooking into traces events for logging, metrics etc.
 
 ## modules
 
+- [ ] **Worker**: a robust and fast Web Worker implementation
+- [ ] **WebSocket**: a robust and fast websocket implementation - client and server
 - [ ] **sqlite**: a robust and fast sqlite implementation
 - [ ] **thread**: thread library