Self-hosted compiler: tokenizing hello world

.github/workflows/linux.yml

@@ -27,3 +27,11 @@ jobs:
             git status --ignored
             exit 1
           fi
+
+  tokenizers:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: sudo apt install -y llvm-13-dev clang-13 make valgrind
+      - run: LLVM_CONFIG=llvm-config-13 make
+      - run: ./tokenizers.sh

.github/workflows/windows.yml

@@ -131,3 +131,17 @@ jobs:
         shell: bash
       - run: cd "test dir" && ./runtests.sh --verbose
         shell: bash
+
+  tokenizers:
+    needs: build
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-zip
+      - run: unzip jou.zip
+      - run: mv tokenizers.sh self_hosted jou
+        shell: bash
+      - run: (cd jou && ./tokenizers.sh)
+        shell: bash

self_hosted/errors_and_warnings.jou

@@ -0,0 +1,25 @@
+from "stdlib/process.jou" import exit
+from "stdlib/io.jou" import stdout, stderr, fprintf, fflush
+
+struct Location:
+    path: byte*  # Not owned. Points to a string that is held elsewhere.
+    lineno: int
+
+def fail(location: Location, message: byte*) -> void:
+    # When stdout is redirected to same place as stderr,
+    # make sure that normal printf()s show up before our error.
+    fflush(stdout)
+    fflush(stderr)
+
+    fprintf(stderr, "compiler error in file \"%s\"", location.path)
+    if location.lineno != 0:
+        fprintf(stderr, ", line %d", location.lineno)
+    fprintf(stderr, ": %s\n", message)
+
+    exit(1)
+
+# TODO: doesn't really belong here
+def assert(b: bool) -> void:
+    if not b:
+        fprintf(stderr, "assertion failed\n")
+        exit(1)