Merge branch 'main' into remove-tags

.env.example

@@ -3,8 +3,6 @@
 LITEFS_DIR=/litefs/data
 
 SOCIAL_PROTOCOLS_DATADIR=$HOME/social-protocols-data
-VOTE_EVENTS_PATH=$SOCIAL_PROTOCOLS_DATADIR/vote-events.jsonl
-SCORE_EVENTS_PATH=$SOCIAL_PROTOCOLS_DATADIR/score-events.jsonl
 APP_DATABASE_PATH=$SOCIAL_PROTOCOLS_DATADIR/sqlite.db
 APP_DATABASE_URL=file:$APP_DATABASE_PATH?connection_limit=1
 GB_DATABASE_PATH="$SOCIAL_PROTOCOLS_DATADIR/global-brain.db"

.github/workflows/test.yml

@@ -37,4 +37,4 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Test
-      run: earthly +ci-test
+      run: earthly --allow-privileged --no-output +ci-test

.gitignore

@@ -18,3 +18,7 @@ node_modules/
 .env
 .DS_Store
 GlobalBrain.jl/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

Earthfile

@@ -63,12 +63,13 @@ app-setup:
 app-build:
   FROM +app-setup
   RUN npm run build
-  SAVE ARTIFACT server-build AS LOCAL server-build
-  SAVE ARTIFACT build AS LOCAL build
-  SAVE ARTIFACT public AS LOCAL public
-  SAVE ARTIFACT node_modules AS LOCAL node_modules
-  SAVE ARTIFACT package-lock.json AS LOCAL package-lock.json
-  SAVE ARTIFACT package.json AS LOCAL package.json
+  SAVE ARTIFACT server-build
+  SAVE ARTIFACT build
+  SAVE ARTIFACT public
+  SAVE ARTIFACT node_modules
+  SAVE ARTIFACT package-lock.json
+  SAVE ARTIFACT package.json
+  SAVE ARTIFACT .npmrc
 
 app-deploy-litefs:
    FROM flyio/litefs:0.5.10
@@ -93,7 +94,7 @@ docker-image:
 
   # npm run build
   COPY --dir other app server public types index.js tsconfig.json remix.config.js tailwind.config.ts postcss.config.js components.json ./
-  COPY --dir +app-build/server-build +app-build/build +app-build/public +app-build/node_modules +app-build/package-lock.json +app-build/package.json ./
+  COPY --dir +app-build/server-build +app-build/build +app-build/public +app-build/node_modules +app-build/package-lock.json +app-build/package.json +app-build/.npmrc ./
 
 
   # startup & migrations
@@ -107,15 +108,40 @@ docker-image:
   ENV CACHE_DATABASE_PATH="/$LITEFS_DIR/$CACHE_DATABASE_FILENAME"
   ENV INTERNAL_PORT="8080"
   ENV PORT="8081"
-  ENV VOTE_EVENTS_PATH=/data/vote-events.jsonl
-  ENV SCORE_EVENTS_PATH=/data/score-events.jsonl
+
+  RUN nix-collect-garbage
+  RUN du -sh /* \
+   && find /app -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 du -sh | sort -hr | head -20 \
+   && find /nix/store -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 du -sh | sort -hr | head -20 \
+   && find /app/node_modules -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 du -sh | sort -hr | head -20
 
   # starting the application is defined in litefs.yml
   # test locally without litefs:
   # docker run -e SESSION_SECRET -e INTERNAL_COMMAND_TOKEN -e HONEYPOT_SECRET sha256:xyzxyz /bin/sh startup.sh
   CMD ["/bin/sh", "-c", "/usr/local/bin/litefs mount"]
   SAVE IMAGE jabble:latest
 
+docker-image-e2e-test:
+  # set up an image with dind (docker in docker) and nix
+  FROM earthly/dind:alpine-3.19-docker-25.0.5-r0
+  RUN apk add curl \
+   && curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \
+    --extra-conf "sandbox = false" \
+    --init none \
+    --no-confirm
+  ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin"
+  WORKDIR /app
+  COPY flake.nix flake.lock .
+  RUN nix develop ".#e2e" --command echo warmed up
+  COPY --dir e2e playwright.config.ts ./
+  COPY docker-compose.yml ./
+  WITH DOCKER --load jabble:latest=+docker-image
+    RUN docker image ls \
+     && (docker-compose up &) \
+     && echo waiting for http server to come online... \
+     && timeout 60s sh -c 'until curl --silent --fail http://localhost:8081 > /dev/null; do sleep 1; done' \
+     && CI=true nix develop --impure ".#e2e" --command playwright test
+  END
 
 app-deploy:
   # run locally:
@@ -146,7 +172,7 @@ app-lint:
 ci-test:
   BUILD +app-typecheck
   BUILD +app-lint
-  BUILD +docker-image
+  BUILD +docker-image-e2e-test
 
 ci-deploy:
   # To run manually:

docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+  jabble:
+    image: jabble:latest
+    container_name: jabble
+    ports:
+      - "8081:8081"
+    environment:
+      - SESSION_SECRET=super-duper-s3cret
+      - HONEYPOT_SECRET=super-duper-s3cret
+      - INTERNAL_COMMAND_TOKEN=some-made-up-token
+    command: /bin/sh startup.sh
+    init: true # make ctrl+c work
+    restart: no

e2e/example.spec.ts

@@ -0,0 +1,7 @@
+import { test, expect } from '@playwright/test'
+
+test('has title', async ({ page }) => {
+	await page.goto('http://localhost:8081')
+
+	await expect(page).toHaveTitle(/Jabble/)
+})

flake.nix

@@ -29,12 +29,20 @@
               python3 # for node-gyp
               gcc # for node-gyp
 
+              playwright-driver.browsers # e2e tests
+              playwright-test # e2e tests
+
               earthly
               docker
+              docker-compose
               flyctl
 
               # darwin.apple_sdk.frameworks.Security
             ];
+            shellHook = ''
+              export PLAYWRIGHT_BROWSERS_PATH="${pkgs.playwright-driver.browsers}"
+              export PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=true
+            '';
           };
           build = pkgs.mkShellNoCC {
             buildInputs = with pkgs; [
@@ -44,6 +52,16 @@
               gcc # for node-gyp
             ];
           };
+          e2e = pkgs.mkShellNoCC {
+            buildInputs = with pkgs; [
+              playwright-driver.browsers # e2e tests
+              playwright-test # e2e tests
+            ];
+            shellHook = ''
+              export PLAYWRIGHT_BROWSERS_PATH="${pkgs.playwright-driver.browsers}"
+              export PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=true
+            '';
+          };
           production = pkgs.mkShellNoCC {
             buildInputs = with pkgs; [
               nodejs_20

justfile

@@ -16,7 +16,6 @@ reset-db:
 reset-all:
   rm -rf ~/social-protocols-data/*
   mkdir -p ~/social-protocols-data
-  touch $SCORE_EVENTS_PATH
   rm -f "$GB_DATABASE_PATH"
   just reset-db
 
@@ -79,11 +78,7 @@ docker-build-mac:
 
 # run the app in the docker container (you must run docker-build first)
 docker-run:
-	docker run --rm -it -p 8081:8081 -e SESSION_SECRET -e INTERNAL_COMMAND_TOKEN -e HONEYPOT_SECRET --name jabble jabble:latest /bin/sh startup.sh 
-
-# delete the docker container
-docker-rm:
-	docker rm -f jabble
+  docker-compose up
 
 # exec /bin/bash in the running docker container
 docker-exec:
@@ -93,8 +88,6 @@ download-production-data:
 	# todo: use sqlite .backup command and download copy
 	rm -rf $SOCIAL_PROTOCOLS_DATADIR/production/
 	mkdir -p $SOCIAL_PROTOCOLS_DATADIR/production/
-	fly ssh sftp get /data/score-events.jsonl $SOCIAL_PROTOCOLS_DATADIR/production/score-events.jsonl
-	fly ssh sftp get /data/vote-events.jsonl $SOCIAL_PROTOCOLS_DATADIR/production/vote-events.jsonl
 	fly ssh sftp get /litefs/data/sqlite.db $SOCIAL_PROTOCOLS_DATADIR/production/sqlite.db
 	fly ssh sftp get /litefs/data/global-brain.db $SOCIAL_PROTOCOLS_DATADIR/production/global-brain.db
 	fly ssh sftp get /litefs/data/sqlite.db $SOCIAL_PROTOCOLS_DATADIR/production/sqlite.db-wal
@@ -117,3 +110,5 @@ install-node-extension-from-earthly:
   (cd ./GlobalBrain.jl/globalbrain-node && npm install)
   npm install --ignore-scripts --save './GlobalBrain.jl/globalbrain-node'
 
+recent-sessions:
+	fly ssh console -C 'other/recent-sessions.sh'

other/recent-sessions.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sqlite3 -column $APP_DATABASE_PATH $'select username, datetime(session.createdAt/1000, \'unixepoch\') as login_time_utc from session join user on userId = user.id order by session.createdAt desc limit 10;'