diff --git a/.env.example b/.env.example index c707b355..afd007f4 100644 --- a/.env.example +++ b/.env.example @@ -3,8 +3,6 @@ LITEFS_DIR=/litefs/data SOCIAL_PROTOCOLS_DATADIR=$HOME/social-protocols-data -VOTE_EVENTS_PATH=$SOCIAL_PROTOCOLS_DATADIR/vote-events.jsonl -SCORE_EVENTS_PATH=$SOCIAL_PROTOCOLS_DATADIR/score-events.jsonl APP_DATABASE_PATH=$SOCIAL_PROTOCOLS_DATADIR/sqlite.db APP_DATABASE_URL=file:$APP_DATABASE_PATH?connection_limit=1 GB_DATABASE_PATH="$SOCIAL_PROTOCOLS_DATADIR/global-brain.db" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f16a0c4e..62e5c6fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,4 +37,4 @@ jobs: uses: actions/checkout@v4 - name: Test - run: earthly +ci-test + run: earthly --allow-privileged --no-output +ci-test diff --git a/.gitignore b/.gitignore index a2e77aeb..cfc05d40 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ node_modules/ .env .DS_Store GlobalBrain.jl/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/Earthfile b/Earthfile index 45006f53..01527052 100644 --- a/Earthfile +++ b/Earthfile @@ -63,12 +63,13 @@ app-setup: app-build: FROM +app-setup RUN npm run build - SAVE ARTIFACT server-build AS LOCAL server-build - SAVE ARTIFACT build AS LOCAL build - SAVE ARTIFACT public AS LOCAL public - SAVE ARTIFACT node_modules AS LOCAL node_modules - SAVE ARTIFACT package-lock.json AS LOCAL package-lock.json - SAVE ARTIFACT package.json AS LOCAL package.json + SAVE ARTIFACT server-build + SAVE ARTIFACT build + SAVE ARTIFACT public + SAVE ARTIFACT node_modules + SAVE ARTIFACT package-lock.json + SAVE ARTIFACT package.json + SAVE ARTIFACT .npmrc app-deploy-litefs: FROM flyio/litefs:0.5.10 @@ -93,7 +94,7 @@ docker-image: # npm run build COPY --dir other app server public types index.js tsconfig.json remix.config.js tailwind.config.ts postcss.config.js components.json ./ - COPY --dir +app-build/server-build +app-build/build +app-build/public +app-build/node_modules +app-build/package-lock.json +app-build/package.json ./ + COPY --dir +app-build/server-build +app-build/build +app-build/public +app-build/node_modules +app-build/package-lock.json +app-build/package.json +app-build/.npmrc ./ # startup & migrations @@ -107,8 +108,12 @@ docker-image: ENV CACHE_DATABASE_PATH="/$LITEFS_DIR/$CACHE_DATABASE_FILENAME" ENV INTERNAL_PORT="8080" ENV PORT="8081" - ENV VOTE_EVENTS_PATH=/data/vote-events.jsonl - ENV SCORE_EVENTS_PATH=/data/score-events.jsonl + + RUN nix-collect-garbage + RUN du -sh /* \ + && find /app -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 du -sh | sort -hr | head -20 \ + && find /nix/store -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 du -sh | sort -hr | head -20 \ + && find /app/node_modules -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 du -sh | sort -hr | head -20 # starting the application is defined in litefs.yml # test locally without litefs: @@ -116,6 +121,27 @@ docker-image: CMD ["/bin/sh", "-c", "/usr/local/bin/litefs mount"] SAVE IMAGE jabble:latest +docker-image-e2e-test: + # set up an image with dind (docker in docker) and nix + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 + RUN apk add curl \ + && curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ + --extra-conf "sandbox = false" \ + --init none \ + --no-confirm + ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + WORKDIR /app + COPY flake.nix flake.lock . + RUN nix develop ".#e2e" --command echo warmed up + COPY --dir e2e playwright.config.ts ./ + COPY docker-compose.yml ./ + WITH DOCKER --load jabble:latest=+docker-image + RUN docker image ls \ + && (docker-compose up &) \ + && echo waiting for http server to come online... \ + && timeout 60s sh -c 'until curl --silent --fail http://localhost:8081 > /dev/null; do sleep 1; done' \ + && CI=true nix develop --impure ".#e2e" --command playwright test + END app-deploy: # run locally: @@ -146,7 +172,7 @@ app-lint: ci-test: BUILD +app-typecheck BUILD +app-lint - BUILD +docker-image + BUILD +docker-image-e2e-test ci-deploy: # To run manually: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..611e49bf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + jabble: + image: jabble:latest + container_name: jabble + ports: + - "8081:8081" + environment: + - SESSION_SECRET=super-duper-s3cret + - HONEYPOT_SECRET=super-duper-s3cret + - INTERNAL_COMMAND_TOKEN=some-made-up-token + command: /bin/sh startup.sh + init: true # make ctrl+c work + restart: no diff --git a/e2e/example.spec.ts b/e2e/example.spec.ts new file mode 100644 index 00000000..a8fbd427 --- /dev/null +++ b/e2e/example.spec.ts @@ -0,0 +1,7 @@ +import { test, expect } from '@playwright/test' + +test('has title', async ({ page }) => { + await page.goto('http://localhost:8081') + + await expect(page).toHaveTitle(/Jabble/) +}) diff --git a/flake.nix b/flake.nix index e8de21a7..61279729 100644 --- a/flake.nix +++ b/flake.nix @@ -29,12 +29,20 @@ python3 # for node-gyp gcc # for node-gyp + playwright-driver.browsers # e2e tests + playwright-test # e2e tests + earthly docker + docker-compose flyctl # darwin.apple_sdk.frameworks.Security ]; + shellHook = '' + export PLAYWRIGHT_BROWSERS_PATH="${pkgs.playwright-driver.browsers}" + export PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=true + ''; }; build = pkgs.mkShellNoCC { buildInputs = with pkgs; [ @@ -44,6 +52,16 @@ gcc # for node-gyp ]; }; + e2e = pkgs.mkShellNoCC { + buildInputs = with pkgs; [ + playwright-driver.browsers # e2e tests + playwright-test # e2e tests + ]; + shellHook = '' + export PLAYWRIGHT_BROWSERS_PATH="${pkgs.playwright-driver.browsers}" + export PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=true + ''; + }; production = pkgs.mkShellNoCC { buildInputs = with pkgs; [ nodejs_20 diff --git a/justfile b/justfile index 5ce218b7..4d4396dc 100644 --- a/justfile +++ b/justfile @@ -16,7 +16,6 @@ reset-db: reset-all: rm -rf ~/social-protocols-data/* mkdir -p ~/social-protocols-data - touch $SCORE_EVENTS_PATH rm -f "$GB_DATABASE_PATH" just reset-db @@ -79,11 +78,7 @@ docker-build-mac: # run the app in the docker container (you must run docker-build first) docker-run: - docker run --rm -it -p 8081:8081 -e SESSION_SECRET -e INTERNAL_COMMAND_TOKEN -e HONEYPOT_SECRET --name jabble jabble:latest /bin/sh startup.sh - -# delete the docker container -docker-rm: - docker rm -f jabble + docker-compose up # exec /bin/bash in the running docker container docker-exec: @@ -93,8 +88,6 @@ download-production-data: # todo: use sqlite .backup command and download copy rm -rf $SOCIAL_PROTOCOLS_DATADIR/production/ mkdir -p $SOCIAL_PROTOCOLS_DATADIR/production/ - fly ssh sftp get /data/score-events.jsonl $SOCIAL_PROTOCOLS_DATADIR/production/score-events.jsonl - fly ssh sftp get /data/vote-events.jsonl $SOCIAL_PROTOCOLS_DATADIR/production/vote-events.jsonl fly ssh sftp get /litefs/data/sqlite.db $SOCIAL_PROTOCOLS_DATADIR/production/sqlite.db fly ssh sftp get /litefs/data/global-brain.db $SOCIAL_PROTOCOLS_DATADIR/production/global-brain.db fly ssh sftp get /litefs/data/sqlite.db $SOCIAL_PROTOCOLS_DATADIR/production/sqlite.db-wal @@ -117,3 +110,5 @@ install-node-extension-from-earthly: (cd ./GlobalBrain.jl/globalbrain-node && npm install) npm install --ignore-scripts --save './GlobalBrain.jl/globalbrain-node' +recent-sessions: + fly ssh console -C 'other/recent-sessions.sh' diff --git a/other/recent-sessions.sh b/other/recent-sessions.sh new file mode 100644 index 00000000..ca14185a --- /dev/null +++ b/other/recent-sessions.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +sqlite3 -column $APP_DATABASE_PATH $'select username, datetime(session.createdAt/1000, \'unixepoch\') as login_time_utc from session join user on userId = user.id order by session.createdAt desc limit 10;' diff --git a/package-lock.json b/package-lock.json index 72126f10..3dae1072 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,6 @@ "@types/cli-progress": "^3.11.5", "address": "^2.0.1", "autoprefixer": "^10.4.16", - "aws-sdk": "^2.1620.0", "bcryptjs": "^2.4.3", "better-sqlite3": "^8.6.0", "bloom-filters": "^3.0.1", @@ -6745,27 +6744,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/aws-sdk": { - "version": "2.1621.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1621.0.tgz", - "integrity": "sha512-v6rxF1u0GpQG1Y9Wul9iaqulSV2uEnp0kHKd6/lZcvEgTYhtJ8N0hOLfqRSWHjH5PaIa46hR9zSAp51r8DJ/OA==", - "hasInstallScript": true, - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/axe-core": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz", @@ -7046,16 +7024,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -9838,14 +9806,6 @@ "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", "dev": true }, - "node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/execa": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", @@ -11766,7 +11726,8 @@ "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true }, "node_modules/isbot": { "version": "3.8.0", @@ -11884,14 +11845,6 @@ "jiti": "bin/jiti.js" } }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/js-beautify": { "version": "1.15.1", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.15.1.tgz", @@ -16061,15 +16014,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -17054,11 +16998,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" - }, "node_modules/saxes": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", @@ -19239,15 +19178,6 @@ "punycode": "^2.1.0" } }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -19258,11 +19188,6 @@ "requires-port": "^1.0.0" } }, - "node_modules/url/node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" - }, "node_modules/use-callback-ref": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.2.tgz", @@ -19329,14 +19254,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", - "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/uvu": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/uvu/-/uvu-0.5.6.tgz", @@ -20883,26 +20800,6 @@ "node": ">=12" } }, - "node_modules/xml2js": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", - "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "engines": { - "node": ">=4.0" - } - }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/package.json b/package.json index 8dc218f4..c0f91f1c 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,6 @@ "@types/cli-progress": "^3.11.5", "address": "^2.0.1", "autoprefixer": "^10.4.16", - "aws-sdk": "^2.1620.0", "bcryptjs": "^2.4.3", "better-sqlite3": "^8.6.0", "bloom-filters": "^3.0.1", diff --git a/playwright.config.ts b/playwright.config.ts index 28b3d59e..dde87879 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,41 +1,77 @@ import { defineConfig, devices } from '@playwright/test' -import 'dotenv/config' -const PORT = process.env.PORT || '3000' +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); +/** + * See https://playwright.dev/docs/test-configuration. + */ export default defineConfig({ - testDir: './tests/e2e', - timeout: 15 * 1000, - expect: { - timeout: 5 * 1000, - }, + testDir: './e2e', + /* Run tests in files in parallel */ fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, + /* Retry on CI only */ retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { - baseURL: `http://localhost:${PORT}/`, + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', }, + /* Configure projects for major browsers */ projects: [ { name: 'chromium', - use: { - ...devices['Desktop Chrome'], - }, + use: { ...devices['Desktop Chrome'] }, }, + + // { + // name: 'firefox', + // use: { ...devices['Desktop Firefox'] }, + // }, + // + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + // }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, ], - webServer: { - command: process.env.CI ? 'npm run start:mocks' : 'npm run dev', - port: Number(PORT), - reuseExistingServer: !process.env.CI, - stdout: 'pipe', - stderr: 'pipe', - env: { - PORT, - }, - }, + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, }) diff --git a/startup.sh b/startup.sh index d9ad0159..99fc581f 100755 --- a/startup.sh +++ b/startup.sh @@ -2,6 +2,10 @@ # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail set -Eeuo pipefail +echo create a backup before migrating... +sqlite3 /litefs/data/sqlite.db '.backup /data/backup.db' +ls -lh /data/backup.db + npm run migrate npm start