no more redis

.env.development

@@ -12,23 +12,16 @@ POSTGRES_DB=db_dev
 POSTGRES_HOST=localhost
 POSTGRES_PORT=3999
 
-# For prisma
+# For prisma and pg-boss
 DATABASE_URL="postgresql://user:password@localhost:3999/db_dev"
 
-# Redis
-REDIS_HOST=localhost
-REDIS_PORT=8999
-REDIS_USERNAME=""
-REDIS_PASSWORD=password
-REDIS_TLS=false
-
 # Next-auth
 NEXTAUTH_URL=http://localhost:3000
-NEXTAUTH_SECRET="wFq419sz6XDoZhvqJybJTcettkEKxJZD6I4KRLwgy48="    #gitleaks:allow
+NEXTAUTH_SECRET="wFq419sz6XDoZhvqJybJTcettkEKxJZD6I4KRLwgy48="
 
 # Beeminder
 NEXT_PUBLIC_BEEMINDER_REDIRECT_URI=http://localhost:3000/api/beeminder/auth-callback
-NEXT_PUBLIC_BEEMINDER_CLIENT_ID=6a2dv586au2n75iq86be7u3k    #gitleaks:allow
+NEXT_PUBLIC_BEEMINDER_CLIENT_ID=6a2dv586au2n75iq86be7u3k
 # BEEMINDER_CLIENT_SECRET=...
 
 # For the frontend

.env.production

@@ -5,7 +5,7 @@ NEXTAUTH_URL=https://windofchange.me
 
 # Beeminder
 NEXT_PUBLIC_BEEMINDER_REDIRECT_URI=https://windofchange.me/api/beeminder/auth-callback
-NEXT_PUBLIC_BEEMINDER_CLIENT_ID=ciknb8fbzuh7q87zg3515cno0    #gitleaks:allow
+NEXT_PUBLIC_BEEMINDER_CLIENT_ID=ciknb8fbzuh7q87zg3515cno0
 
 # For the frontend
 NEXT_PUBLIC_APP_URL=https://windofchange.me

README.md

@@ -121,7 +121,8 @@ As of Jun 2023, the tests are flaky. You might be getting `waiting for locator('
 ## Connecting to local database
 
 ```bash
-dotenv -e .env.development -- npx prisma studio
+dotenv -e .env.development -- npx prisma studio    # GUI
+dotenv -e .env.development -- psql                 # psql
 ```
 
 ## Upgrading dependencies

docker-compose.yml

@@ -12,16 +12,5 @@ services:
       - "pg:/var/lib/postgresql/data"
     restart: "always"
 
-  redis:
-    image: "redis"
-    container_name: "woc-redis"
-    command: "redis-server --save 10 1 --loglevel warning --requirepass password"
-    ports:
-      - "8999:6379"
-    volumes:
-      - "redis:/data"
-    restart: "always"
-
 volumes:
   pg: null
-  redis: null

docs/secrets.md

@@ -41,4 +41,6 @@ sops <file name>
 
 Secrets can be detected across commits with `gitleaks detect -v`.
 
+Don't use `gitleaks:allow` in .env files, since dotenv doesn't strip the comments for whatever reason.
+
 Secrets can be removed with `bfg --replace-text <(echo "...secret...")`. Before doing this operation, you need to make a commit that also actually removes the secret — `bfg` doesn't touch the last commit.

lib/job-queue.ts

@@ -1,53 +1,48 @@
-import { Worker, Queue, Job, RedisOptions } from 'bullmq'
+import PgBoss from 'pg-boss'
 import { beeminderSyncCard, BeeminderSyncCardPayload } from './jobs/beeminder-sync-card'
+import { error } from 'console'
 
-const redisConnection: RedisOptions = {
-  host: process.env.REDIS_HOST!,
-  port: parseInt(process.env.REDIS_PORT!),
-  username: process.env.REDIS_USERNAME!,
-  password: process.env.REDIS_PASSWORD!,
-  ...(process.env.REDIS_TLS! === 'true' ? { tls: {} } : {}),
+type Job = {
+  tag: 'beeminder-sync-card'
+  payload: BeeminderSyncCardPayload
 }
 
-const jobQueue = new Queue('jobs', {
-  connection: redisConnection,
-})
+const boss = new PgBoss(process.env.DATABASE_URL!)
 
-export async function addJob(
-  job: 'beeminder-sync-card',
-  payload: BeeminderSyncCardPayload
-): Promise<Job>
-export async function addJob(job: string, payload: any) {
-  return await jobQueue.add(job, payload)
+export async function addJob<T extends Job['tag']>(
+  tag: T,
+  payload: Extract<Job, { tag: T }>['payload']
+): Promise<void> {
+  console.log(`Queueing job ${tag}`, payload)
+  // Without .start, it doesn't work for whatever reason
+  await boss.start()
+  const id = await boss.send('jobs', { tag, payload } satisfies Job)
+  if (id === null) {
+    throw new Error('Failed to queue job')
+  } else {
+    console.log(`Queued job ${tag} with id ${id}`)
+  }
 }
 
 export async function startJobQueueProcessing() {
-  const worker = new Worker(
-    'jobs',
-    async (job) => {
-      switch (job.name) {
+  boss.on('error', (err) => {
+    console.error(`pg-boss error: ${err.message}`)
+  })
+  await boss.start()
+  await boss.work<Job>('jobs', async (job) => {
+    console.log(`Processing job ${job.id}`, job.data)
+    try {
+      switch (job.data.tag) {
         case 'beeminder-sync-card': {
-          await beeminderSyncCard(job.data)
+          await beeminderSyncCard(job.data.payload)
           break
         }
         default: {
-          console.log(`Unknown job ${job.name}`)
+          throw new Error(`Unknown job type`)
         }
       }
-    },
-    {
-      connection: redisConnection,
+    } catch (err) {
+      console.error(`Error processing job ${job.id}`, job.data, err)
     }
-  )
-  worker.on('error', (err) => {
-    console.error(`job-queue worker failed: ${err.message}`)
-  })
-  worker.on('failed', (job, err) => {
-    console.error(`Job ${job?.name || '<undefined>'} failed: ${err.message}`)
   })
-  // Wait for the queue to actually get connected
-  await jobQueue.waitUntilReady()
-  await worker.waitUntilReady()
-  console.log('job-queue: connected to Redis and ready')
-  return
 }

lib/jobs/beeminder-sync-card.ts

@@ -24,12 +24,15 @@ export async function beeminderSyncCard(payload: BeeminderSyncCardPayload) {
   if (!beeminderAccessToken) throw new Error(`No Beeminder access token for user ${card.owner.id}`)
   if (!beeminderGoal) throw new Error(`No Beeminder goal for card ${card.id}`)
   await axios
-    .post(`https://www.beeminder.com/api/v1/users/${beeminderUsername}/goals/${beeminderGoal}/datapoints.json`, {
-      access_token: beeminderAccessToken,
-      value: payload.commentCount,
-      timestamp: Math.floor(payload.timestamp / 1000),
-      comment: 'Auto-posted by WOC',
-    })
+    .post(
+      `https://www.beeminder.com/api/v1/users/${beeminderUsername}/goals/${beeminderGoal}/datapoints.json`,
+      {
+        access_token: beeminderAccessToken,
+        value: payload.commentCount,
+        timestamp: Math.floor(payload.timestamp / 1000),
+        comment: 'Auto-posted by WOC',
+      }
+    )
     .catch((error) => {
       if (error.response) {
         throw new Error(