feat: add ucanto aggregation api service

.env.tpl

@@ -0,0 +1,12 @@
+# These variables are only available in your SST code.
+
+# uncomment to try out deploying the api under a custom domain.
+# the value should match a hosted zone configured in route53 that your aws account has access to.
+# HOSTED_ZONE=spade-proxy.web3.storage
+
+# uncomment to set SENTRY_DSN
+# SENTRY_DSN = ''
+
+SPADE_PROXY_DID = ''
+
+UCAN_LOG_URL = ''

.gitignore

@@ -12,6 +12,7 @@ node_modules
 # misc
 .DS_Store
 .vscode
+cdk.context.json
 
 # local env files
 .env*.local

README.md

@@ -2,27 +2,117 @@
 
 > A proxy API which will provide the UCAN server for receiving `aggregate/*` invocations and other resources for Spade to scrape - a way to retrieve the list of aggregates ready for a deal.
 
-## Commands
+## Getting Started
 
-### `npm run dev`
+The repo contains the infra deployment code and the api implementation.
 
-Starts the Live Lambda Development environment.
+To work on this codebase **you need**:
 
-### `npm run build`
+- Node.js >= 18 (prod env is node 18)
+- Install the deps with `pnpm i`
 
-Build your app and synthesize your stacks.
+You can then run the tests locally with `pnpm test`. 
 
-### `npm run deploy [stack]`
+To try out a change submit a PR and you'll get temporary infra rolled out for you automatically at `https://<pr#>.spade-proxy.web3.storage`.
 
-Deploy all your stacks to AWS. Or optionally deploy, a specific stack.
+[`sst`](https://sst.dev) is the framework we use to define what to deploy. Read the docs! https://sst.dev
 
-### `npm run remove [stack]`
+## Deployment 
 
-Remove all your stacks and all of their resources from AWS. Or optionally removes, a specific stack.
+Deployments are managed by [seed.run]. 
 
-## Documentation
+The `main` branch is deployed to https://staging.spade-proxy.web3.storage and staging builds are promoted to prod manually via the UI at https://console.seed.run
 
-Learn more about the SST.
+### Local dev
 
-- [Docs](https://docs.sst.dev/)
-- [sst](https://docs.sst.dev/packages/sst)
+You can use `sst` to create a custom dev deployment on aws, with a local dev console for debugging.
+
+To do that **you need**
+
+- An AWS account with the AWS CLI configured locally
+- Copy `.env.tpl` to `.env.local`
+
+Then run `npm start` to deploy dev services to your aws account and start dev console
+
+```console
+pnpm run start
+```
+
+See: https://docs.sst.dev for more info on how things get deployed.
+
+## Package Tests
+
+To run per-package tests, first install Docker Desktop (https://www.docker.com/) and ensure it is running.
+
+Next, ensure the `AWS_REGION`, `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables are set in your terminal. They do 
+not need to be set to real values - the following works in `bash`-like shells:
+
+```
+export AWS_REGION='us-west-2'; export AWS_ACCESS_KEY_ID='NOSUCH'; export AWS_SECRET_ACCESS_KEY='NOSUCH'
+```
+
+Finally, to run the tests for all packages, run:
+
+```
+pnpm test
+```
+
+Or to run the tests for a single package, run:
+
+```
+pnpm --filter <package-name> test
+```
+
+### Environment Variables
+
+Ensure the following variables are set in the env when deploying
+
+#### `HOSTED_ZONE`
+
+The root domain to deploy the API to. e.g `spade-proxy.web3.storage`. The value should match a hosted zone configured in route53 that your aws account has access to.
+
+### Secrets
+
+Set production secrets in aws SSM via [`sst secrets`](https://docs.sst.dev/config#sst-secrets). The region must be set to the one you deploy that stage to
+
+```sh
+# set `PRIVATE_KEY` for prod
+$ npx sst secrets set --region us-west-2 --stage prod PRIVATE_KEY "MgCblCY...="
+```
+
+To set a fallback value for `staging` or an ephmeral PR build use [`sst secrets set-fallback`](https://docs.sst.dev/config#fallback-values)
+
+```sh
+# set `PRIVATE_KEY` for any stage in us-east-2
+$ npx sst secrets set --fallback --region us-east-2 PRIVATE_KEY "MgCZG7...="
+```
+
+**note** The fallback value can only be inherited by stages deployed in the same AWS account and region.
+
+Confirm the secret value using [`sst secrets list`](https://docs.sst.dev/config#sst-secrets)
+
+```sh
+$ npx sst secrets list --region us-east-2
+PRIVATE_KEY MgCZG7...= (fallback)
+
+$ npx sst secrets list --region us-west-2 --stage prod
+PRIVATE_KEY M...=
+```
+
+#### `PRIVATE_KEY`
+
+The [`multibase`](https://github.com/multiformats/multibase) encoded ED25519 keypair used as the signing key for the upload-api.
+
+Generated by [@ucanto/principal `EdSigner`](https://github.com/web3-storage/ucanto) via [`ucan-key`](https://www.npmjs.com/package/ucan-key)
+
+_Example:_ `MgCZG7EvaA...1pX9as=`
+
+#### `UCAN_LOG_BASIC_AUTH`
+
+The HTTP Basic auth token for the UCAN Invocation entrypoint, where UCAN invocations can be stored and proxied to the UCAN Stream.
+
+_Example:_ `MgCZG7EvaA...1pX9as=`
+
+## License
+
+Dual-licensed under [MIT + Apache 2.0](license.md)

package.json

@@ -12,7 +12,8 @@
     "deploy": "sst deploy",
     "remove": "sst remove",
     "console": "sst console",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf node_modules pnpm-lock.yml packages/*/{pnpm-lock.yml,.next,out,coverage,.nyc_output,worker,dist,node_modules}"
   },
   "devDependencies": {
     "@sentry/serverless": "^7.52.1",
@@ -22,8 +23,9 @@
     "aws-cdk-lib": "2.72.1",
     "constructs": "10.1.156",
     "git-rev-sync": "^3.0.2",
+    "ts-node": "^10.9.1",
     "typescript": "^5.0.4",
-    "@tsconfig/node16": "^1.0.3"
+    "@tsconfig/node18": "^2.0.1"
   },
   "workspaces": [
     "packages/*"

packages/core/buckets/offer-store.ts

@@ -0,0 +1,47 @@
+import {
+  S3Client,
+  ServiceInputTypes,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3'
+import pRetry from 'p-retry'
+
+import * as AggregateAPI from '@web3-storage/aggregate-api'
+
+export function createOfferStore(region: string, bucketName: string, options?: ServiceInputTypes) {
+  const s3 = new S3Client({
+    region,
+    ...(options || {}),
+  })
+  return useOfferStore(s3, bucketName)
+}
+
+export function useOfferStore(s3client: S3Client, bucketName: string) {
+  return {
+    put: async (commitmentProof: string, offers: AggregateAPI.Offer[]) => {
+      const putCmd = new PutObjectCommand({
+        Bucket: bucketName,
+        ContentType: 'application/json',
+        Key: `${getNextUtcDateName()} ${commitmentProof}`,
+        Body: JSON.stringify({
+          commitmentProof,
+          offers
+        })
+      })
+
+      await pRetry(() => s3client.send(putCmd))
+    }
+  } as AggregateAPI.OfferStore
+}
+
+export function getNextUtcDateName() {
+  const cDate = new Date()
+
+  // normalize date to multiple of 15 minutes
+  const currentMinute = cDate.getUTCMinutes()
+  const factor = Math.floor(currentMinute / 15) + 1
+  const additionalTime = ((factor * 15) - currentMinute) * 60000
+
+  const nDate = new Date(cDate.getTime() + additionalTime)
+
+  return `${nDate.getUTCFullYear()}-${nDate.getUTCMonth()}-${nDate.getUTCDay()} ${nDate.getUTCHours()}:${nDate.getUTCMinutes()}:00`
+}

packages/core/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@spade-proxy/core",
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "test": "ava --node-arguments='--experimental-fetch' --verbose --timeout=60s 'test/**.test.js'",
+    "typecheck": "tsc -noEmit"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.338.0",
+    "@ipld/dag-ucan": "3.3.2",
+    "@ucanto/interface": "8.0.0",
+    "@ucanto/principal": "^8.0.0",
+    "@web3-storage/capabilities": "file:../../../w3up/packages/capabilities",
+    "@web3-storage/aggregate-api": "file:../../../w3up/packages/aggregate-api",
+    "@web3-storage/aggregate-client": "file:../../../w3up/packages/aggregate-client",
+    "node-fetch": "^3.3.1",
+    "p-retry": "^5.1.2"
+  },
+  "devDependencies": {
+    "@types/node": "^18.16.3",
+    "@web-std/blob": "^3.0.4",
+    "ava": "^4.3.3",
+    "nanoid": "^4.0.0",
+    "sst": "^2.8.3",
+    "testcontainers": "^8.13.0",
+    "typescript": "^5.0.4"
+  },
+  "ava": {
+    "extensions": {
+      "ts": "module",
+      "js": true
+    },
+    "nodeArguments": [
+      "--loader=ts-node/esm"
+    ]
+  }
+}

packages/core/service.ts

@@ -0,0 +1,31 @@
+import * as ed25519 from '@ucanto/principal/ed25519'
+import * as DID from '@ipld/dag-ucan/did'
+import { Signer } from '@ucanto/interface'
+
+import * as AggregateAPI from '@web3-storage/aggregate-api'
+
+export const createUcantoServer = (servicePrincipal: Signer, context: AggregateAPI.ServiceContext, errorReporter: AggregateAPI.ErrorReporter) =>
+AggregateAPI.createServer({
+  ...context,
+  id: servicePrincipal,
+  errorReporter
+})
+
+/**
+ * Given a config, return a ucanto Signer object representing the service
+ */
+export function getServiceSigner(config: ServiceSignerCtx) {
+  const signer = ed25519.parse(config.PRIVATE_KEY)
+  if (config.SPADE_PROXY_DID) {
+    const did = DID.parse(config.SPADE_PROXY_DID).did()
+    return signer.withDID(did)
+  }
+  return signer
+}
+
+export type ServiceSignerCtx = {
+  // multiformats private key of primary signing key
+  PRIVATE_KEY: string
+  // public DID for the upload service (did:key:... derived from PRIVATE_KEY if not set)
+  SPADE_PROXY_DID?: string
+}

packages/core/tables/aggregate-store.ts

@@ -0,0 +1,13 @@
+import * as AggregateAPI from '@web3-storage/aggregate-api'
+
+export function createAggregateStore() {
+  return useAggregateStore()
+}
+
+export function useAggregateStore() {
+  return {
+    get: async (commitmentProof: string) => {
+      throw new Error('not yet implemented')
+    }
+  } as AggregateAPI.AggregateStore
+}

packages/core/tables/arranged-offer-store.ts

@@ -0,0 +1,13 @@
+import * as AggregateAPI from '@web3-storage/aggregate-api'
+
+export function createArrangedOfferStore() {
+  return useArrangedOfferStore()
+}
+
+export function useArrangedOfferStore() {
+  return {
+    get: async (commitmentProof: string) => {
+      throw new Error('not yet implemented')
+    }
+  } as AggregateAPI.ArrangedOfferStore
+}

packages/core/test/helpers/context.js

@@ -0,0 +1,11 @@
+import anyTest from 'ava'
+
+/**
+ * @typedef {object} S3Context
+ * @property {import('@aws-sdk/client-s3').S3Client} s3
+ * 
+ * @typedef {import("ava").TestFn<S3Context>} Test
+ */
+
+// eslint-disable-next-line unicorn/prefer-export-from
+export const test = /** @type {Test} */ (anyTest)

packages/core/test/helpers/resources.js

@@ -0,0 +1,44 @@
+import { customAlphabet } from 'nanoid'
+import { GenericContainer as Container } from 'testcontainers'
+import { S3Client, CreateBucketCommand } from '@aws-sdk/client-s3'
+
+/**
+ * @param {object} [opts]
+ * @param {number} [opts.port]
+ * @param {string} [opts.region]
+ */
+export async function createS3(opts = {}) {
+  const region = opts.region || 'us-west-2'
+  const port = opts.port || 9000
+
+  const minio = await new Container('quay.io/minio/minio')
+    .withCmd(['server', '/data'])
+    .withExposedPorts(port)
+    .start()
+
+  const clientOpts = {
+    endpoint: `http://${minio.getHost()}:${minio.getMappedPort(port)}`,
+    forcePathStyle: true,
+    region,
+    credentials: {
+      accessKeyId: 'minioadmin',
+      secretAccessKey: 'minioadmin',
+    },
+  }
+
+  return {
+    client: new S3Client(clientOpts),
+    clientOpts,
+  }
+}
+
+/**
+ * @param {S3Client} s3
+ */
+export async function createBucket(s3) {
+  const id = customAlphabet('1234567890abcdefghijklmnopqrstuvwxyz', 10)
+  const Bucket = id()
+  await s3.send(new CreateBucketCommand({ Bucket }))
+  return Bucket
+}
+