From d59ec883ace9c3f084772f9520b6df81cc13b7af Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 5 Nov 2024 10:38:22 +0000 Subject: [PATCH 1/4] feat: remove store protocol The store protocol in not compatible with decentralized storage as there are no "bucket events" on remote storage nodes. --- packages/upload-api/src/admin.js | 5 - .../upload-api/src/admin/store/inspect.js | 43 - packages/upload-api/src/blob/add.js | 18 +- packages/upload-api/src/lib.js | 2 - packages/upload-api/src/store.js | 17 - packages/upload-api/src/store/add.js | 85 -- packages/upload-api/src/store/get.js | 23 - packages/upload-api/src/store/lib.js | 29 - packages/upload-api/src/store/list.js | 15 - packages/upload-api/src/store/remove.js | 22 - packages/upload-api/src/types.ts | 92 +- .../test/handlers/admin/store/inspect.js | 66 -- .../test/handlers/admin/store/inspect.spec.js | 3 - .../test/handlers/rate-limit/add.js | 8 +- packages/upload-api/test/handlers/store.js | 874 ------------------ .../upload-api/test/handlers/store.spec.js | 4 - packages/upload-api/test/handlers/usage.js | 28 +- packages/upload-api/test/helpers/context.js | 3 - packages/upload-api/test/lib.js | 6 - .../test/storage/car-store-bucket.js | 160 ---- packages/upload-api/test/storage/index.js | 8 +- .../upload-api/test/storage/store-table.js | 134 --- .../upload-api/test/storage/usage-storage.js | 16 +- packages/upload-client/src/index.js | 3 +- packages/upload-client/src/store.js | 303 ------ packages/upload-client/src/types.ts | 31 - packages/upload-client/test/helpers/mocks.js | 7 - packages/upload-client/test/store.test.js | 786 ---------------- packages/w3up-client/src/capability/store.js | 71 -- packages/w3up-client/src/client.js | 22 +- packages/w3up-client/src/types.ts | 6 - .../w3up-client/test/capability/store.test.js | 147 --- packages/w3up-client/test/space.test.js | 9 +- 33 files changed, 34 insertions(+), 3012 deletions(-) delete mode 100644 packages/upload-api/src/admin/store/inspect.js delete mode 100644 packages/upload-api/src/store.js delete mode 100644 packages/upload-api/src/store/add.js delete mode 100644 packages/upload-api/src/store/get.js delete mode 100644 packages/upload-api/src/store/lib.js delete mode 100644 packages/upload-api/src/store/list.js delete mode 100644 packages/upload-api/src/store/remove.js delete mode 100644 packages/upload-api/test/handlers/admin/store/inspect.js delete mode 100644 packages/upload-api/test/handlers/admin/store/inspect.spec.js delete mode 100644 packages/upload-api/test/handlers/store.js delete mode 100644 packages/upload-api/test/handlers/store.spec.js delete mode 100644 packages/upload-api/test/storage/car-store-bucket.js delete mode 100644 packages/upload-api/test/storage/store-table.js delete mode 100644 packages/upload-client/src/store.js delete mode 100644 packages/upload-client/test/store.test.js delete mode 100644 packages/w3up-client/src/capability/store.js delete mode 100644 packages/w3up-client/test/capability/store.test.js diff --git a/packages/upload-api/src/admin.js b/packages/upload-api/src/admin.js index d43255d00..1b1b5c27e 100644 --- a/packages/upload-api/src/admin.js +++ b/packages/upload-api/src/admin.js @@ -1,15 +1,10 @@ import * as Types from './types.js' -import * as StoreInspect from './admin/store/inspect.js' import * as UploadInspect from './admin/upload/inspect.js' /** * @param {Types.AdminServiceContext} context */ export const createService = (context) => ({ - store: { - inspect: StoreInspect.provide(context), - }, - upload: { inspect: UploadInspect.provide(context), }, diff --git a/packages/upload-api/src/admin/store/inspect.js b/packages/upload-api/src/admin/store/inspect.js deleted file mode 100644 index 19529f2c9..000000000 --- a/packages/upload-api/src/admin/store/inspect.js +++ /dev/null @@ -1,43 +0,0 @@ -import * as API from '../../types.js' -import * as Provider from '@ucanto/server' -import { Admin } from '@storacha/capabilities' - -/** - * @param {API.AdminServiceContext} context - */ -export const provide = (context) => - Provider.provide(Admin.store.inspect, (input) => inspect(input, context)) - -/** - * @param {API.Input} input - * @param {API.AdminServiceContext} context - * @returns {Promise} - */ -const inspect = async ({ capability }, context) => { - /** - * Ensure that resource is the service DID, which implies it's either - * invoked by service itself or an authorized delegate (like admin). - * In other words no user will be able to invoke this unless service - * explicitly delegated capability to them to do so. - */ - if (capability.with !== context.signer.did()) { - return { error: new UnknownProvider(capability.with) } - } - - return await context.storeTable.inspect(capability.nb.link) -} - -class UnknownProvider extends Provider.Failure { - /** - * @param {API.DID} did - */ - constructor(did) { - super() - this.did = did - this.name = /** @type {const} */ ('UnknownProvider') - } - - describe() { - return `Provider ${this.did} not found` - } -} diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index f547f0e94..5d431ef0f 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -5,10 +5,11 @@ import { ed25519 } from '@ucanto/principal' import * as Blob from '@storacha/capabilities/blob' import * as SpaceBlob from '@storacha/capabilities/space/blob' import * as HTTP from '@storacha/capabilities/http' +import * as Digest from 'multiformats/hashes/digest' import * as API from '../types.js' +import { allocate as spaceAllocate } from '../space-allocate.js' import { createConcludeInvocation } from '../ucan/conclude.js' import { AwaitError } from './lib.js' -import * as Digest from 'multiformats/hashes/digest' import { AgentMessage } from '../lib.js' /** @@ -101,21 +102,14 @@ async function allocate({ context, blob, space, cause }) { // First we check if space has storage provider associated. If it does not // we return `InsufficientStorage` error as storage capacity is considered // to be 0. - const provisioned = await context.provisionsStorage.hasStorageProvider(space) + const provisioned = await spaceAllocate( + { capability: { with: space } }, + context + ) if (provisioned.error) { return provisioned } - if (!provisioned.ok) { - return { - /** @type {API.AllocationError} */ - error: { - name: 'InsufficientStorage', - message: `${space} has no storage provider`, - }, - } - } - // 1. Create blob/allocate invocation and task const { router } = context const digest = Digest.decode(blob.digest) diff --git a/packages/upload-api/src/lib.js b/packages/upload-api/src/lib.js index 175ecc943..c4547f26b 100644 --- a/packages/upload-api/src/lib.js +++ b/packages/upload-api/src/lib.js @@ -6,7 +6,6 @@ import * as Types from './types.js' import * as Legacy from '@ucanto/transport/legacy' import * as CAR from '@ucanto/transport/car' import { create as createRevocationChecker } from './utils/revocation.js' -import { createService as createStoreService } from './store.js' import { createService as createUploadService } from './upload.js' import { createService as createConsoleService } from './console.js' import { createService as createAccessService } from './access.js' @@ -181,7 +180,6 @@ export const createService = (context) => ({ 'rate-limit': createRateLimitService(context), admin: createAdminService(context), space: createSpaceService(context), - store: createStoreService(context), subscription: createSubscriptionService(context), upload: createUploadService(context), ucan: createUcanService(context), diff --git a/packages/upload-api/src/store.js b/packages/upload-api/src/store.js deleted file mode 100644 index 146bcf09c..000000000 --- a/packages/upload-api/src/store.js +++ /dev/null @@ -1,17 +0,0 @@ -import { storeAddProvider } from './store/add.js' -import { storeGetProvider } from './store/get.js' -import { storeListProvider } from './store/list.js' -import { storeRemoveProvider } from './store/remove.js' -import * as API from './types.js' - -/** - * @param {API.StoreServiceContext} context - */ -export function createService(context) { - return { - add: storeAddProvider(context), - get: storeGetProvider(context), - list: storeListProvider(context), - remove: storeRemoveProvider(context), - } -} diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js deleted file mode 100644 index 3d57bd7d0..000000000 --- a/packages/upload-api/src/store/add.js +++ /dev/null @@ -1,85 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Store from '@storacha/capabilities/store' -import * as API from '../types.js' -import { allocate } from '../space-allocate.js' - -/** - * @param {API.StoreServiceContext} context - * @returns {API.ServiceMethod} - */ -export function storeAddProvider(context) { - const { storeTable, carStoreBucket, maxUploadSize } = context - return Server.provide(Store.add, async ({ capability, invocation }) => { - const { link, origin, size } = capability.nb - - if (size > maxUploadSize) { - return { - error: new Server.Failure( - `Maximum size exceeded: ${maxUploadSize}, split DAG into smaller shards.` - ), - } - } - - const space = /** @type {import('@ucanto/interface').DIDKey} */ ( - Server.DID.parse(capability.with).did() - ) - - const [allocated, carExists] = await Promise.all([ - allocate( - { - capability: { - with: space, - }, - }, - context - ), - carStoreBucket.has(link), - ]) - - // If failed to allocate space, fail with allocation error - if (allocated.error) { - return allocated - } - - let allocatedSize = size - const res = await storeTable.insert({ - space, - link, - size, - origin, - invocation: invocation.cid, - }) - if (res.error) { - // if the insert failed with conflict then this item has already been - // added to the space and there is no allocation change. - if (res.error.name === 'RecordKeyConflict') { - allocatedSize = 0 - } else { - return res - } - } - - if (carExists) { - return { - ok: { - status: 'done', - allocated: allocatedSize, - with: space, - link, - }, - } - } - - const { url, headers } = await carStoreBucket.createUploadUrl(link, size) - return { - ok: { - status: 'upload', - allocated: allocatedSize, - with: space, - link, - url: url.toString(), - headers, - }, - } - }) -} diff --git a/packages/upload-api/src/store/get.js b/packages/upload-api/src/store/get.js deleted file mode 100644 index 3536ca586..000000000 --- a/packages/upload-api/src/store/get.js +++ /dev/null @@ -1,23 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Store from '@storacha/capabilities/store' -import * as API from '../types.js' -import { StoreItemNotFound } from './lib.js' - -/** - * @param {API.StoreServiceContext} context - * @returns {API.ServiceMethod} - */ -export function storeGetProvider(context) { - return Server.provide(Store.get, async ({ capability }) => { - const { link } = capability.nb - if (!link) { - return Server.fail('nb.link must be set') - } - const space = Server.DID.parse(capability.with).did() - const res = await context.storeTable.get(space, link) - if (res.error && res.error.name === 'RecordNotFound') { - return Server.error(new StoreItemNotFound(space, link)) - } - return res - }) -} diff --git a/packages/upload-api/src/store/lib.js b/packages/upload-api/src/store/lib.js deleted file mode 100644 index e66063c49..000000000 --- a/packages/upload-api/src/store/lib.js +++ /dev/null @@ -1,29 +0,0 @@ -import { Failure } from '@ucanto/server' - -export class StoreItemNotFound extends Failure { - /** - * @param {import('@ucanto/interface').DID} space - * @param {import('@ucanto/interface').UnknownLink} link - */ - constructor(space, link) { - super() - this.space = space - this.link = link - } - - get name() { - return 'StoreItemNotFound' - } - - describe() { - return `${this.link} not found in ${this.space}` - } - - toJSON() { - return { - ...super.toJSON(), - space: this.space, - link: { '/': this.link.toString() }, - } - } -} diff --git a/packages/upload-api/src/store/list.js b/packages/upload-api/src/store/list.js deleted file mode 100644 index 76fe329ec..000000000 --- a/packages/upload-api/src/store/list.js +++ /dev/null @@ -1,15 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Store from '@storacha/capabilities/store' -import * as API from '../types.js' - -/** - * @param {API.StoreServiceContext} context - * @returns {API.ServiceMethod} - */ -export function storeListProvider(context) { - return Server.provide(Store.list, async ({ capability }) => { - const { cursor, size, pre } = capability.nb - const space = Server.DID.parse(capability.with).did() - return await context.storeTable.list(space, { size, cursor, pre }) - }) -} diff --git a/packages/upload-api/src/store/remove.js b/packages/upload-api/src/store/remove.js deleted file mode 100644 index 386652358..000000000 --- a/packages/upload-api/src/store/remove.js +++ /dev/null @@ -1,22 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Store from '@storacha/capabilities/store' -import * as API from '../types.js' -import { StoreItemNotFound } from './lib.js' - -/** - * @param {API.StoreServiceContext} context - * @returns {API.ServiceMethod} - */ -export function storeRemoveProvider(context) { - return Server.provide(Store.remove, async ({ capability }) => { - const { link } = capability.nb - const space = Server.DID.parse(capability.with).did() - - const res = await context.storeTable.remove(space, link) - if (res.error && res.error.name === 'RecordNotFound') { - return Server.error(new StoreItemNotFound(space, link)) - } - - return res - }) -} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index dc27b85a3..e7e060fa5 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -77,15 +77,6 @@ import { SpaceBlobGet, SpaceBlobGetSuccess, SpaceBlobGetFailure, - StoreAdd, - StoreGet, - StoreAddSuccess, - StoreRemove, - StoreRemoveSuccess, - StoreRemoveFailure, - StoreList, - StoreListSuccess, - StoreListItem, UploadAdd, UploadGet, UploadAddSuccess, @@ -129,9 +120,6 @@ import { RateLimitList, RateLimitListSuccess, RateLimitListFailure, - AdminStoreInspect, - AdminStoreInspectSuccess, - AdminStoreInspectFailure, AdminUploadInspect, AdminUploadInspectSuccess, AdminUploadInspectFailure, @@ -140,12 +128,10 @@ import { ProviderAddFailure, SpaceInfo, ProviderDID, - StoreGetFailure, + UploadGetSuccess, UploadGetFailure, ListResponse, CARLink, - StoreGetSuccess, - UploadGetSuccess, UCANConclude, UCANConcludeSuccess, UCANConcludeFailure, @@ -217,12 +203,6 @@ export type { } from './types/content-claims.js' export interface Service extends StorefrontService { - store: { - add: ServiceMethod - get: ServiceMethod - remove: ServiceMethod - list: ServiceMethod - } upload: { add: ServiceMethod get: ServiceMethod @@ -302,13 +282,6 @@ export interface Service extends StorefrontService { } admin: { - store: { - inspect: ServiceMethod< - AdminStoreInspect, - AdminStoreInspectSuccess, - AdminStoreInspectFailure - > - } upload: { inspect: ServiceMethod< AdminUploadInspect, @@ -376,12 +349,6 @@ export type BlobServiceContext = SpaceServiceContext & { registry: BlobRegistry } -export type StoreServiceContext = SpaceServiceContext & { - maxUploadSize: number - storeTable: StoreTable - carStoreBucket: CarStoreBucket -} - export type UploadServiceContext = ConsumerServiceContext & SpaceServiceContext & RevocationServiceContext & @@ -415,7 +382,6 @@ export interface CustomerServiceContext { export interface AdminServiceContext { signer: Signer uploadTable: UploadTable - storeTable: StoreTable } export interface ConsoleServiceContext {} @@ -473,7 +439,6 @@ export interface ServiceContext CustomerServiceContext, ProviderServiceContext, SpaceServiceContext, - StoreServiceContext, BlobServiceContext, ConcludeServiceContext, SubscriptionServiceContext, @@ -578,8 +543,8 @@ export interface UcantoServerTestContext grantAccess: (mail: { url: string | URL }) => Promise - carStoreBucket: CarStoreBucket & Deactivator claimsService: ClaimsClientConfig & ClaimReader & Deactivator + storageProviders: Deactivator[] } export interface ClaimReader { @@ -598,33 +563,6 @@ export interface ErrorReporter { catch: (error: HandlerExecutionError | WriteError) => void } -export interface CarStoreBucket { - has: (link: UnknownLink) => Promise - createUploadUrl: ( - link: UnknownLink, - size: number - ) => Promise<{ - url: URL - headers: { - 'x-amz-checksum-sha256': string - 'content-length': string - } & Record - }> -} - -export interface CarStoreBucketOptions { - accessKeyId?: string - secretAccessKey?: string - region?: string - bucket?: string - sessionToken?: string - expires?: number -} - -export interface CarStoreBucketService { - use(options?: CarStoreBucketOptions): Promise -} - /** * Indicates the requested record was not present in the table. */ @@ -640,28 +578,6 @@ export interface RecordKeyConflict extends Failure { name: 'RecordKeyConflict' } -export interface StoreTable { - inspect: (link: UnknownLink) => Promise> - exists: (space: DID, link: UnknownLink) => Promise> - get: ( - space: DID, - link: UnknownLink - ) => Promise> - /** Inserts an item in the table if it does not already exist. */ - insert: ( - item: StoreAddInput - ) => Promise> - /** Removes an item from the table but fails if the item does not exist. */ - remove: ( - space: DID, - link: UnknownLink - ) => Promise> - list: ( - space: DID, - options?: ListOptions - ) => Promise, Failure>> -} - export interface UploadTable { inspect: (link: UnknownLink) => Promise> exists: (space: DID, root: UnknownLink) => Promise> @@ -699,10 +615,6 @@ export type SubscriptionGetResult = Result< SubscriptionGetSuccess, SubscriptionGetFailure > -export type AdminStoreInspectResult = Result< - AdminStoreInspectSuccess, - AdminStoreInspectFailure -> export type AdminUploadInspectResult = Result< AdminUploadInspectSuccess, AdminUploadInspectFailure diff --git a/packages/upload-api/test/handlers/admin/store/inspect.js b/packages/upload-api/test/handlers/admin/store/inspect.js deleted file mode 100644 index e3120e1a8..000000000 --- a/packages/upload-api/test/handlers/admin/store/inspect.js +++ /dev/null @@ -1,66 +0,0 @@ -import * as API from '../../../types.js' -import { alice, registerSpace } from '../../../util.js' -import { createServer, connect } from '../../../../src/lib.js' - -import { delegate } from '@ucanto/core' -import * as CAR from '@ucanto/transport/car' -import { Admin, Store } from '@storacha/capabilities' - -/** - * @type {API.Tests} - */ -export const test = { - 'admin/store/inspect returns information about an uploaded CID': async ( - assert, - context - ) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - const size = data.byteLength - - const storeAdd = await Store.add - .invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { link, size }, - proofs: [proof], - }) - .execute(connection) - - assert.ok(storeAdd.out.ok) - - const service = context.service - const adminStoreInspect = await Admin.store.inspect - .invoke({ - issuer: alice, - audience: connection.id, - with: service.did(), - nb: { link }, - proofs: [ - await delegate({ - issuer: service, - audience: alice, - capabilities: [{ with: service.did(), can: 'admin/store/inspect' }], - }), - ], - }) - .execute(connection) - - assert.ok( - adminStoreInspect.out.ok, - `failed to get shard: ${adminStoreInspect.out.error?.message}` - ) - assert.equal(adminStoreInspect.out.ok?.spaces[0].did, spaceDid) - assert.equal( - typeof adminStoreInspect.out.ok?.spaces[0].insertedAt, - 'string' - ) - }, -} diff --git a/packages/upload-api/test/handlers/admin/store/inspect.spec.js b/packages/upload-api/test/handlers/admin/store/inspect.spec.js deleted file mode 100644 index 7a786c0c7..000000000 --- a/packages/upload-api/test/handlers/admin/store/inspect.spec.js +++ /dev/null @@ -1,3 +0,0 @@ -import * as Inspect from './inspect.js' -import { test } from '../../../test.js' -test({ 'admin/store/inspect': Inspect.test }) diff --git a/packages/upload-api/test/handlers/rate-limit/add.js b/packages/upload-api/test/handlers/rate-limit/add.js index 527e20696..998d28fa2 100644 --- a/packages/upload-api/test/handlers/rate-limit/add.js +++ b/packages/upload-api/test/handlers/rate-limit/add.js @@ -2,7 +2,7 @@ import * as API from '../../types.js' import { alice, bob } from '../../helpers/utils.js' import { Absentee } from '@ucanto/principal' import { delegate } from '@ucanto/core' -import { Access, RateLimit, Store } from '@storacha/capabilities' +import { Access, RateLimit, SpaceBlob } from '@storacha/capabilities' import * as CAR from '@ucanto/transport/car' import * as DidMailto from '@storacha/did-mailto' @@ -112,17 +112,17 @@ export const test = { const data = new Uint8Array([11, 22, 34, 44, 55]) const link = await CAR.codec.link(data) const size = data.byteLength - const storeResult = await Store.add + const storeResult = await SpaceBlob.add .invoke({ issuer: agent, audience: service, with: space.did(), - nb: { link, size }, + nb: { blob: { digest: link.multihash.bytes, size } }, proofs: [ await delegate({ issuer: space, audience: agent, - capabilities: [{ with: space.did(), can: 'store/add' }], + capabilities: [{ with: space.did(), can: SpaceBlob.add.can }], }), ], }) diff --git a/packages/upload-api/test/handlers/store.js b/packages/upload-api/test/handlers/store.js deleted file mode 100644 index 0360d1680..000000000 --- a/packages/upload-api/test/handlers/store.js +++ /dev/null @@ -1,874 +0,0 @@ -import { createServer, connect } from '../../src/lib.js' -import * as API from '../../src/types.js' -import * as CAR from '@ucanto/transport/car' -import { base64pad } from 'multiformats/bases/base64' -import * as Raw from 'multiformats/codecs/raw' -import { sha256 } from 'multiformats/hashes/sha2' -import * as Link from 'multiformats/link' -import * as StoreCapabilities from '@storacha/capabilities/store' -import { invoke } from '@ucanto/core' -import { alice, bob, createSpace, registerSpace } from '../util.js' -import { Absentee } from '@ucanto/principal' -import { provisionProvider } from '../helpers/utils.js' -import * as Result from '../helpers/result.js' - -/** - * @type {API.Tests} - */ -export const test = { - 'store/add returns signed url for uploading': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - const size = data.byteLength - - const invocation = StoreCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { link, size }, - proofs: [proof], - }) - - // invoke a store/add with proof - const storeAdd = await invocation.execute(connection) - - if (!storeAdd.out.ok) { - throw new Error('invocation failed', { cause: storeAdd }) - } - if (storeAdd.out.ok.status !== 'upload') { - throw new Error(`unexpected status: ${storeAdd.out.ok.status}`) - } - - assert.equal(storeAdd.out.ok.with, spaceDid) - assert.deepEqual(storeAdd.out.ok.link.toString(), link.toString()) - - assert.equal(storeAdd.out.ok.headers?.['content-length'], String(size)) - assert.deepEqual( - storeAdd.out.ok.headers?.['x-amz-checksum-sha256'], - base64pad.baseEncode(link.multihash.digest) - ) - - const url = storeAdd.out.ok.url && new URL(storeAdd.out.ok.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const signedHeaders = url.searchParams.get('X-Amz-SignedHeaders') - - assert.equal( - signedHeaders, - 'content-length;host;x-amz-checksum-sha256', - 'content-length and checksum must be part of the signature' - ) - - // May have bucket name at start of path - assert.equal(url.pathname.endsWith(`/${link}/${link}.car`), true) - - const goodPut = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: storeAdd.out.ok.headers, - }) - - assert.equal(goodPut.status, 200, await goodPut.text()) - - const item = Result.unwrap(await context.storeTable.get(spaceDid, link)) - - assert.deepEqual( - { - link: item.link.toString(), - size: item.size, - }, - { - link: link.toString(), - size: data.byteLength, - } - ) - - assert.equal( - Date.now() - new Date(item?.insertedAt).getTime() < 60_000, - true - ) - - const { spaces } = Result.unwrap(await context.storeTable.inspect(link)) - assert.equal(spaces.length, 1) - assert.equal(spaces[0].did, spaceDid) - }, - - 'store/add should allow add the same content to be stored in multiple spaces': - async (assert, context) => { - const { proof: aliceProof, spaceDid: aliceSpaceDid } = - await registerSpace(alice, context) - const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( - bob, - context, - 'bob' - ) - - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - const size = data.byteLength - - const aliceStoreAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { link, size }, - proofs: [aliceProof], - }) - .execute(connection) - - assert.ok( - aliceStoreAdd.out.ok, - `Alice failed to store ${link.toString()}` - ) - - const bobStoreAdd = await StoreCapabilities.add - .invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { link, size }, - proofs: [bobProof], - }) - .execute(connection) - - assert.ok(bobStoreAdd.out.ok, `Bob failed to store ${link.toString()}`) - - const { spaces } = Result.unwrap(await context.storeTable.inspect(link)) - assert.equal(spaces.length, 2) - const spaceDids = spaces.map((space) => space.did) - assert.ok(spaceDids.includes(aliceSpaceDid)) - assert.ok(spaceDids.includes(bobSpaceDid)) - }, - - 'store/add should create a presigned url that can only PUT a payload with the right length': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) - const link = await CAR.codec.link(data) - const size = data.byteLength - - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { link, size }, - proofs: [proof], - }) - .execute(connection) - - if (!storeAdd.out.ok) { - throw new Error('invocation failed', { cause: storeAdd }) - } - if (storeAdd.out.ok.status !== 'upload') { - throw new Error(`unexpected status: ${storeAdd.out.ok.status}`) - } - - const url = new URL(storeAdd.out.ok.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - - const contentLengthFailSignature = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: longer, - headers: { - ...storeAdd.out.ok.headers, - 'content-length': longer.byteLength.toString(10), - }, - }) - - assert.equal( - contentLengthFailSignature.status >= 400, - true, - 'should fail to upload as content-length differs from that used to sign the url' - ) - }, - - 'store/add should create a presigned url that can only PUT the exact bytes we signed for': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const other = new Uint8Array([10, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - const size = data.byteLength - - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { link, size }, - proofs: [proof], - }) - .execute(connection) - - if (!storeAdd.out.ok) { - throw new Error('invocation failed', { cause: storeAdd }) - } - if (storeAdd.out.ok.status !== 'upload') { - throw new Error(`unexpected status: ${storeAdd.out.ok.status}`) - } - - const url = new URL(storeAdd.out.ok.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - - const failChecksum = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: other, - headers: storeAdd.out.ok.headers, - }) - - assert.equal( - failChecksum.status, - 400, - 'should fail to upload any other data.' - ) - }, - - 'store/add returns done if already uploaded': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - - const { url, headers } = await context.carStoreBucket.createUploadUrl( - link, - data.length - ) - - // simulate an already stored CAR - const put = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers, - }) - assert.equal(put.ok, true, 'should be able to upload to presigned url') - - const storeAddInvocation = StoreCapabilities.add.invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link, size: data.byteLength }, - proofs: [proof], - }) - - const storeAdd = await storeAddInvocation.execute(connection) - - if (!storeAdd.out.ok) { - throw new Error('invocation failed', { cause: storeAdd }) - } - - assert.equal(storeAdd.out.ok.status, 'done') - assert.equal(storeAdd.out.ok.allocated, 5) - assert.equal(storeAdd.out.ok.with, spaceDid) - assert.deepEqual(storeAdd.out.ok.link.toString(), link.toString()) - // @ts-expect-error making sure it's not an upload status - assert.equal(storeAdd.out.ok.url == null, true) - - const item = Result.unwrap(await context.storeTable.get(spaceDid, link)) - - assert.deepEqual( - { - link: item.link.toString(), - size: item.size, - }, - { - link: link.toString(), - size: data.byteLength, - } - ) - - assert.equal( - Date.now() - new Date(item.insertedAt).getTime() < 60_000, - true - ) - }, - - 'store/add returns allocated: 0 if already added to space': async ( - assert, - context - ) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - - const { url, headers } = await context.carStoreBucket.createUploadUrl( - link, - data.length - ) - - // simulate an already stored CAR - const put = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers, - }) - assert.equal(put.ok, true, 'should be able to upload to presigned url') - - const inv0 = StoreCapabilities.add.invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link, size: data.byteLength }, - nonce: '0', - proofs: [proof], - }) - - const r0 = await inv0.execute(connection) - - assert.equal(r0.out.ok?.status, 'done') - assert.equal(r0.out.ok?.allocated, 5) - assert.equal(r0.out.ok?.with, spaceDid) - - const inv1 = StoreCapabilities.add.invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link, size: data.byteLength }, - nonce: '1', - proofs: [proof], - }) - - const r1 = await inv1.execute(connection) - - assert.equal(r1.out.ok?.status, 'done') - assert.equal(r1.out.ok?.allocated, 0) - assert.equal(r1.out.ok?.with, spaceDid) - }, - - 'store/add disallowed if invocation fails access verification': async ( - assert, - context - ) => { - const { proof, space, spaceDid } = await createSpace(alice) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - - // invoke a store/add with proof - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link, size: data.byteLength }, - proofs: [proof], - }) - .execute(connection) - - assert.ok(storeAdd.out.error) - assert.equal(storeAdd.out.error?.message.includes('no storage'), true) - - // Register space and retry - const account = Absentee.from({ - id: 'did:mailto:test.storacha.network:alice', - }) - const providerAdd = await provisionProvider({ - service: /** @type {API.Signer>} */ (context.signer), - agent: alice, - space, - account, - connection, - }) - assert.ok(providerAdd.out.ok) - - const retryStoreAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link, size: data.byteLength }, - proofs: [proof], - nonce: 'retry', - }) - .execute(connection) - - assert.equal(retryStoreAdd.out.error, undefined) - }, - - 'store/add fails when size too large to PUT': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - const size = context.maxUploadSize + 1 - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link, size }, - proofs: [proof], - }) - .execute(connection) - - assert.ok(storeAdd.out.error) - assert.equal( - storeAdd.out.error?.message.startsWith('Maximum size exceeded:'), - true - ) - }, - - 'store/add fails with non-car link': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - /** @type {API.Link} */ - const link = Link.create(Raw.code, await sha256.digest(data)) - const size = context.maxUploadSize + 1 - - // Throws because invocation builder expects CAR link - try { - StoreCapabilities.add.invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { - link, - size, - }, - proofs: [proof], - }) - assert.ok(false, 'should have throw exception') - } catch (error) { - assert.ok(String(error).match(/0x202 codec/)) - } - - // Going around client validation will still fail because server handler - // expects CAR link. - const invocation = await invoke({ - issuer: alice, - audience: connection.id, - capability: { - can: 'store/add', - with: spaceDid, - nb: { - link, - size, - }, - }, - - proofs: [proof], - }).delegate() - - const [storeAdd] = await connection.execute(invocation) - assert.ok(storeAdd.out.error) - assert.ok(storeAdd.out.error?.message.match('0x202 codec')) - }, - - 'store/remove fails for non existent link': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = new Uint8Array([11, 22, 34, 44, 55]) - const link = await CAR.codec.link(data) - - const storeRemove = await StoreCapabilities.remove - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link }, - proofs: [proof], - }) - .execute(connection) - - assert.equal(storeRemove.out.error?.name, 'StoreItemNotFound') - }, - - 'store/list does not fail for empty list': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const storeList = await StoreCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: {}, - }) - .execute(connection) - - assert.deepEqual(storeList.out.ok, { results: [], size: 0 }) - }, - - 'store/list returns items previously stored by the user': async ( - assert, - context - ) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = [ - new Uint8Array([11, 22, 34, 44, 55]), - new Uint8Array([22, 34, 44, 55, 66]), - ] - const links = [] - for (const datum of data) { - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link: await CAR.codec.link(datum), size: datum.byteLength }, - proofs: [proof], - }) - .execute(connection) - - if (storeAdd.out.error) { - throw new Error('invocation failed', { cause: storeAdd }) - } - - assert.equal(storeAdd.out.ok.status, 'upload') - links.push(storeAdd.out.ok.link) - } - - const storeList = await StoreCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: {}, - }) - .execute(connection) - - if (storeList.out.error) { - throw new Error('invocation failed', { cause: storeList }) - } - - assert.equal(storeList.out.ok.size, links.length) - - // list order last-in-first-out - assert.deepEqual( - storeList.out.ok.results.map(({ link }) => ({ link, size: 5 })), - links.reverse().map((link) => ({ link, size: 5 })) - ) - }, - - 'store/list can be paginated with custom size': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = [ - new Uint8Array([11, 22, 34, 44, 55]), - new Uint8Array([22, 34, 44, 55, 66]), - ] - const links = [] - - for (const datum of data) { - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link: await CAR.codec.link(datum), size: datum.byteLength }, - proofs: [proof], - }) - .execute(connection) - if (storeAdd.out.error) { - throw new Error('invocation failed', { cause: storeAdd }) - } - - links.push(storeAdd.out.ok.link) - } - - // Get list with page size 1 (two pages) - const size = 1 - const listPages = [] - /** @type {string} */ - let cursor = '' - - do { - const storeList = await StoreCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - ...(cursor ? { cursor } : {}), - }, - }) - .execute(connection) - - if (storeList.out.error) { - throw new Error('invocation failed', { cause: storeList }) - } - - // Add page if it has size - storeList.out.ok.size > 0 && listPages.push(storeList.out.ok.results) - - if (storeList.out.ok.after) { - cursor = storeList.out.ok.after - } else { - break - } - } while (cursor) - - assert.equal( - listPages.length, - data.length, - 'has number of pages of added CARs' - ) - - // Inspect content - const storeList = listPages.flat() - assert.deepEqual( - // list order last-in-first-out - storeList.map(({ link }) => ({ link, size: 5 })), - links.reverse().map((link) => ({ link, size: 5 })) - ) - }, - - 'store/list can page backwards': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = [ - new Uint8Array([11, 22, 33, 44, 55]), - new Uint8Array([22, 33, 44, 55, 66]), - new Uint8Array([33, 44, 55, 66, 77]), - new Uint8Array([44, 55, 66, 77, 88]), - new Uint8Array([55, 66, 77, 88, 99]), - new Uint8Array([66, 77, 88, 99, 11]), - ] - const links = [] - - for (const datum of data) { - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link: await CAR.codec.link(datum), size: datum.byteLength }, - proofs: [proof], - }) - .execute(connection) - if (storeAdd.out.error) { - throw new Error('invocation failed', { cause: storeAdd }) - } - - links.push(storeAdd.out.ok.link) - } - - const size = 3 - - const listResponse = await StoreCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - }, - }) - .execute(connection) - if (listResponse.out.error) { - throw new Error('invocation failed', { cause: listResponse.out.error }) - } - - const secondListResponse = await StoreCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - cursor: listResponse.out.ok.after, - }, - }) - .execute(connection) - if (secondListResponse.out.error) { - throw new Error('invocation failed', { - cause: secondListResponse.out.error, - }) - } - - const prevListResponse = await StoreCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - cursor: secondListResponse.out.ok.before, - pre: true, - }, - }) - .execute(connection) - if (prevListResponse.out.error) { - throw new Error('invocation failed', { - cause: prevListResponse.out.error, - }) - } - - assert.equal(listResponse.out.ok.results.length, 3) - // listResponse is the first page. we used its after to get the second page, and then used the before of the second - // page with the `pre` caveat to list the first page again. the results and cursors should remain the same. - assert.deepEqual( - prevListResponse.out.ok.results[0], - listResponse.out.ok.results[0] - ) - assert.deepEqual( - prevListResponse.out.ok.results[1], - listResponse.out.ok.results[1] - ) - assert.deepEqual( - prevListResponse.out.ok.results[2], - listResponse.out.ok.results[2] - ) - assert.deepEqual(prevListResponse.out.ok.before, listResponse.out.ok.before) - assert.deepEqual(prevListResponse.out.ok.after, listResponse.out.ok.after) - }, - - 'store/get returns shard info': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = [ - new Uint8Array([11, 22, 34, 44, 55]), - new Uint8Array([22, 34, 44, 55, 66]), - ] - const links = [] - for (const datum of data) { - const storeAdd = await StoreCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { link: await CAR.codec.link(datum), size: datum.byteLength }, - proofs: [proof], - }) - .execute(connection) - - if (storeAdd.out.error) { - throw new Error('invocation failed', { cause: storeAdd }) - } - - assert.equal(storeAdd.out.ok.status, 'upload') - links.push(storeAdd.out.ok.link) - } - - const storeGet = await StoreCapabilities.get - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - link: links[0], - }, - }) - .execute(connection) - - if (storeGet.out.error) { - throw new Error('invocation failed', { cause: storeGet }) - } - - assert.deepEqual(storeGet.out.ok.link, links[0]) - assert.equal(storeGet.out.ok.size, data[0].byteLength) - assert.ok(storeGet.out.ok.insertedAt) - }, - - 'store/get returns StoreItemNotFound Failure': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const link = await CAR.codec.link(new Uint8Array([11, 22, 34, 44, 55])) - - const storeGet = await StoreCapabilities.get - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - link, - }, - }) - .execute(connection) - - assert.ok(storeGet.out.error) - assert.equal(storeGet.out.error?.name, 'StoreItemNotFound') - }, -} diff --git a/packages/upload-api/test/handlers/store.spec.js b/packages/upload-api/test/handlers/store.spec.js deleted file mode 100644 index bbfa9646c..000000000 --- a/packages/upload-api/test/handlers/store.spec.js +++ /dev/null @@ -1,4 +0,0 @@ -import { test } from '../test.js' -import * as Store from './store.js' - -test({ 'store/*': Store.test }) diff --git a/packages/upload-api/test/handlers/usage.js b/packages/upload-api/test/handlers/usage.js index b940d8610..e86cd02d3 100644 --- a/packages/upload-api/test/handlers/usage.js +++ b/packages/upload-api/test/handlers/usage.js @@ -1,8 +1,9 @@ import * as CAR from '@ucanto/transport/car' -import { Store, Usage } from '@storacha/capabilities' +import { Usage } from '@storacha/capabilities' import * as API from '../../src/types.js' import { createServer, connect } from '../../src/lib.js' import { alice, registerSpace } from '../util.js' +import { uploadBlob } from '../helpers/blob.js' /** @type {API.Tests} */ export const test = { @@ -17,17 +18,16 @@ export const test = { const link = await CAR.codec.link(data) const size = data.byteLength - const storeAddRes = await Store.add - .invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { link, size }, - proofs: [proof], - }) - .execute(connection) - - assert.ok(storeAddRes.out.ok) + await uploadBlob({ + connection, + issuer: alice, + audience: context.id, + with: spaceDid, + proofs: [proof], + }, { + digest: link.multihash, + bytes: data + }) const usageReportRes = await Usage.report .invoke({ @@ -48,9 +48,5 @@ export const test = { assert.equal(report?.size.final, size) assert.equal(report?.events.length, 1) assert.equal(report?.events[0].delta, size) - assert.equal( - report?.events[0].cause.toString(), - storeAddRes.ran.link().toString() - ) }, } diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index f6c5b177b..3ed5dd34c 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -77,7 +77,6 @@ export const createContext = async ( }, }, // Filecoin - maxUploadSize: 5_000_000_000, filecoinSubmitQueue, pieceOfferQueue, pieceStore, @@ -116,8 +115,6 @@ export const createContext = async ( */ export const cleanupContext = (context) => Promise.all([ - context.carStoreBucket.deactivate(), context.claimsService.deactivate(), - // @ts-expect-error ...context.storageProviders.map((p) => p.deactivate()), ]) diff --git a/packages/upload-api/test/lib.js b/packages/upload-api/test/lib.js index 94b5c66a6..046956264 100644 --- a/packages/upload-api/test/lib.js +++ b/packages/upload-api/test/lib.js @@ -1,12 +1,10 @@ import * as AccessAuthorize from './handlers/access/authorize.js' import * as AccessClaim from './handlers/access/claim.js' import * as AccessDelegate from './handlers/access/delegate.js' -import * as AdminStoreInspect from './handlers/admin/store/inspect.js' import * as AdminUploadInspect from './handlers/admin/upload/inspect.js' import * as RateLimitAdd from './handlers/rate-limit/add.js' import * as RateLimitList from './handlers/rate-limit/list.js' import * as RateLimitRemove from './handlers/rate-limit/remove.js' -import * as Store from './handlers/store.js' import * as Blob from './handlers/blob.js' import * as Ucan from './handlers/ucan.js' import * as Subscription from './handlers/subscription.js' @@ -27,7 +25,6 @@ export * as Context from './helpers/context.js' export * from './util.js' export const test = { - ...Store.test, ...Blob.test, ...Index.test, ...Upload.test, @@ -48,12 +45,10 @@ export const handlerTests = { ...AccessAuthorize, ...AccessClaim, ...AccessDelegate, - ...AdminStoreInspect, ...AdminUploadInspect, ...RateLimitAdd, ...RateLimitList, ...RateLimitRemove, - ...Store.test, ...Blob.test, ...Index.test, ...Ucan.test, @@ -64,7 +59,6 @@ export const handlerTests = { } export { - Store, Upload, Blob, Index, diff --git a/packages/upload-api/test/storage/car-store-bucket.js b/packages/upload-api/test/storage/car-store-bucket.js deleted file mode 100644 index f0db90b67..000000000 --- a/packages/upload-api/test/storage/car-store-bucket.js +++ /dev/null @@ -1,160 +0,0 @@ -import * as API from '../../src/types.js' -import { base64pad } from 'multiformats/bases/base64' -import { SigV4 } from '@web3-storage/sigv4' -import { sha256 } from 'multiformats/hashes/sha2' - -/** - * @implements {API.CarStoreBucket} - */ -export class CarStoreBucket { - /** - * @param {API.CarStoreBucketOptions & {http?: import('http')}} options - */ - static async activate({ http, ...options } = {}) { - const content = new Map() - if (http) { - const server = http.createServer(async (request, response) => { - if (request.method === 'PUT') { - const buffer = new Uint8Array( - parseInt(request.headers['content-length'] || '0') - ) - let offset = 0 - - for await (const chunk of request) { - buffer.set(chunk, offset) - offset += chunk.length - } - - const hash = await sha256.digest(buffer) - const checksum = base64pad.baseEncode(hash.digest) - - if (checksum !== request.headers['x-amz-checksum-sha256']) { - response.writeHead(400, `checksum mismatch`) - } else { - const { pathname } = new URL(request.url || '/', url) - content.set(pathname, buffer) - response.writeHead(200) - } - } else { - response.writeHead(405) - } - - response.end() - // otherwise it keep connection lingering - response.destroy() - }) - await new Promise((resolve) => server.listen(resolve)) - - // @ts-ignore - this is actually what it returns on http - const port = server.address().port - const url = new URL(`http://localhost:${port}`) - - return new CarStoreBucket({ - ...options, - content, - url, - server, - }) - } else { - return new CarStoreBucket({ - ...options, - content, - url: new URL(`http://localhost:8989`), - }) - } - } - - /** - * @param {API.CarStoreBucketOptions & { server?: import('http').Server, url: URL, content: Map }} options - */ - constructor({ - content, - url, - server, - accessKeyId = 'id', - secretAccessKey = 'secret', - bucket = 'my-bucket', - region = 'eu-central-1', - expires, - }) { - this.server = server - this.baseURL = url - this.accessKeyId = accessKeyId - this.secretAccessKey = secretAccessKey - this.bucket = bucket - this.region = region - this.expires = expires - this.content = content - } - - /** - * @returns {Promise} - */ - async deactivate() { - const { server } = this - if (server) { - await new Promise((resolve, reject) => { - // does not exist in node 16 - if (typeof server.closeAllConnections === 'function') { - server.closeAllConnections() - } - - server.close((error) => { - if (error) { - reject(error) - } else { - resolve(undefined) - } - }) - }) - } - } - - /** - * - * @param {API.UnknownLink} link - */ - async has(link) { - return this.content.has(`/${this.bucket}/${link}/${link}.car`) - } - - /** - * @param {API.UnknownLink} link - * @param {number} size - */ - async createUploadUrl(link, size) { - const { bucket, expires, accessKeyId, secretAccessKey, region, baseURL } = - this - // sigv4 - const sig = new SigV4({ - accessKeyId, - secretAccessKey, - region, - }) - - const checksum = base64pad.baseEncode(link.multihash.digest) - const { pathname, search, hash } = sig.sign({ - key: `${link}/${link}.car`, - checksum, - bucket, - expires, - }) - - const url = new URL(baseURL) - url.search = search - url.pathname = `/${bucket}${pathname}` - url.hash = hash - url.searchParams.set( - 'X-Amz-SignedHeaders', - ['content-length', 'host', 'x-amz-checksum-sha256'].join(';') - ) - - return { - url, - headers: { - 'x-amz-checksum-sha256': checksum, - 'content-length': String(size), - }, - } - } -} diff --git a/packages/upload-api/test/storage/index.js b/packages/upload-api/test/storage/index.js index 3775063c8..d2bcbba87 100644 --- a/packages/upload-api/test/storage/index.js +++ b/packages/upload-api/test/storage/index.js @@ -1,6 +1,4 @@ import { Registry as BlobRegistry } from './blob-registry.js' -import { CarStoreBucket } from './car-store-bucket.js' -import { StoreTable } from './store-table.js' import { UploadTable } from './upload-table.js' import { ProvisionsStorage } from './provisions-storage.js' import { DelegationsStorage } from './delegations-storage.js' @@ -19,13 +17,11 @@ import * as AgentStore from './agent-store.js' * @param {{fail(error:unknown): unknown}} [options.assert] */ export async function getServiceStorageImplementations(options) { - const storeTable = new StoreTable() const registry = new BlobRegistry() const uploadTable = new UploadTable() - const carStoreBucket = await CarStoreBucket.activate(options) const revocationsStorage = new RevocationsStorage() const plansStorage = new PlansStorage() - const usageStorage = new UsageStorage(storeTable, registry) + const usageStorage = new UsageStorage(registry) const provisionsStorage = new ProvisionsStorage(options.providers) const subscriptionsStorage = new SubscriptionsStorage(provisionsStorage) const delegationsStorage = new DelegationsStorage() @@ -33,10 +29,8 @@ export async function getServiceStorageImplementations(options) { const agentStore = AgentStore.memory() return { - storeTable, registry, uploadTable, - carStoreBucket, revocationsStorage, plansStorage, usageStorage, diff --git a/packages/upload-api/test/storage/store-table.js b/packages/upload-api/test/storage/store-table.js deleted file mode 100644 index 3cb197048..000000000 --- a/packages/upload-api/test/storage/store-table.js +++ /dev/null @@ -1,134 +0,0 @@ -import * as API from '../../src/types.js' - -/** - * @implements {API.StoreTable} - */ -export class StoreTable { - constructor() { - /** @type {(API.StoreAddInput & API.StoreListItem)[]} */ - this.items = [] - } - - /** - * @param {API.StoreAddInput} input - * @returns {ReturnType} - */ - async insert({ space, issuer, invocation, ...output }) { - if ( - this.items.some((i) => i.space === space && i.link.equals(output.link)) - ) { - return { - error: { name: 'RecordKeyConflict', message: 'record key conflict' }, - } - } - this.items.unshift({ - space, - issuer, - invocation, - ...output, - insertedAt: new Date().toISOString(), - }) - return { ok: output } - } - - /** - * @param {API.UnknownLink} link - * @returns {ReturnType} - */ - async inspect(link) { - const items = this.items.filter((item) => item.link.equals(link)) - return { - ok: { - spaces: items.map((item) => ({ - did: item.space, - insertedAt: item.insertedAt, - })), - }, - } - } - - /** - * @param {API.DID} space - * @param {API.UnknownLink} link - * @returns {ReturnType} - */ - async get(space, link) { - const item = this.items.find( - (i) => i.space === space && i.link.equals(link) - ) - if (!item) { - return { error: { name: 'RecordNotFound', message: 'record not found' } } - } - return { ok: item } - } - - /** - * @param {API.DID} space - * @param {API.UnknownLink} link - * @returns {ReturnType} - */ - async exists(space, link) { - const item = this.items.find( - (i) => i.space === space && i.link.equals(link) - ) - return { ok: !!item } - } - - /** - * @param {API.DID} space - * @param {API.UnknownLink} link - * @returns {ReturnType} - */ - async remove(space, link) { - const item = this.items.find( - (i) => i.space === space && i.link.equals(link) - ) - if (!item) { - return { error: { name: 'RecordNotFound', message: 'record not found' } } - } - this.items = this.items.filter((i) => i !== item) - return { ok: item } - } - - /** - * @param {API.DID} space - * @param {API.ListOptions} options - * @returns {ReturnType} - */ - async list( - space, - { cursor = '0', pre = false, size = this.items.length } = {} - ) { - const offset = parseInt(cursor, 10) - const items = pre ? this.items.slice(0, offset) : this.items.slice(offset) - - const matches = [...items.entries()] - .filter(([n, item]) => item.space === space) - .slice(0, size) - - if (matches.length === 0) { - return { ok: { size: 0, results: [] } } - } - - const first = matches[0] - const last = matches[matches.length - 1] - - const start = first[0] || 0 - const end = last[0] || 0 - const values = matches.map(([_, item]) => item) - - const [before, after, results] = pre - ? [`${start}`, `${end + 1}`, values] - : [`${start + offset}`, `${end + 1 + offset}`, values] - - return { - ok: { - size: values.length, - before, - after, - cursor: after, - results, - }, - } - } -} diff --git a/packages/upload-api/test/storage/usage-storage.js b/packages/upload-api/test/storage/usage-storage.js index 740521a97..f364edf6e 100644 --- a/packages/upload-api/test/storage/usage-storage.js +++ b/packages/upload-api/test/storage/usage-storage.js @@ -3,11 +3,9 @@ /** @implements {UsageStore} */ export class UsageStorage { /** - * @param {import('./store-table.js').StoreTable} storeTable * @param {import('./blob-registry.js').Registry} blobRegistry */ - constructor(storeTable, blobRegistry) { - this.storeTable = storeTable + constructor(blobRegistry) { this.blobRegistry = blobRegistry /** * @type {Record} @@ -16,15 +14,9 @@ export class UsageStorage { } get items() { - return [ - ...this.storeTable.items.map((item) => ({ - ...item, - cause: item.invocation, - })), - ...[...this.blobRegistry.data.entries()].flatMap(([space, entries]) => - entries.map((e) => ({ space, size: e.blob.size, ...e })) - ), - ] + return [...this.blobRegistry.data.entries()].flatMap(([space, entries]) => + entries.map((e) => ({ space, size: e.blob.size, ...e })) + ) } /** diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 19c13fb3b..ef8b707cb 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -3,7 +3,6 @@ import { Storefront } from '@storacha/filecoin-client' import * as Link from 'multiformats/link' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' -import * as Store from './store.js' import * as Blob from './blob/index.js' import * as BlobAdd from './blob/add.js' import * as Index from './index/index.js' @@ -15,7 +14,7 @@ import * as CAR from './car.js' import { ShardingStream, defaultFileComparator } from './sharding.js' import { indexShardedDAG } from '@storacha/blob-index' -export { Blob, Index, Store, Upload, UnixFS, CAR } +export { Blob, Index, Upload, UnixFS, CAR } export * from './sharding.js' export { receiptsEndpoint } from './service.js' export * as Receipt from './receipts.js' diff --git a/packages/upload-client/src/store.js b/packages/upload-client/src/store.js deleted file mode 100644 index a2cbb47a0..000000000 --- a/packages/upload-client/src/store.js +++ /dev/null @@ -1,303 +0,0 @@ -import { CAR } from '@ucanto/transport' -import * as StoreCapabilities from '@storacha/capabilities/store' -import { SpaceDID } from '@storacha/capabilities/utils' -import retry, { AbortError } from 'p-retry' -import { servicePrincipal, connection } from './service.js' -import { REQUEST_RETRIES } from './constants.js' - -/** - * - * @param {string} url - * @param {import('./types.js').ProgressFn} handler - */ -function createUploadProgressHandler(url, handler) { - /** - * - * @param {import('./types.js').ProgressStatus} status - */ - function onUploadProgress({ total, loaded, lengthComputable }) { - return handler({ total, loaded, lengthComputable, url }) - } - return onUploadProgress -} - -/** - * Store a DAG encoded as a CAR file. The issuer needs the `store/add` - * delegated capability. - * - * Required delegated capability proofs: `store/add` - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `store/add` delegated capability. - * @param {Blob|Uint8Array} car CAR file data. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} - */ -export async function add( - { issuer, with: resource, proofs, audience }, - car, - options = {} -) { - // TODO: validate blob contains CAR data - /* c8 ignore next 2 */ - const bytes = - car instanceof Uint8Array ? car : new Uint8Array(await car.arrayBuffer()) - const link = await CAR.codec.link(bytes) - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await retry( - async () => { - return await StoreCapabilities.add - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { link, size: bytes.length }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - }, - { - onFailedAttempt: console.warn, - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!result.out.ok) { - throw new Error(`failed ${StoreCapabilities.add.can} invocation`, { - cause: result.out.error, - }) - } - - // Return early if it was already uploaded. - if (result.out.ok.status === 'done') { - return link - } - - const responseAddUpload = result.out.ok - - const fetchWithUploadProgress = - options.fetchWithUploadProgress || - options.fetch || - globalThis.fetch.bind(globalThis) - - let fetchDidCallUploadProgressCb = false - const res = await retry( - async () => { - try { - const res = await fetchWithUploadProgress(responseAddUpload.url, { - method: 'PUT', - body: car, - headers: responseAddUpload.headers, - signal: options.signal, - onUploadProgress: (status) => { - fetchDidCallUploadProgressCb = true - if (options.onUploadProgress) - createUploadProgressHandler( - responseAddUpload.url, - options.onUploadProgress - )(status) - }, - // @ts-expect-error - this is needed by recent versions of node - see https://github.com/bluesky-social/atproto/pull/470 for more info - duplex: 'half', - }) - if (res.status >= 400 && res.status < 500) { - throw new AbortError(`upload failed: ${res.status}`) - } - return res - } catch (err) { - if (options.signal?.aborted === true) { - throw new AbortError('upload aborted') - } - throw err - } - }, - { - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!fetchDidCallUploadProgressCb && options.onUploadProgress) { - // the fetch implementation didn't support onUploadProgress - const carBlob = new Blob([car]) - options.onUploadProgress({ - total: carBlob.size, - loaded: carBlob.size, - lengthComputable: false, - }) - } - - if (!res.ok) { - throw new Error(`upload failed: ${res.status}`) - } - - return link -} - -/** - * Get details of a stored item. - * - * Required delegated capability proofs: `store/get` - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `store/get` delegated capability. - * @param {import('multiformats/link').Link} link CID of stored CAR file. - * @param {import('./types.js').RequestOptions} [options] - * @returns {Promise} - */ -export async function get( - { issuer, with: resource, proofs, audience }, - link, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await retry( - async () => { - return await StoreCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { link }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - }, - { - onFailedAttempt: console.warn, - retries: options.retries ?? REQUEST_RETRIES, - } - ) - - if (!result.out.ok) { - throw new Error(`failed ${StoreCapabilities.get.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * List CAR files stored by the issuer. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `store/list` delegated capability. - * @param {import('./types.js').ListRequestOptions} [options] - * @returns {Promise} - */ -export async function list( - { issuer, with: resource, proofs, audience }, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await StoreCapabilities.list - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - proofs, - nb: { - cursor: options.cursor, - size: options.size, - pre: options.pre, - }, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${StoreCapabilities.list.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out.ok -} - -/** - * Remove a stored CAR file by CAR CID. - * - * @param {import('./types.js').InvocationConfig} conf Configuration - * for the UCAN invocation. An object with `issuer`, `with` and `proofs`. - * - * The `issuer` is the signing authority that is issuing the UCAN - * invocation(s). It is typically the user _agent_. - * - * The `with` is the resource the invocation applies to. It is typically the - * DID of a space. - * - * The `proofs` are a set of capability delegations that prove the issuer - * has the capability to perform the action. - * - * The issuer needs the `store/remove` delegated capability. - * @param {import('./types.js').CARLink} link CID of CAR file to remove. - * @param {import('./types.js').RequestOptions} [options] - */ -export async function remove( - { issuer, with: resource, proofs, audience }, - link, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - const result = await StoreCapabilities.remove - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: SpaceDID.from(resource), - nb: { link }, - proofs, - nonce: options.nonce, - }) - .execute(conn) - - if (!result.out.ok) { - throw new Error(`failed ${StoreCapabilities.remove.can} invocation`, { - cause: result.out.error, - }) - } - - return result.out -} diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 54fedca02..147dec25e 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -40,18 +40,6 @@ import { SpaceIndexAdd, SpaceIndexAddSuccess, SpaceIndexAddFailure, - StoreAdd, - StoreAddSuccess, - StoreAddSuccessUpload, - StoreAddSuccessDone, - StoreGet, - StoreGetFailure, - StoreList, - StoreListSuccess, - StoreListItem, - StoreRemove, - StoreRemoveSuccess, - StoreRemoveFailure, UploadAdd, UploadAddSuccess, UploadList, @@ -62,7 +50,6 @@ import { ListResponse, CARLink, PieceLink, - StoreGetSuccess, UploadGet, UploadGetSuccess, UploadGetFailure, @@ -115,18 +102,6 @@ export type { SpaceIndexAdd, SpaceIndexAddSuccess, SpaceIndexAddFailure, - StoreAdd, - StoreAddSuccess, - StoreAddSuccessUpload, - StoreAddSuccessDone, - StoreGetSuccess, - StoreGetFailure, - StoreList, - StoreListSuccess, - StoreListItem, - StoreRemove, - StoreRemoveSuccess, - StoreRemoveFailure, UploadAdd, UploadAddSuccess, UploadGetSuccess, @@ -197,12 +172,6 @@ export interface Service extends StorefrontService { > } } - store: { - add: ServiceMethod - get: ServiceMethod - remove: ServiceMethod - list: ServiceMethod - } upload: { add: ServiceMethod get: ServiceMethod diff --git a/packages/upload-client/test/helpers/mocks.js b/packages/upload-client/test/helpers/mocks.js index d12fb5cc6..1805eb168 100644 --- a/packages/upload-client/test/helpers/mocks.js +++ b/packages/upload-client/test/helpers/mocks.js @@ -11,7 +11,6 @@ const notImplemented = () => { * blob: Partial * index: Partial * }> - * store: Partial * upload: Partial * usage: Partial * filecoin: Partial @@ -38,12 +37,6 @@ export function mockService(impl) { add: withCallCount(impl.space?.index?.add ?? notImplemented), }, }, - store: { - add: withCallCount(impl.store?.add ?? notImplemented), - get: withCallCount(impl.store?.get ?? notImplemented), - list: withCallCount(impl.store?.list ?? notImplemented), - remove: withCallCount(impl.store?.remove ?? notImplemented), - }, upload: { add: withCallCount(impl.upload?.add ?? notImplemented), get: withCallCount(impl.upload?.get ?? notImplemented), diff --git a/packages/upload-client/test/store.test.js b/packages/upload-client/test/store.test.js deleted file mode 100644 index f7dc84418..000000000 --- a/packages/upload-client/test/store.test.js +++ /dev/null @@ -1,786 +0,0 @@ -import assert from 'assert' -import * as Client from '@ucanto/client' -import * as Server from '@ucanto/server' -import { provide } from '@ucanto/server' -import * as CAR from '@ucanto/transport/car' -import * as Signer from '@ucanto/principal/ed25519' -import * as StoreCapabilities from '@storacha/capabilities/store' -import * as Store from '../src/store.js' -import { serviceSigner } from './fixtures.js' -import { randomCAR } from './helpers/random.js' -import { mockService } from './helpers/mocks.js' -import { validateAuthorization } from './helpers/utils.js' -import { fetchWithUploadProgress } from '../src/fetch-with-upload-progress.js' - -describe('Store.add', () => { - it('stores a DAG with the service', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - /** @type {import('../src/types.js').StoreAddSuccessUpload} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - link: car.cid, - with: space.did(), - allocated: car.size, - } - - const service = mockService({ - store: { - add: provide(StoreCapabilities.add, ({ invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, StoreCapabilities.add.can) - assert.equal(invCap.with, space.did()) - assert.equal(String(invCap.nb?.link), car.cid.toString()) - return { ok: res } - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - /** @type {import('../src/types.js').ProgressStatus[]} */ - const progress = [] - const carCID = await Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { - connection, - onUploadProgress: (status) => { - assert(typeof status.loaded === 'number' && status.loaded > 0) - progress.push(status) - }, - fetchWithUploadProgress, - } - ) - - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 1) - assert.equal( - progress.reduce((max, { loaded }) => Math.max(max, loaded), 0), - 225 - ) - - assert(carCID) - assert.equal(carCID.toString(), car.cid.toString()) - - // make sure it can also work without fetchWithUploadProgress - /** @type {import('../src/types.js').ProgressStatus[]} */ - let progressWithoutUploadProgress = [] - const addedWithoutUploadProgress = await Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { - connection, - onUploadProgress: (status) => { - progressWithoutUploadProgress.push(status) - }, - } - ) - assert.equal(addedWithoutUploadProgress.toString(), car.cid.toString()) - assert.equal( - progressWithoutUploadProgress.reduce( - (max, { loaded }) => Math.max(max, loaded), - 0 - ), - 225 - ) - }) - - it('throws for bucket URL client error 4xx', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - /** @type {import('../src/types.js').StoreAddSuccessUpload} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9400', // this bucket always returns a 400 - link: car.cid, - with: space.did(), - allocated: car.size, - } - - const service = mockService({ - store: { - add: provide(StoreCapabilities.add, () => ({ ok: res })), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - await assert.rejects( - Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { connection } - ), - { - message: 'upload failed: 400', - } - ) - }) - - it('throws for bucket URL server error 5xx', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - /** @type {import('../src/types.js').StoreAddSuccessUpload} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9500', // this bucket always returns a 500 - link: car.cid, - with: space.did(), - allocated: car.size, - } - - const service = mockService({ - store: { - add: provide(StoreCapabilities.add, () => ({ ok: res })), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - await assert.rejects( - Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { connection } - ), - { - message: 'upload failed: 500', - } - ) - }) - - it('skips sending CAR if status = done', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - /** @type {import('../src/types.js').StoreAddSuccessDone} */ - const res = { - status: 'done', - // @ts-expect-error - headers: { 'x-test': 'true' }, - } - - const service = mockService({ - store: { - add: provide(StoreCapabilities.add, () => ({ ok: res })), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - const carCID = await Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { - connection, - } - ) - - assert(service.store.add.called) - assert.equal(service.store.add.callCount, 1) - - assert(carCID) - assert.equal(carCID.toString(), car.cid.toString()) - }) - - it('aborts', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - /** @type {import('../src/types.js').StoreAddSuccess} */ - const res = { - status: 'upload', - headers: { 'x-test': 'true' }, - url: 'http://localhost:9200', - link: car.cid, - with: space.did(), - allocated: car.size, - } - - const service = mockService({ - store: { - add: provide(StoreCapabilities.add, () => ({ ok: res })), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - const proofs = [ - await StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const controller = new AbortController() - controller.abort() // already aborted - - await assert.rejects( - Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { - connection, - signal: controller.signal, - } - ), - { name: 'Error', message: 'upload aborted' } - ) - }) - - it('throws on service error', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.add.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - add: provide(StoreCapabilities.add, () => { - throw new Server.Failure('boom') - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - await assert.rejects( - Store.add( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car, - { connection } - ), - { message: 'failed store/add invocation' } - ) - }) -}) - -describe('Store.list', () => { - it('lists stored CAR files', async () => { - const car = await randomCAR(128) - const res = { - cursor: 'test', - size: 1000, - results: [ - { - link: car.cid, - size: 123, - insertedAt: '1970-01-01T00:00:00.000Z', - }, - ], - } - - const space = await Signer.generate() - const agent = await Signer.generate() - - const proofs = [ - await StoreCapabilities.list.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - list: provide(StoreCapabilities.list, ({ invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, StoreCapabilities.list.can) - assert.equal(invCap.with, space.did()) - return { ok: res } - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - const list = await Store.list( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - { connection } - ) - - assert(service.store.list.called) - assert.equal(service.store.list.callCount, 1) - - assert.equal(list.cursor, res.cursor) - assert.equal(list.size, res.size) - assert(list.results) - assert.equal(list.results.length, res.results.length) - list.results.forEach((r, i) => { - assert.deepEqual(r.link, res.results[i].link) - assert.deepEqual(r.size, res.results[i].size) - }) - }) - - it('paginates', async () => { - const cursor = 'test' - const page0 = { - cursor, - size: 1, - results: [ - { - link: (await randomCAR(128)).cid, - size: 123, - insertedAt: '1970-01-01T00:00:00.000Z', - }, - ], - } - const page1 = { - size: 1, - results: [ - { - link: (await randomCAR(128)).cid, - size: 123, - insertedAt: '1970-01-01T00:00:00.000Z', - }, - ], - } - - const space = await Signer.generate() - const agent = await Signer.generate() - - const proofs = [ - await StoreCapabilities.list.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - list: provide(StoreCapabilities.list, ({ invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, StoreCapabilities.list.can) - assert.equal(invCap.with, space.did()) - assert.equal(invCap.nb?.size, 1) - return { ok: invCap.nb?.cursor === cursor ? page1 : page0 } - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - const results0 = await Store.list( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - { size: 1, connection } - ) - const results1 = await Store.list( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - { size: 1, cursor: results0.cursor, connection } - ) - - assert(service.store.list.called) - assert.equal(service.store.list.callCount, 2) - - assert.equal(results0.cursor, cursor) - assert(results0.results) - assert.equal(results0.results.length, page0.results.length) - results0.results.forEach((r, i) => { - assert.equal(r.link.toString(), page0.results[i].link.toString()) - assert.equal(r.size, page0.results[i].size) - }) - - assert(results1.results) - assert.equal(results1.cursor, undefined) - assert.equal(results1.results.length, page1.results.length) - results1.results.forEach((r, i) => { - assert.equal(r.link.toString(), page1.results[i].link.toString()) - assert.equal(r.size, page1.results[i].size) - }) - }) - - it('throws on service error', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - - const proofs = [ - await StoreCapabilities.list.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - list: provide(StoreCapabilities.list, () => { - throw new Server.Failure('boom') - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - await assert.rejects( - Store.list( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - { connection } - ), - { - message: 'failed store/list invocation', - } - ) - }) -}) - -describe('Store.remove', () => { - it('removes a stored CAR file', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.remove.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - remove: provide(StoreCapabilities.remove, ({ invocation }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.equal(invCap.can, StoreCapabilities.remove.can) - assert.equal(invCap.with, space.did()) - assert.equal(String(invCap.nb?.link), car.cid.toString()) - return { ok: { size: car.size } } - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - const result = await Store.remove( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car.cid, - { connection } - ) - - assert(service.store.remove.called) - assert.equal(service.store.remove.callCount, 1) - - assert(result.ok) - assert.equal(result.ok.size, car.size) - }) - - it('throws on service error', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.remove.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - remove: provide(StoreCapabilities.remove, () => { - throw new Server.Failure('boom') - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - await assert.rejects( - Store.remove( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car.cid, - { connection } - ), - { message: 'failed store/remove invocation' } - ) - }) -}) - -describe('Store.get', () => { - it('gets stored item', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.get.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - get: provide(StoreCapabilities.get, ({ invocation, capability }) => { - assert.equal(invocation.issuer.did(), agent.did()) - assert.equal(invocation.capabilities.length, 1) - assert.equal(capability.can, StoreCapabilities.get.can) - assert.equal(capability.with, space.did()) - assert.equal(String(capability.nb?.link), car.cid.toString()) - return { - ok: { - link: car.cid, - size: car.size, - insertedAt: new Date().toISOString(), - }, - } - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - const result = await Store.get( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car.cid, - { connection } - ) - - assert(service.store.get.called) - assert.equal(service.store.get.callCount, 1) - - assert.equal(result.link.toString(), car.cid.toString()) - assert.equal(result.size, car.size) - }) - - it('throws on service error', async () => { - const space = await Signer.generate() - const agent = await Signer.generate() - const car = await randomCAR(128) - - const proofs = [ - await StoreCapabilities.get.delegate({ - issuer: space, - audience: agent, - with: space.did(), - expiration: Infinity, - }), - ] - - const service = mockService({ - store: { - get: provide(StoreCapabilities.get, () => { - throw new Server.Failure('boom') - }), - }, - }) - - const server = Server.create({ - id: serviceSigner, - service, - codec: CAR.inbound, - validateAuthorization, - }) - const connection = Client.connect({ - id: serviceSigner, - codec: CAR.outbound, - channel: server, - }) - - await assert.rejects( - Store.get( - { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, - car.cid, - { connection } - ), - { message: 'failed store/get invocation' } - ) - }) -}) diff --git a/packages/w3up-client/src/capability/store.js b/packages/w3up-client/src/capability/store.js deleted file mode 100644 index b651174cf..000000000 --- a/packages/w3up-client/src/capability/store.js +++ /dev/null @@ -1,71 +0,0 @@ -import { Store } from '@storacha/upload-client' -import { Store as StoreCapabilities } from '@storacha/capabilities' -import { Base } from '../base.js' - -/** - * Client for interacting with the `store/*` capabilities. - */ -export class StoreClient extends Base { - /** - * Store a DAG encoded as a CAR file. - * - * Required delegated capabilities: - * - `store/add` - * - * @deprecated Use `client.capability.blob.add()` instead. - * @param {Blob} car - CAR file data. - * @param {import('../types.js').RequestOptions} [options] - */ - async add(car, options = {}) { - const conf = await this._invocationConfig([StoreCapabilities.add.can]) - options.connection = this._serviceConf.upload - return Store.add(conf, car, options) - } - - /** - * Get details of a stored item. - * - * Required delegated capabilities: - * - `store/get` - * - * @deprecated Use `client.capability.blob.get()` instead. - * @param {import('../types.js').CARLink} link - Root data CID for the DAG that was stored. - * @param {import('../types.js').RequestOptions} [options] - */ - async get(link, options = {}) { - const conf = await this._invocationConfig([StoreCapabilities.get.can]) - options.connection = this._serviceConf.upload - return Store.get(conf, link, options) - } - - /** - * List CAR files stored to the resource. - * - * Required delegated capabilities: - * - `store/list` - * - * @deprecated Use `client.capability.blob.list()` instead. - * @param {import('../types.js').ListRequestOptions} [options] - */ - async list(options = {}) { - const conf = await this._invocationConfig([StoreCapabilities.list.can]) - options.connection = this._serviceConf.upload - return Store.list(conf, options) - } - - /** - * Remove a stored CAR file by CAR CID. - * - * Required delegated capabilities: - * - `store/remove` - * - * @deprecated Use `client.capability.blob.remove()` instead. - * @param {import('../types.js').CARLink} link - CID of CAR file to remove. - * @param {import('../types.js').RequestOptions} [options] - */ - async remove(link, options = {}) { - const conf = await this._invocationConfig([StoreCapabilities.remove.can]) - options.connection = this._serviceConf.upload - return Store.remove(conf, link, options) - } -} diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index 40f740ce0..5398f002e 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -17,7 +17,6 @@ import { Space } from './space.js' import { AgentDelegation } from './delegation.js' import { BlobClient } from './capability/blob.js' import { IndexClient } from './capability/index.js' -import { StoreClient } from './capability/store.js' import { UploadClient } from './capability/upload.js' import { SpaceClient } from './capability/space.js' import { SubscriptionClient } from './capability/subscription.js' @@ -35,7 +34,6 @@ export { FilecoinClient, IndexClient, PlanClient, - StoreClient, SpaceClient, SubscriptionClient, UploadClient, @@ -58,7 +56,6 @@ export class Client extends Base { plan: new PlanClient(agentData, options), space: new SpaceClient(agentData, options), blob: new BlobClient(agentData, options), - store: new StoreClient(agentData, options), subscription: new SubscriptionClient(agentData, options), upload: new UploadClient(agentData, options), usage: new UsageClient(agentData, options), @@ -488,24 +485,7 @@ export class Client extends Base { // Remove shards if (upload.shards?.length) { await Promise.allSettled( - upload.shards.map(async (shard) => { - try { - const res = await this.capability.blob.remove(shard.multihash) - /* c8 ignore start */ - // if no size, the blob was not found, try delete from store - if (res.ok && res.ok.size === 0) { - await this.capability.store.remove(shard) - } - } catch (/** @type {any} */ error) { - // If not found, we can tolerate error as it may be a consecutive call for deletion where first failed - if (error?.cause?.name !== 'StoreItemNotFound') { - throw new Error(`failed to remove shard: ${shard}`, { - cause: error, - }) - } - /* c8 ignore next 4 */ - } - }) + upload.shards.map(shard => this.capability.blob.remove(shard.multihash)) ) } diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 72a0a96f9..0b0914658 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -127,12 +127,6 @@ export type { SpaceBlobRemoveFailure, SpaceIndexAddSuccess, SpaceIndexAddFailure, - StoreAddSuccess, - StoreGetSuccess, - StoreGetFailure, - StoreRemoveSuccess, - StoreRemoveFailure, - StoreListSuccess, UploadAddSuccess, UploadGetSuccess, UploadGetFailure, diff --git a/packages/w3up-client/test/capability/store.test.js b/packages/w3up-client/test/capability/store.test.js deleted file mode 100644 index ce616c44a..000000000 --- a/packages/w3up-client/test/capability/store.test.js +++ /dev/null @@ -1,147 +0,0 @@ -import { AgentData } from '@storacha/access/agent' -import { randomCAR } from '../helpers/random.js' -import { Client } from '../../src/client.js' -import * as Test from '../test.js' - -export const StoreClient = Test.withContext({ - 'should store a CAR file': async ( - assert, - { connection, provisionsStorage, storeTable } - ) => { - const alice = new Client(await AgentData.create(), { - // @ts-ignore - serviceConf: { - access: connection, - upload: connection, - }, - }) - - const space = await alice.createSpace('test') - const auth = await space.createAuthorization(alice) - await alice.addSpace(auth) - await alice.setCurrentSpace(space.did()) - - // Then we setup a billing for this account - await provisionsStorage.put({ - // @ts-expect-error - provider: connection.id.did(), - account: alice.agent.did(), - consumer: space.did(), - }) - - const car = await randomCAR(128) - const carCID = await alice.capability.store.add(car) - - assert.deepEqual(await storeTable.exists(space.did(), car.cid), { - ok: true, - }) - - assert.equal(carCID.toString(), car.cid.toString()) - }, - - 'should list stored CARs': async ( - assert, - { connection, provisionsStorage, storeTable } - ) => { - const alice = new Client(await AgentData.create(), { - // @ts-ignore - serviceConf: { - access: connection, - upload: connection, - }, - }) - - const space = await alice.createSpace('test') - const auth = await space.createAuthorization(alice) - await alice.addSpace(auth) - await alice.setCurrentSpace(space.did()) - - // Then we setup a billing for this account - await provisionsStorage.put({ - // @ts-expect-error - provider: connection.id.did(), - account: alice.agent.did(), - consumer: space.did(), - }) - - const car = await randomCAR(128) - const carCID = await alice.capability.store.add(car) - assert.deepEqual(carCID, car.cid) - - const { - results: [entry], - } = await alice.capability.store.list() - - assert.deepEqual(entry.link, car.cid) - assert.deepEqual(entry.size, car.size) - }, - 'should remove a stored CAR': async ( - assert, - { connection, provisionsStorage } - ) => { - const alice = new Client(await AgentData.create(), { - // @ts-ignore - serviceConf: { - access: connection, - upload: connection, - }, - }) - - const space = await alice.createSpace('test') - const auth = await space.createAuthorization(alice) - await alice.addSpace(auth) - await alice.setCurrentSpace(space.did()) - - // Then we setup a billing for this account - await provisionsStorage.put({ - // @ts-expect-error - provider: connection.id.did(), - account: alice.agent.did(), - consumer: space.did(), - }) - - const car = await randomCAR(128) - const cid = await alice.capability.store.add(car) - - const result = await alice.capability.store.remove(cid) - assert.ok(result.ok) - }, - - 'should get a stored item': async ( - assert, - { connection, provisionsStorage } - ) => { - const car = await randomCAR(128) - - const alice = new Client(await AgentData.create(), { - // @ts-ignore - serviceConf: { - access: connection, - upload: connection, - }, - }) - - const space = await alice.createSpace('test') - const auth = await space.createAuthorization(alice) - await alice.addSpace(auth) - await alice.setCurrentSpace(space.did()) - - // Then we setup a billing for this account - await provisionsStorage.put({ - // @ts-expect-error - provider: connection.id.did(), - account: alice.agent.did(), - consumer: space.did(), - }) - - const cid = await alice.capability.store.add(car) - assert.deepEqual(cid, car.cid) - - const result = await alice.capability.store.get(car.cid) - - assert.equal(result.link.toString(), car.cid.toString()) - assert.equal(result.size, car.size) - }, -}) - -Test.test({ StoreClient }) diff --git a/packages/w3up-client/test/space.test.js b/packages/w3up-client/test/space.test.js index e36baf2a6..47f47410a 100644 --- a/packages/w3up-client/test/space.test.js +++ b/packages/w3up-client/test/space.test.js @@ -1,10 +1,10 @@ import * as Signer from '@ucanto/principal/ed25519' -import * as StoreCapabilities from '@storacha/capabilities/store' import * as Test from './test.js' import { Space } from '../src/space.js' import * as Account from '../src/account.js' import * as Result from '../src/result.js' import { randomCAR } from './helpers/random.js' +import { receiptsEndpoint } from './helpers/utils.js' /** * @type {Test.Suite} @@ -38,11 +38,8 @@ export const testSpace = Test.withContext({ const size = 1138 const archive = await randomCAR(size) - await client.agent.invokeAndExecute(StoreCapabilities.add, { - nb: { - link: archive.cid, - size, - }, + await client.capability.blob.add(new Blob([archive.bytes]), { + receiptsEndpoint, }) const found = client.spaces().find((s) => s.did() === space.did()) From e60c74dbbd2449a07a7e23671c82aa664d8b82b3 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 5 Nov 2024 10:45:56 +0000 Subject: [PATCH 2/4] chore: remove unused dependency --- packages/upload-api/package.json | 1 - pnpm-lock.yaml | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/packages/upload-api/package.json b/packages/upload-api/package.json index e85e7ceff..ccbc2e693 100644 --- a/packages/upload-api/package.json +++ b/packages/upload-api/package.json @@ -217,7 +217,6 @@ "@types/mocha": "^10.0.1", "@ucanto/core": "^10.0.1", "@web-std/blob": "^3.0.5", - "@web3-storage/sigv4": "^1.0.2", "is-subset": "^0.1.1", "mocha": "^10.2.0", "typescript": "5.2.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c40dc54a..e4e7ca359 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -486,9 +486,6 @@ importers: '@web-std/blob': specifier: ^3.0.5 version: 3.0.5 - '@web3-storage/sigv4': - specifier: ^1.0.2 - version: 1.0.2 is-subset: specifier: ^0.1.1 version: 0.1.1 @@ -2370,9 +2367,6 @@ packages: '@web3-storage/data-segment@5.3.0': resolution: {integrity: sha512-zFJ4m+pEKqtKatJNsFrk/2lHeFSbkXZ6KKXjBe7/2ayA9wAar7T/unewnOcZrrZTnCWmaxKsXWqdMFy9bXK9dw==} - '@web3-storage/sigv4@1.0.2': - resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==} - '@webassemblyjs/ast@1.12.1': resolution: {integrity: sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==} @@ -9710,10 +9704,6 @@ snapshots: multiformats: 13.3.1 sync-multihash-sha2: 1.0.0 - '@web3-storage/sigv4@1.0.2': - dependencies: - '@noble/hashes': 1.5.0 - '@webassemblyjs/ast@1.12.1': dependencies: '@webassemblyjs/helper-numbers': 1.11.6 From cad14add236349a72696478b0defb2e4ae163a4d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 5 Nov 2024 10:51:58 +0000 Subject: [PATCH 3/4] chore: attw --- packages/w3up-client/package.json | 4 ---- pnpm-lock.yaml | 8 ++++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index a0b36bee1..b57c8fb62 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -80,10 +80,6 @@ "types": "./dist/src/capability/space.d.ts", "import": "./dist/src/capability/space.js" }, - "./capability/store": { - "types": "./dist/src/capability/store.d.ts", - "import": "./dist/src/capability/store.js" - }, "./capability/subscription": { "types": "./dist/src/capability/subscription.d.ts", "import": "./dist/src/capability/subscription.js" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4e7ca359..2c471b108 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3362,8 +3362,8 @@ packages: resolution: {integrity: sha512-M9qw6oUILGVrcENMSRRefE1MbHPIz0h79EKIeJWK9v563aT9Qkh8aEHPO1H5vi970wPirNY+jO9OpFoLiMsMGA==} engines: {node: '>=6'} - electron-to-chromium@1.5.50: - resolution: {integrity: sha512-eMVObiUQ2LdgeO1F/ySTXsvqvxb6ZH2zPGaMYsWzRDdOddUa77tdmI0ltg+L16UpbWdhPmuF3wIQYyQq65WfZw==} + electron-to-chromium@1.5.51: + resolution: {integrity: sha512-kKeWV57KSS8jH4alKt/jKnvHPmJgBxXzGUSbMd4eQF+iOsVPl7bz2KUmu6eo80eMP8wVioTfTyTzdMgM15WXNg==} emoji-regex@10.4.0: resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==} @@ -10123,7 +10123,7 @@ snapshots: browserslist@4.24.2: dependencies: caniuse-lite: 1.0.30001677 - electron-to-chromium: 1.5.50 + electron-to-chromium: 1.5.51 node-releases: 2.0.18 update-browserslist-db: 1.1.1(browserslist@4.24.2) @@ -10878,7 +10878,7 @@ snapshots: dependencies: encoding: 0.1.13 - electron-to-chromium@1.5.50: {} + electron-to-chromium@1.5.51: {} emoji-regex@10.4.0: {} From f4f5f5accc0c80f3dfb1b6916d04df0e594c89af Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 5 Nov 2024 10:54:05 +0000 Subject: [PATCH 4/4] chore: formatter --- .../upload-api/test/external-service/index.js | 50 ++++++----- .../test/external-service/storage-node.js | 83 ++++++++++++------- packages/upload-api/test/handlers/usage.js | 23 ++--- packages/w3up-client/src/client.js | 4 +- 4 files changed, 96 insertions(+), 64 deletions(-) diff --git a/packages/upload-api/test/external-service/index.js b/packages/upload-api/test/external-service/index.js index 73e767d46..005db99bb 100644 --- a/packages/upload-api/test/external-service/index.js +++ b/packages/upload-api/test/external-service/index.js @@ -22,29 +22,33 @@ export const getExternalServiceImplementations = async (config) => { const claimsService = await ClaimsService.activate(config) const blobRetriever = BlobRetriever.create(claimsService) - const storageProviders = await Promise.all(config.http ? [ - StorageNode.activate({ - http: config.http, - claimsService, - ...principalResolver, - }), - StorageNode.activate({ - http: config.http, - claimsService, - ...principalResolver, - }), - ] : [ - BrowserStorageNode.activate({ - port: 8989, - claimsService, - ...principalResolver, - }), - BrowserStorageNode.activate({ - port: 8990, - claimsService, - ...principalResolver, - }), - ]) + const storageProviders = await Promise.all( + config.http + ? [ + StorageNode.activate({ + http: config.http, + claimsService, + ...principalResolver, + }), + StorageNode.activate({ + http: config.http, + claimsService, + ...principalResolver, + }), + ] + : [ + BrowserStorageNode.activate({ + port: 8989, + claimsService, + ...principalResolver, + }), + BrowserStorageNode.activate({ + port: 8990, + claimsService, + ...principalResolver, + }), + ] + ) const router = Router.create(config.serviceID, storageProviders) return { claimsService, diff --git a/packages/upload-api/test/external-service/storage-node.js b/packages/upload-api/test/external-service/storage-node.js index dc3c44f5b..60be4c642 100644 --- a/packages/upload-api/test/external-service/storage-node.js +++ b/packages/upload-api/test/external-service/storage-node.js @@ -11,18 +11,18 @@ import { CAR, HTTP } from '@ucanto/transport' import * as Server from '@ucanto/server' import { connect } from '@ucanto/client' -/** +/** + * @typedef {{ + * has: (digest: API.MultihashDigest) => Promise + * get: (digest: API.MultihashDigest) => Promise + * set: (digest: API.MultihashDigest, bytes: Uint8Array) => Promise + * }} ContentStore * @typedef {{ -* has: (digest: API.MultihashDigest) => Promise -* get: (digest: API.MultihashDigest) => Promise -* set: (digest: API.MultihashDigest, bytes: Uint8Array) => Promise -* }} ContentStore -* @typedef {{ -* has: (digest: API.MultihashDigest) => Promise -* add: (digest: API.MultihashDigest) => Promise -* }} AllocationStore -* @typedef {import('../../src/types/blob.js').BlobService} BlobService -*/ + * has: (digest: API.MultihashDigest) => Promise + * add: (digest: API.MultihashDigest) => Promise + * }} AllocationStore + * @typedef {import('../../src/types/blob.js').BlobService} BlobService + */ export const MaxUploadSize = 127 * (1 << 25) @@ -34,7 +34,9 @@ const contentKey = (digest) => { /** @param {string} key */ const contentDigest = (key) => - Digest.decode(base58btc.decode(key.split('/').pop()?.replace('.blob', '') ?? '')) + Digest.decode( + base58btc.decode(key.split('/').pop()?.replace('.blob', '') ?? '') + ) /** * @param {{ @@ -45,7 +47,12 @@ const contentDigest = (key) => * }} config * @returns {BlobService} */ -const createService = ({ baseURL, claimsService, contentStore, allocationStore }) => ({ +const createService = ({ + baseURL, + claimsService, + contentStore, + allocationStore, +}) => ({ blob: { allocate: Server.provideAdvanced({ capability: BlobCapabilities.allocate, @@ -59,7 +66,9 @@ const createService = ({ baseURL, claimsService, contentStore, allocationStore } return ok({ size: 0 }) } - const size = await allocationStore.has(digest) ? 0 : capability.nb.blob.size + const size = (await allocationStore.has(digest)) + ? 0 + : capability.nb.blob.size await allocationStore.add(digest) return ok({ @@ -80,13 +89,16 @@ const createService = ({ baseURL, claimsService, contentStore, allocationStore } return error(new AllocatedMemoryNotWrittenError()) } - const receipt = await publishLocationCommitment({ claimsService }, { - space: capability.nb.space, - digest, - location: - /** @type {API.URI} */ - (new URL(contentKey(digest), baseURL()).toString()), - }) + const receipt = await publishLocationCommitment( + { claimsService }, + { + space: capability.nb.space, + digest, + location: + /** @type {API.URI} */ + (new URL(contentKey(digest), baseURL()).toString()), + } + ) if (receipt.out.error) { return receipt.out } @@ -117,21 +129,28 @@ export class BrowserStorageNode { return res.status === 200 ? new Uint8Array(await res.arrayBuffer()) : undefined - } + }, } - + const allocations = new Set() const allocationStore = { /** @param {API.MultihashDigest} digest */ has: async (digest) => allocations.has(contentKey(digest)), /** @param {API.MultihashDigest} digest */ - add: async (digest) => { allocations.add(contentKey(digest)) } + add: async (digest) => { + allocations.add(contentKey(digest)) + }, } const server = Server.create({ id, codec: CAR.inbound, - service: createService({ baseURL: () => baseURL, claimsService, contentStore, allocationStore }), + service: createService({ + baseURL: () => baseURL, + claimsService, + contentStore, + allocationStore, + }), // @ts-expect-error resolveDIDKey, validateAuthorization: () => ({ ok: {} }), @@ -168,7 +187,7 @@ export class StorageNode { const id = await ed25519.generate() /** @type {URL} */ let baseURL - + const content = new Map() const contentStore = { /** @param {API.MultihashDigest} digest */ @@ -179,7 +198,9 @@ export class StorageNode { * @param {API.MultihashDigest} digest * @param {Uint8Array} bytes */ - set: async (digest, bytes) => { content.set(contentKey(digest), bytes) } + set: async (digest, bytes) => { + content.set(contentKey(digest), bytes) + }, } const allocations = new Set() @@ -187,7 +208,9 @@ export class StorageNode { /** @param {API.MultihashDigest} digest */ has: async (digest) => allocations.has(contentKey(digest)), /** @param {API.MultihashDigest} digest */ - add: async (digest) => { allocations.add(contentKey(digest)) } + add: async (digest) => { + allocations.add(contentKey(digest)) + }, } const server = Server.create({ @@ -197,13 +220,13 @@ export class StorageNode { baseURL: () => baseURL, claimsService, contentStore, - allocationStore + allocationStore, }), // @ts-expect-error resolveDIDKey, validateAuthorization: () => ({ ok: {} }), }) - + const httpServer = http.createServer(async (request, response) => { try { const { pathname } = new URL(request.url ?? '/', baseURL) diff --git a/packages/upload-api/test/handlers/usage.js b/packages/upload-api/test/handlers/usage.js index e86cd02d3..ed62e06d4 100644 --- a/packages/upload-api/test/handlers/usage.js +++ b/packages/upload-api/test/handlers/usage.js @@ -18,16 +18,19 @@ export const test = { const link = await CAR.codec.link(data) const size = data.byteLength - await uploadBlob({ - connection, - issuer: alice, - audience: context.id, - with: spaceDid, - proofs: [proof], - }, { - digest: link.multihash, - bytes: data - }) + await uploadBlob( + { + connection, + issuer: alice, + audience: context.id, + with: spaceDid, + proofs: [proof], + }, + { + digest: link.multihash, + bytes: data, + } + ) const usageReportRes = await Usage.report .invoke({ diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index 5398f002e..7ee1bcde9 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -485,7 +485,9 @@ export class Client extends Base { // Remove shards if (upload.shards?.length) { await Promise.allSettled( - upload.shards.map(shard => this.capability.blob.remove(shard.multihash)) + upload.shards.map((shard) => + this.capability.blob.remove(shard.multihash) + ) ) }