diff --git a/README.md b/README.md index 1caa8e7..9a93e98 100644 --- a/README.md +++ b/README.md @@ -222,16 +222,41 @@ export interface DurableEventTypeData extends UnknownEvent { export interface DurableEventData extends Record, DurableEventTypeData, Expiring { timeStamp?: number; - eventId?: string; + durableEventId?: string; schedule?: DurableEventSchedule; retain?: boolean; virtual?: boolean; } export interface DurableEvent extends DurableEventData { - eventId: string; + durableEventId: string } +export interface DurableRequestData extends Pick, Expiring { + headers?: Record + body?: string; +} + +export interface DurableResponseData extends Pick { + headers?: Record + body?: string; +} + +export interface DurableRequest extends DurableRequestData { + durableRequestId: string; + response?: DurableResponseData; + createdAt: string; + updatedAt: string; +} + +export interface RequestQueryInfo extends DurableRequestData { + +} + +export type RequestQuery = RequestQueryInfo | RequestInfo | URL + +export type PartialDurableRequest = DurableRequestData & Partial; + export interface Expiring { expiresAt?: string; } diff --git a/src/background/index.ts b/src/background/index.ts index e772e62..13463c3 100644 --- a/src/background/index.ts +++ b/src/background/index.ts @@ -13,6 +13,7 @@ export interface BackgroundInput extends Record { export interface BackgroundQuery extends Record { event?: string; eventId?: string; + durableEventId?: string; eventTimeStamp?: string | `${number}` | number; cron?: string; seed?: string; @@ -71,12 +72,14 @@ async function backgroundScheduleWithOptions(input: BackgroundInput) { } else if (event) { const { eventId, + durableEventId: givenDurableEventId, eventTimeStamp: timeStamp } = input.query; - if (eventId) { + const durableEventId = givenDurableEventId || eventId; + if (durableEventId) { options.event = { type: event, - eventId + durableEventId } if (isNumberString(timeStamp)) { options.event.timeStamp = +timeStamp; diff --git a/src/client/interface.readonly.ts b/src/client/interface.readonly.ts index 25f2a16..dd1dd9d 100644 --- a/src/client/interface.readonly.ts +++ b/src/client/interface.readonly.ts @@ -7,6 +7,7 @@ export * from "../data/authorisation/types" export * from "../data/authorisation-notification/types" export * from "../data/change/types" export * from "../data/durable-event/types" +export * from "../data/durable-request/types" export * from "../data/expiring/types" export * from "../data/file/types" export * from "../data/form-meta/types" diff --git a/src/data/data.ts b/src/data/data.ts index ddf6303..e90b0a9 100644 --- a/src/data/data.ts +++ b/src/data/data.ts @@ -31,4 +31,5 @@ export * from "./appointment"; export * from "./change"; export * from "./membership"; export * from "./service"; -export * from "./durable-event"; \ No newline at end of file +export * from "./durable-event"; +export * from "./durable-request"; \ No newline at end of file diff --git a/src/data/durable-event/add-durable-event.ts b/src/data/durable-event/add-durable-event.ts index a4ce3ab..d9d5494 100644 --- a/src/data/durable-event/add-durable-event.ts +++ b/src/data/durable-event/add-durable-event.ts @@ -5,12 +5,12 @@ import {ok} from "../../is"; export async function addDurableEvent(event: DurableEventData) { const createdAt = new Date().toISOString(); - const eventId = event.eventId || v4(); + const eventId = event.durableEventId || v4(); const durable: DurableEvent = { timeStamp: Date.now(), createdAt, updatedAt: createdAt, - eventId, + durableEventId: eventId, ...event }; ok(!durable.virtual, "Cannot store virtual event"); diff --git a/src/data/durable-event/delete-durable-event.ts b/src/data/durable-event/delete-durable-event.ts index b183604..feb8b41 100644 --- a/src/data/durable-event/delete-durable-event.ts +++ b/src/data/durable-event/delete-durable-event.ts @@ -3,6 +3,6 @@ import {DurableEvent, DurableEventData} from "./types"; export function deleteDurableEvent(event: DurableEventData) { const store = getDurableEventStore(event); - if (!event.eventId) return undefined; - return store.delete(event.eventId); + if (!event.durableEventId) return undefined; + return store.delete(event.durableEventId); } \ No newline at end of file diff --git a/src/data/durable-event/get-durable-event.ts b/src/data/durable-event/get-durable-event.ts index f3dfe46..d57c652 100644 --- a/src/data/durable-event/get-durable-event.ts +++ b/src/data/durable-event/get-durable-event.ts @@ -2,7 +2,7 @@ import {DurableEventData} from "./types"; import {getDurableEventStore} from "./store"; export function getDurableEvent(event: DurableEventData) { - if (!event.eventId) return undefined; + if (!event.durableEventId) return undefined; const store = getDurableEventStore(event); - return store.get(event.eventId); + return store.get(event.durableEventId); } \ No newline at end of file diff --git a/src/data/durable-event/types.ts b/src/data/durable-event/types.ts index 24f7a1f..a83f19e 100644 --- a/src/data/durable-event/types.ts +++ b/src/data/durable-event/types.ts @@ -20,12 +20,12 @@ export interface DurableEventTypeData extends UnknownEvent { export interface DurableEventData extends Record, DurableEventTypeData, Expiring { timeStamp?: number; - eventId?: string; + durableEventId?: string; schedule?: DurableEventSchedule; retain?: boolean; virtual?: boolean; } export interface DurableEvent extends DurableEventData { - eventId: string; + durableEventId: string } \ No newline at end of file diff --git a/src/data/durable-request/delete-durable-request.ts b/src/data/durable-request/delete-durable-request.ts new file mode 100644 index 0000000..330d1f6 --- /dev/null +++ b/src/data/durable-request/delete-durable-request.ts @@ -0,0 +1,6 @@ +import {getDurableRequestStore} from "./store"; + +export function deleteDurableRequest(durableRequestId: string) { + const store = getDurableRequestStore(); + return store.delete(durableRequestId); +} \ No newline at end of file diff --git a/src/data/durable-request/from.ts b/src/data/durable-request/from.ts new file mode 100644 index 0000000..7ca7911 --- /dev/null +++ b/src/data/durable-request/from.ts @@ -0,0 +1,75 @@ +import {DurableRequest, DurableRequestData, DurableResponseData} from "./types"; + +export function fromDurableRequest(durableRequest: DurableRequestData, getOrigin?: () => string) { + const { url, method, headers, body } = durableRequest; + return new Request( + new URL(url, getOrigin?.()), + { + method, + headers, + body + } + ); +} + +export function fromDurableResponse(durableResponse: DurableResponseData) { + const { body, statusText, status, headers } = durableResponse; + return new Response( + body, + { + status, + statusText, + headers + } + ); +} + +export async function fromRequestResponse(request: Request, response: Response) { + const clonedResponse = response.clone(); + + const durableResponse: DurableResponseData = { + headers: getResponseHeadersObject(), + status: response.status, + statusText: response.statusText, + // response.url is empty if it was constructed manually + // Should be same value anyway... + url: response.url || request.url, + // TODO investigate non string responses and storage + // we could just use something like + // await save(`fetch/cache/${durableRequestId}`, Buffer.from(await clonedResponse.arrayBuffer())) + body: await clonedResponse.text() + }; + + const createdAt = new Date().toISOString(); + const { method, url } = request; + + return { + durableRequestId: `${method}:${url}`, + method, + url, + response: durableResponse, + createdAt, + updatedAt: createdAt + }; + + function getResponseHeadersObject() { + const headers = new Headers(response.headers); + // Not sure if we ever get this header in node fetch + // https://developer.mozilla.org/en-US/docs/Web/API/Cache#cookies_and_cache_objects + // Maybe these headers were constructed by a user though + headers.delete("Set-Cookie"); + return getHeadersObject(headers); + } + +} + +function getHeadersObject(headers?: Headers) { + const output: Record = {}; + if (!headers) { + return output; + } + headers.forEach((value, key) => { + output[key] = value; + }) + return output; +} \ No newline at end of file diff --git a/src/data/durable-request/get-durable-request.ts b/src/data/durable-request/get-durable-request.ts new file mode 100644 index 0000000..9295aa6 --- /dev/null +++ b/src/data/durable-request/get-durable-request.ts @@ -0,0 +1,17 @@ +import {getDurableRequestStore} from "./store"; +import {DurableEvent} from "../durable-event"; + +export function getDurableRequest(durableRequestId: string) { + const store = getDurableRequestStore(); + return store.get(durableRequestId); +} + +export function getDurableRequestIdForEvent(event: DurableEvent) { + return `${event.type}:request:${event.durableEventId}`; +} + +export function getDurableRequestForEvent(event: DurableEvent) { + return getDurableRequest( + getDurableRequestIdForEvent(event) + ); +} \ No newline at end of file diff --git a/src/data/durable-request/index.ts b/src/data/durable-request/index.ts new file mode 100644 index 0000000..4d7533d --- /dev/null +++ b/src/data/durable-request/index.ts @@ -0,0 +1,7 @@ +export * from "./types"; +export * from "./store"; +export * from "./set-durable-request"; +export * from "./delete-durable-request"; +export * from "./get-durable-request"; +export * from "./list-durable-requests"; +export * from "./from"; \ No newline at end of file diff --git a/src/data/durable-request/list-durable-requests.ts b/src/data/durable-request/list-durable-requests.ts new file mode 100644 index 0000000..1cd28a7 --- /dev/null +++ b/src/data/durable-request/list-durable-requests.ts @@ -0,0 +1,6 @@ +import {getDurableRequestStore} from "./store"; + +export function listDurableRequests() { + const store = getDurableRequestStore(); + return store.values(); +} \ No newline at end of file diff --git a/src/data/durable-request/set-durable-request.ts b/src/data/durable-request/set-durable-request.ts new file mode 100644 index 0000000..6629f02 --- /dev/null +++ b/src/data/durable-request/set-durable-request.ts @@ -0,0 +1,27 @@ +import {v4} from "uuid"; +import {DurableRequest, PartialDurableRequest} from "./types"; +import {getDurableRequestStore} from "./store"; +import {DurableEvent} from "../durable-event"; +import {getDurableRequestIdForEvent} from "./get-durable-request"; + + +export async function setDurableRequest(data: PartialDurableRequest) { + const createdAt = new Date().toISOString(); + const durableRequestId = data.durableRequestId || v4(); + const durableRequest: DurableRequest = { + ...data, + createdAt, + updatedAt: createdAt, + durableRequestId, + }; + const store = getDurableRequestStore(); + await store.set(durableRequestId, durableRequest); + return durableRequest; +} + +export function setDurableRequestForEvent(data: PartialDurableRequest, event: DurableEvent) { + return setDurableRequest({ + ...data, + durableRequestId: getDurableRequestIdForEvent(event) + }); +} \ No newline at end of file diff --git a/src/data/durable-request/store.ts b/src/data/durable-request/store.ts new file mode 100644 index 0000000..b838994 --- /dev/null +++ b/src/data/durable-request/store.ts @@ -0,0 +1,16 @@ +import {getExpiringStore} from "../expiring-kv"; +import {DurableRequest} from "./types"; + +const STORE_NAME = "request" as const; + +export interface DurableRequestStoreOptions { + name?: string; + prefix?: string; +} + +export function getDurableRequestStore({ name, prefix }: DurableRequestStoreOptions = {}) { + return getExpiringStore(name || STORE_NAME, { + counter: false, + prefix + }); +} \ No newline at end of file diff --git a/src/fetch/types.ts b/src/data/durable-request/types.ts similarity index 58% rename from src/fetch/types.ts rename to src/data/durable-request/types.ts index 387b615..59c4a24 100644 --- a/src/fetch/types.ts +++ b/src/data/durable-request/types.ts @@ -1,4 +1,6 @@ -export interface DurableRequestData extends Pick { +import {Expiring} from "../expiring"; + +export interface DurableRequestData extends Pick, Expiring { headers?: Record body?: string; } @@ -12,4 +14,13 @@ export interface DurableRequest extends DurableRequestData { durableRequestId: string; response?: DurableResponseData; createdAt: string; -} \ No newline at end of file + updatedAt: string; +} + +export interface RequestQueryInfo extends DurableRequestData { + +} + +export type RequestQuery = RequestQueryInfo | RequestInfo | URL + +export type PartialDurableRequest = DurableRequestData & Partial; \ No newline at end of file diff --git a/src/events/dispatch/index.ts b/src/events/dispatch/index.ts index 8588d60..865418e 100644 --- a/src/events/dispatch/index.ts +++ b/src/events/dispatch/index.ts @@ -34,8 +34,8 @@ export async function onDispatchEvent(event: UnknownEvent) { virtual: true, }; // If the instance has no id, give it one - if (!dispatching.eventId) { - dispatching.eventId = v4(); + if (!dispatching.durableEventId) { + dispatching.durableEventId = v4(); } await dispatchScheduledDurableEvents({ event: dispatching diff --git a/src/events/schedule/dispatch-scheduled.ts b/src/events/schedule/dispatch-scheduled.ts index 6481f2f..3e856f4 100644 --- a/src/events/schedule/dispatch-scheduled.ts +++ b/src/events/schedule/dispatch-scheduled.ts @@ -47,7 +47,7 @@ export async function dispatchScheduledDurableEvents(options: BackgroundSchedule } async function dispatchScheduledEvent(event: DurableEventData) { - if (event.eventId) { + if (event.durableEventId) { const schedule = (await getDurableEvent(event)) ?? (event.virtual ? event : undefined); if (!isMatchingSchedule(schedule)) { return; @@ -66,7 +66,7 @@ export async function dispatchScheduledDurableEvents(options: BackgroundSchedule } async function dispatchEvent(event: DurableEventData) { - const done = await lock(`dispatchEvent:${event.type}:${event.eventId || "no-event-id"}`); + const done = await lock(`dispatchEvent:${event.type}:${event.durableEventId || "no-event-id"}`); // TODO detect if this event tries to dispatch again try { await dispatchEventToHandlers(event); diff --git a/src/events/schedule/event.ts b/src/events/schedule/event.ts index 4534526..2a0f824 100644 --- a/src/events/schedule/event.ts +++ b/src/events/schedule/event.ts @@ -15,7 +15,7 @@ export async function dispatchEvent(event: DurableEventData) { let durable = event; if (!event.virtual) { - if (!event.eventId || !(await hasDurableEvent(event))) { + if (!event.durableEventId || !(await hasDurableEvent(event))) { durable = await addDurableEvent(event); } } @@ -30,7 +30,7 @@ export async function dispatchEvent(event: DurableEventData) { await background({ query: { event: durable.type, - eventId: durable.eventId + eventId: durable.durableEventId }, quiet: true }); diff --git a/src/events/schedule/qstash.ts b/src/events/schedule/qstash.ts index 4cb44cf..392b3b8 100644 --- a/src/events/schedule/qstash.ts +++ b/src/events/schedule/qstash.ts @@ -21,8 +21,8 @@ interface ScheduleMeta { function getMetaStore(event: DurableEventData) { - ok(event.eventId, "Expected eventId"); - return getDurableEventStore(event).meta(event.eventId); + ok(event.durableEventId, "Expected eventId"); + return getDurableEventStore(event).meta(event.durableEventId); } export async function dispatchQStash(event: DurableEventData) { @@ -84,7 +84,7 @@ export async function dispatchQStash(event: DurableEventData) { } const background: BackgroundQuery = { event: event.type, - eventId: event.eventId, + eventId: event.durableEventId, eventTimeStamp: event.timeStamp }; const response = await fetch( diff --git a/src/events/schedule/schedule.ts b/src/events/schedule/schedule.ts index 84e2c46..dbb9e50 100644 --- a/src/events/schedule/schedule.ts +++ b/src/events/schedule/schedule.ts @@ -77,8 +77,8 @@ export function getScheduledFunctionCorrelation(options: ScheduledOptions !seen.has(event)); if (!yieldingEvents.length) continue; for (const event of yieldingEvents) { - if (!event.eventId) { + if (!event.durableEventId) { // Provide it or have it mutated... - event.eventId = v4(); + event.durableEventId = v4(); } if (!event.virtual) { // Provide it or have it mutated... diff --git a/src/fetch/cache.ts b/src/fetch/cache.ts index a8f8dc2..2a3b4bb 100644 --- a/src/fetch/cache.ts +++ b/src/fetch/cache.ts @@ -1,9 +1,17 @@ -import {getKeyValueStore, KeyValueStore} from "../data"; -import {DurableRequest, DurableRequestData, DurableResponseData} from "./types"; +import { + DurableRequest, + DurableRequestData, + DurableResponseData, + getKeyValueStore, + KeyValueStore, + getDurableRequestStore as getBaseRequestStore, + RequestQuery +} from "../data"; import {ok} from "../is"; import {HeaderList} from "http-header-list"; import {getOrigin} from "../listen"; import {getConfig} from "../config"; +import {fromDurableRequest, fromDurableResponse, fromRequestResponse} from "../data/durable-request/from"; export interface DurableCacheStorageConfig { getDurableCacheStorageOrigin?(): string @@ -42,11 +50,7 @@ function getRequestQueryURL(requestQuery: RequestQuery, getOrigin: () => string) return requestQuery.url; } -export interface RequestQueryInfo extends DurableRequestData { -} - -export type RequestQuery = RequestQueryInfo | RequestInfo | URL // https://w3c.github.io/ServiceWorker/#query-cache function isQueryCacheMatch(requestQuery: RequestQuery | undefined, request: DurableRequest, options?: CacheQueryOptions) { @@ -132,21 +136,7 @@ function getCacheURLString(string: string, options?: CacheQueryOptions) { return instance.toString(); } -function fromDurableRequest(durableRequest: DurableRequestData, getOrigin: () => string) { - const { url, ...init } = durableRequest; - return new Request( - new URL(url, getOrigin()), - init - ); -} -function fromDurableResponse(durableResponse: DurableResponseData) { - const { body, ...init } = durableResponse; - return new Response( - body, - init - ); -} function getRequestQueryHeaders(requestQuery: RequestQuery) { if (typeof requestQuery === "string" || requestQuery instanceof URL) { @@ -155,17 +145,6 @@ function getRequestQueryHeaders(requestQuery: RequestQuery) { return new Headers(requestQuery.headers); } -function getHeadersObject(headers?: Headers) { - const output: Record = {}; - if (!headers) { - return output; - } - headers.forEach((value, key) => { - output[key] = value; - }) - return output; -} - function assertVaryValid(vary: string) { if (!vary) { return true; @@ -260,35 +239,22 @@ export class DurableCache implements Cache { const requestStore = getDurableRequestStore(this.name); ok(!response.bodyUsed); - assertVaryValid(response.headers.get("Vary")); - const clonedResponse = response.clone(); - const method = getRequestMethod(); - ok(method === "GET" || method === "HEAD", "Requests that aren't GET or HEAD will not be matchable") - const durableResponse: DurableResponseData = { - headers: getResponseHeadersObject(), - status: response.status, - statusText: response.statusText, - url: response.url, - // TODO investigate non string responses and storage - // we could just use something like - // await save(`fetch/cache/${durableRequestId}`, Buffer.from(await clonedResponse.arrayBuffer())) - body: await clonedResponse.text() - }; - const cacheUrl = getCacheURLString(url); - const durableRequest: DurableRequest = { - durableRequestId: `${method}:${cacheUrl}`, + const clonedRequest = new Request(cacheUrl, { method, - url: cacheUrl, - response: durableResponse, - createdAt: new Date().toISOString() - }; + headers: getRequestQueryHeaders(requestQuery) + }); + + const durableRequest = await fromRequestResponse( + clonedRequest, + response + ); await requestStore.set(durableRequest.durableRequestId, durableRequest); @@ -298,14 +264,6 @@ export class DurableCache implements Cache { } return requestQuery.method; } - - function getResponseHeadersObject() { - const headers = new Headers(response.headers); - // Not sure if we ever get this header in node fetch - // https://developer.mozilla.org/en-US/docs/Web/API/Cache#cookies_and_cache_objects - headers.delete("Set-Cookie"); - return getHeadersObject(headers); - } } async keys(requestQuery?: RequestInfo | URL, options?: CacheQueryOptions): Promise> { @@ -318,15 +276,26 @@ export class DurableCache implements Cache { } -function getDurableCacheStore(prefix?: string) { - return getKeyValueStore(`fetch:cache`, { - counter: false, - prefix +const CACHE_STORE_NAME = "fetch:cache"; + +function getDurableRequestStore(name: string) { + return getBaseRequestStore({ + name: CACHE_STORE_NAME, + prefix: `${name}:request`, }); } -export function getDurableRequestStore(name: string) { - return getDurableCacheStore(`${name}:request`); +interface DurableCacheReference { + cacheName: string; + createdAt: string; + lastOpenedAt: string; +} + +function getDurableCacheReferenceStore() { + return getKeyValueStore(CACHE_STORE_NAME, { + prefix: "reference", + counter: false + }); } async function firstNext(iterable: AsyncIterable): Promise> { @@ -359,12 +328,6 @@ async function * matchDurableRequests(cacheName: string, requestQuery?: RequestQ } } -interface DurableCacheReference { - cacheName: string; - createdAt: string; - lastOpenedAt: string; -} - export interface DurableCacheStorageOptions { url(): string; } @@ -378,7 +341,7 @@ export class DurableCacheStorage implements CacheStorage { constructor(options: DurableCacheStorageOptions) { this.url = options.url this.caches = new Map(); - this.store = getDurableCacheStore() + this.store = getDurableCacheReferenceStore(); } async open(cacheName: string): Promise { diff --git a/src/fetch/events.ts b/src/fetch/events.ts index 1a8ee11..0583bdd 100644 --- a/src/fetch/events.ts +++ b/src/fetch/events.ts @@ -1,9 +1,9 @@ -import {DurableEventData, UnknownEvent} from "../data"; +import {setDurableRequest, DurableEventData, DurableRequestData} from "../data"; import {on} from "../events"; import {dispatcher} from "../events/schedule/schedule"; import {defer} from "@virtualstate/promise"; import {isLike, isPromise, isSignalled, ok} from "../is"; -import {DurableRequestData} from "./types"; +import {fromDurableRequest, fromRequestResponse} from "../data/durable-request/from"; const FETCH = "fetch" as const; type ScheduleFetchEventType = typeof FETCH; @@ -98,14 +98,7 @@ export const removeFetchDispatcherFunction = dispatcher(FETCH, async (event, dis wait, waitUntil } = createWaitUntil(); - const { url, ...init } = event.request; - const request = new Request( - url, - { - ...init, - signal - } - ); + const request = fromDurableRequest(event.request); try { await dispatch({ ...event, @@ -116,11 +109,11 @@ export const removeFetchDispatcherFunction = dispatcher(FETCH, async (event, dis waitUntil }); const response = await handled; - - // no response usage - // we don't care if its resolved etc - void response; - + const durableRequest = await fromRequestResponse(request, response); + await setDurableRequest({ + ...durableRequest, + durableRequestId: `${event.type}:request:${event.durableEventId}` + }); } catch (error) { if (!signal.aborted) { controller?.abort(error); diff --git a/src/react/server/paths/durable-event/list.tsx b/src/react/server/paths/durable-event/list.tsx index 585bf6e..86f3e5c 100644 --- a/src/react/server/paths/durable-event/list.tsx +++ b/src/react/server/paths/durable-event/list.tsx @@ -31,7 +31,7 @@ export async function submit(request: FastifyRequest) { if (request.body?.dispatch) { const event = await getDurableEvent({ type: "dispatch", - eventId: request.body.dispatch + durableEventId: request.body.dispatch }); if (event) { await dispatchEvent(event); @@ -47,12 +47,12 @@ export function ListDurableEvents() { {!isUnauthenticated ? Schedule Event : undefined}
{events.map(event => ( -
+
{event.dispatch.type}
{ !isUnauthenticated ? (
- + diff --git a/src/tests/schedule/dispatch.ts b/src/tests/schedule/dispatch.ts index 60ad3da..352e070 100644 --- a/src/tests/schedule/dispatch.ts +++ b/src/tests/schedule/dispatch.ts @@ -71,14 +71,14 @@ import {getDurableEvent} from "../../data"; { const type = v4(); - const eventId = v4(); + const durableEventId = v4(); const fn = spy(); const value = v4() const initial = { type, - eventId, + durableEventId, value }; ok(!await getDurableEvent(initial)); @@ -92,7 +92,7 @@ import {getDurableEvent} from "../../data"; await dispatchScheduledDurableEvents({ event: { type, - eventId + durableEventId } }); @@ -102,7 +102,7 @@ import {getDurableEvent} from "../../data"; const [[event]] = fn.args; ok(event); - ok(event.eventId === eventId); + ok(event.durableEventId === durableEventId); ok(event.value === value); const schedule = await getDurableEvent(event);