Durable body from cache itself, #72, #73, #76

opennetwork · Sep 7, 2023 · 40b6384 · 40b6384
1 parent cc174c5
commit 40b6384
Show file tree

Hide file tree

Showing 7 changed files with 109 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -232,9 +232,11 @@ export interface DurableEvent extends DurableEventData {
     durableEventId: string
 }
 
+export type DurableBodyLike = string | DurableBody;
+
 export interface DurableRequestData extends Pick<Request, "url" | "method">, Expiring {
     headers?: Record<string, string>
-    body?: string | DurableBody;
+    body?: DurableBodyLike;
     response?: DurableResponseData;
 }
 
@@ -243,13 +245,14 @@ export interface DurableResponseCache {
 }
 
 export interface DurableBody {
-    type: "file" | "base64";
+    type: "file" | "base64" | "cache";
     value: string;
+    url?: string;
 }
 
 export interface DurableResponseData extends Pick<Response, "url" | "status" | "statusText"> {
     headers?: Record<string, string>
-    body?: string | DurableBody;
+    body?: DurableBodyLike;
 }
 
 export interface DurableRequest extends DurableRequestData {

diff --git a/src/content-index/content-index.ts b/src/content-index/content-index.ts
@@ -58,7 +58,7 @@ export class DurableContentIndex {
                 type: "content",
                 id: contentDescription.id
             }
-        })
+        });
     }
 
     async delete(id: string) {

diff --git a/src/data/durable-request/from.ts b/src/data/durable-request/from.ts
@@ -1,8 +1,9 @@
-import {DurableBody, DurableRequestData, DurableResponseData} from "./types";
+import {DurableBody, DurableBodyLike, DurableRequestData, DurableResponseData} from "./types";
 import {isDurableBody} from "./is";
 import {ok} from "../../is";
 import {getFile, readFile, save} from "../file";
 import {v4} from "uuid";
+import {caches} from "../../fetch";
 
 export async function fromMaybeDurableBody(body: unknown): Promise<RequestInit["body"]> {
     if (typeof body === "string") {
@@ -27,6 +28,14 @@ export async function fromDurableBody(body: DurableBody): Promise<RequestInit["b
     if (body.type === "base64") {
         return Buffer.from(body.value, body.type);
     }
+    if (body.type === "cache") {
+        const { url, value: cacheName } = body;
+        ok(url, "Expected url to be provided to resolve cache body");
+        const cache = await caches.open(cacheName);
+        const match = await cache.match(url);
+        ok(match, "Expected match from cache for body");
+        return match.blob();
+    }
     ok(body.type === "file", `Unknown body type ${body.type}`);
     const file = await getFile(body.value);
     ok(file, `Expected to find file ${file.fileId}`);
@@ -63,27 +72,32 @@ export async function fromDurableResponse(durableResponse: DurableResponseData)
 
 export interface FromRequestResponseOptions {
     fileName?: string;
+    body?: DurableBodyLike;
 }
 
 export async function fromRequestResponse(request: Request, response: Response, options?: FromRequestResponseOptions): Promise<DurableRequestData> {
     const clonedResponse = response.clone();
 
-    let body: DurableResponseData["body"];
+    let body: DurableBodyLike;
 
-    // TODO detect string based contentTypes
-    const contentType = response.headers.get("Content-Type");
-    if (contentType === "text/html" || contentType === "text/plain" || contentType?.startsWith("application/json") || contentType === "application/javascript") {
-        body = await clonedResponse.text();
+    if (options.body) {
+        body = options.body;
     } else {
-        // TODO warning, we might mislink some of these files...
-        const file = await save({
-            fileName: options?.fileName || v4(),
-            contentType
-        }, await clonedResponse.blob());
-        body = {
-            type: "file",
-            value: file.fileId
-        };
+        // TODO detect string based contentTypes
+        const contentType = response.headers.get("Content-Type");
+        if (contentType === "text/html" || contentType === "text/plain" || contentType?.startsWith("application/json") || contentType === "application/javascript") {
+            body = await clonedResponse.text();
+        } else {
+            // TODO warning, we might mislink some of these files...
+            const file = await save({
+                fileName: options?.fileName || v4(),
+                contentType
+            }, await clonedResponse.blob());
+            body = {
+                type: "file",
+                value: file.fileId
+            };
+        }
     }
 
     const durableResponse: DurableResponseData = {

diff --git a/src/data/durable-request/is.ts b/src/data/durable-request/is.ts
@@ -7,7 +7,8 @@ export function isDurableBody(value: unknown): value is DurableBody {
         typeof value.type === "string" &&
         (
             value.type === "file" ||
-            value.type === "base64"
+            value.type === "base64" ||
+            value.type === "cache"
         ) &&
         typeof value.value === "string"
     )

diff --git a/src/data/durable-request/types.ts b/src/data/durable-request/types.ts
@@ -1,8 +1,10 @@
 import {Expiring} from "../expiring";
 
+export type DurableBodyLike = string | DurableBody;
+
 export interface DurableRequestData extends Pick<Request, "url" | "method">, Expiring {
     headers?: Record<string, string>
-    body?: string | DurableBody;
+    body?: DurableBodyLike;
     response?: DurableResponseData;
 }
 
@@ -11,13 +13,14 @@ export interface DurableResponseCache {
 }
 
 export interface DurableBody {
-    type: "file" | "base64";
+    type: "file" | "base64" | "cache";
     value: string;
+    url?: string;
 }
 
 export interface DurableResponseData extends Pick<Response, "url" | "status" | "statusText"> {
     headers?: Record<string, string>
-    body?: string | DurableBody;
+    body?: DurableBodyLike;
 }
 
 export interface DurableRequest extends DurableRequestData {

diff --git a/src/data/file/disk.ts b/src/data/file/disk.ts
@@ -1,20 +1,32 @@
 import {FileData} from "./types";
-import {writeFile, mkdir, stat, readFile, unlink} from "fs/promises";
-import {dirname, join} from "node:path";
+import {writeFile, mkdir, stat, readFile, unlink, readdir, rmdir} from "fs/promises";
+import {dirname, join, resolve} from "node:path";
 import {getRemoteSourceKey, getRemoteSourcePrefix} from "./source";
 
-function getFullPath(file: FileData) {
-    const path = getRemoteSourceKey(file.source, "store") ?? ".cache/.store";
+function getDirectoryBase(file: FileData) {
+    return resolve(getRemoteSourceKey(file.source, "store") ?? ".cache/.store");
+}
+
+function getDirectoryPrefix(file: FileData) {
     let prefix = file.source ? getRemoteSourcePrefix(file.source) : "";
-    if (prefix && !prefix.endsWith("/")) {
+    if (!prefix) {
+        return "";
+    }
+    if (!prefix.endsWith("/")) {
         prefix = `${prefix}/`;
     }
+    return prefix;
+}
+
+function getFullPath(file: FileData) {
+    const path = getDirectoryBase(file);
+    const prefix = getDirectoryPrefix(file);
     const key = `${prefix}${file.fileName}`
     let fullPath = join(path, key);
     if (!fullPath.startsWith("/")) {
         fullPath = join(process.cwd(), fullPath);
     }
-    return fullPath;
+    return resolve(fullPath);
 }
 
 export async function saveToDisk(file: FileData, contents: Buffer | Blob): Promise<Partial<FileData>> {
@@ -62,4 +74,30 @@ export async function unlinkFromDisk(file: FileData) {
         return;
     }
     await unlink(fullPath);
-}
+    await unlinkDirectoryIfNeeded();
+
+    async function unlinkDirectoryIfNeeded() {
+        const base = resolve(getDirectoryBase(file));
+        const absolute = resolve(fullPath);
+        if (!absolute.startsWith(base)) {
+            // We shouldn't mess with anything
+            // Outside of normal cache root
+            return;
+        }
+        let directory = dirname(absolute);
+        while (directory.startsWith(base) && directory !== base && directory !== `${base}/`) {
+            const paths = await readdir(directory);
+            if (paths.length) {
+                return;
+            }
+            await rmdir(directory);
+            directory = dirname(directory);
+        }
+    }
+
+
+
+
+
+
+}
diff --git a/src/fetch/events.ts b/src/fetch/events.ts
@@ -1,4 +1,4 @@
-import {DurableEventData, DurableRequest, DurableRequestData, setDurableRequestForEvent} from "../data";
+import {DurableBody, DurableEventData, DurableRequest, DurableRequestData, setDurableRequestForEvent} from "../data";
 import {dispatchEvent, on} from "../events";
 import {dispatcher} from "../events/schedule/schedule";
 import {defer} from "@virtualstate/promise";
@@ -136,23 +136,33 @@ export const removeFetchDispatcherFunction = dispatcher(FETCH, async (event, dis
             waitUntil
         });
         const response = await handled;
-        if (isDurableFetchEventCache(event.cache)) {
-            const { name, always } = event.cache;
-            if (response.ok || always) {
-                const store = await caches.open(name);
-                await store.put(request, response);
-            }
-        }
         let durableEventDispatch: DurableEventData;
         if (event.dispatch) {
             durableEventDispatch = {
                 durableEventId: v4(),
                 ...event.dispatch
             };
         }
+        const isRetain = durableEventDispatch || (event.durableEventId && event.retain !== false);
+        let body: DurableBody;
+        const cache = isDurableFetchEventCache(event.cache) ? event.cache : (isRetain ? { name: FETCH } : undefined);
+        if (cache) {
+            const { name, always } = cache;
+            if (response.ok || always) {
+                const store = await caches.open(name);
+                await store.put(request, response);
+                body = {
+                    type: "cache",
+                    value: name,
+                    url: request.url
+                };
+            }
+        }
         let durableRequest: DurableRequest;
-        if (durableEventDispatch || (event.durableEventId && event.retain !== false)) {
-            const durableRequestData = await fromRequestResponse(request, response);
+        if (isRetain) {
+            const durableRequestData = await fromRequestResponse(request, response, {
+                body
+            });
             durableRequest = await setDurableRequestForEvent(durableRequestData, durableEventDispatch || event);
             if (durableEventDispatch) {
                 const { response, ...request } = durableRequest;