From 194e17f3ea4ce40178ed18e67a28dbc8453d2778 Mon Sep 17 00:00:00 2001 From: yuudi Date: Fri, 21 Jul 2023 20:50:07 +0000 Subject: [PATCH] improve download performance --- assets/homepage.ts | 20 ++--- assets/receive.ts | 8 +- minify.sh | 1 + onesend.go | 4 + virtual-downloader.ts | 179 +++++++++++++++++++++++++++--------------- 5 files changed, 135 insertions(+), 77 deletions(-) diff --git a/assets/homepage.ts b/assets/homepage.ts index 78a103b..b20af6a 100644 --- a/assets/homepage.ts +++ b/assets/homepage.ts @@ -1,3 +1,5 @@ +/// + function humanFileSize(bytes: number, si = false, dp = 1) { const thresh = si ? 1000 : 1024; if (Math.abs(bytes) < thresh) { @@ -35,14 +37,14 @@ async function generate_aes_ctr_keys() { return { key: key, key_base64: btoa(String.fromCharCode.apply(null, key_array)) - .replace("+", "-") - .replace("/", "_") - .replace("=", ""), + .replaceAll("+", "-") + .replaceAll("/", "_") + .replaceAll("=", ""), nonce: nonce_array, nonce_base64: btoa(String.fromCharCode.apply(null, nonce_array)) - .replace("+", "-") - .replace("/", "_") - .replace("=", ""), + .replaceAll("+", "-") + .replaceAll("/", "_") + .replaceAll("=", ""), }; } @@ -77,9 +79,9 @@ async function encrypt_file_name( new Uint8Array(encrypted_filename_array) ) ) - .replace("+", "-") - .replace("/", "_") - .replace("=", ""); + .replaceAll("+", "-") + .replaceAll("/", "_") + .replaceAll("=", ""); return file_id + "." + encrypted_filename_base64; } diff --git a/assets/receive.ts b/assets/receive.ts index aba6e51..5a7f292 100644 --- a/assets/receive.ts +++ b/assets/receive.ts @@ -1,3 +1,5 @@ +/// + function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -30,9 +32,9 @@ async function recover_aes_ctr_key(key_base64: string, nonce_base64: string) { throw new Error("nonce is broken"); } let original_key_base64 = - key_base64.replace("-", "+").replace("_", "/") + "="; + key_base64.replaceAll("-", "+").replaceAll("_", "/") + "="; let original_nonce_base64 = - nonce_base64.replace("-", "+").replace("_", "/") + "="; + nonce_base64.replaceAll("-", "+").replaceAll("_", "/") + "="; let key_array = atob(original_key_base64) .split("") .map((c) => c.charCodeAt(0)); @@ -74,7 +76,7 @@ async function decrypt_file_name( padding_equals = 4 - padding_equals; } let name_encrypted_original_base64 = - name_encrypted.replace("-", "+").replace("_", "/") + + name_encrypted.replaceAll("-", "+").replaceAll("_", "/") + "=".repeat(padding_equals); let name_encrypted_array = atob(name_encrypted_original_base64) .split("") diff --git a/minify.sh b/minify.sh index 43cbd59..4476917 100644 --- a/minify.sh +++ b/minify.sh @@ -20,5 +20,6 @@ npx tsc ./assets.src/receive.ts --target es2017 npx webpack ./assets.src/receive.js -o ./dist --mode production mv dist/main.js assets/receive.js npx tsc ./virtual-downloader.ts --target es2017 +sed -i '/export {};/d' ./virtual-downloader.js npx webpack ./virtual-downloader.js -o ./dist --mode production mv dist/main.js virtual-downloader.js diff --git a/onesend.go b/onesend.go index 2e350bf..4b816ff 100644 --- a/onesend.go +++ b/onesend.go @@ -219,6 +219,10 @@ func entry() error { c.Data(200, "text/html", publicIndex) }) r.GET("/auth.html", func(c *gin.Context) { + if client != nil { + c.Redirect(302, "/") + return + } c.Header("Cache-Control", "public, max-age=604800") c.Data(200, "text/html", publicAuth) }) diff --git a/virtual-downloader.ts b/virtual-downloader.ts index f2cccc1..f308130 100644 --- a/virtual-downloader.ts +++ b/virtual-downloader.ts @@ -1,7 +1,7 @@ /// /// -export type {}; // let typescript shut up +export type {}; // make typescript shut up, this line should be deleted after transpiled declare let self: ServiceWorkerGlobalScope; const CACHE_KEY = "v1.0.1"; @@ -36,22 +36,56 @@ async function decrypt_file_part( return plain; } -self.addEventListener("install", function (event) { - event.waitUntil( - (async function () { - let cache = await caches.open(CACHE_KEY); - return cache.addAll([ - "/assets/favicon.ico", - "/", - "/assets/homepage.js", - "/assets/homepage.css", - "/s/*", - "/assets/receive.js", - "/assets/receive.css", - ]); - })() - ); -}); +class Chunker { + done = false; + private remaining: Uint8Array | undefined; + remainingSize = 0; + private reader: ReadableStreamDefaultReader; + + constructor(stream: ReadableStream, private size = 16) { + this.reader = stream.getReader(); + } + + async read(): Promise< + { done: true; value: undefined } | { done: false; value: Uint8Array } + > { + if (this.done) { + return { done: true, value: undefined }; + } + const { done, value } = await this.reader.read(); + if (done || value === undefined) { + this.done = true; + if (this.remaining === undefined) { + return { done: true, value: undefined }; + } else { + return { done: false, value: this.remaining }; + } + } + const inSize = value.byteLength + this.remainingSize; + const remainingSize = inSize % this.size; + const outSize = inSize - remainingSize; + let out: Uint8Array; + if (this.remaining !== undefined) { + out = new Uint8Array(outSize); + out.set(this.remaining); + out.set( + value.slice(0, value.byteLength - remainingSize), + this.remainingSize + ); + } else { + out = value.slice(0, value.byteLength - remainingSize); + } + + this.remainingSize = remainingSize; + if (remainingSize > 0) { + this.remaining = value.slice(value.byteLength - remainingSize); + } else { + this.remaining = undefined; + } + + return { done: false, value: out }; + } +} self.addEventListener("activate", function (event) { event.waitUntil(self.clients.claim()); @@ -63,17 +97,6 @@ self.addEventListener("message", function (event) { } }); -async function try_fetch(input, init, tries = 3) { - try { - return await fetch(input, init); - } catch (e) { - if (tries > 0) { - return try_fetch(input, init, tries - 1); - } - throw e; - } -} - self.addEventListener("fetch", function (event) { let request = event.request; let url = new URL(request.url); @@ -82,7 +105,7 @@ self.addEventListener("fetch", function (event) { } let path = url.pathname; if (path.startsWith("/s/download")) { - event.respondWith(virtual_downloading_response(path)); + event.respondWith(virtual_downloading_response(request)); return; } if (path.startsWith("/s/")) { @@ -91,7 +114,22 @@ self.addEventListener("fetch", function (event) { event.respondWith(cached_response(request)); }); -async function virtual_downloading_response(path: string) { +function rangeOf(request: Request) { + let range = request.headers.get("Range"); + if (range === null) { + return null; + } + let range_match = range.match(/^bytes=(\d+)-(\d+)$/); + if (range_match === null) { + return null; + } + let start = parseInt(range_match[1]); + let end = parseInt(range_match[2]); + return [start, end]; +} + +async function virtual_downloading_response(request: Request) { + const path = new URL(request.url).pathname; let path_list = path.split("/"); let file_path = path_list[path_list.length - 1]; let file_info = FilesData[file_path]; @@ -101,53 +139,64 @@ async function virtual_downloading_response(path: string) { statusText: "Not Found", }); } + let headers = new Headers(); + // let range = rangeOf(request); + // let start: number; + // if (range !== null) { + // start = range[0]; + // } else { + // start = 0; + // } + // if (range !== null) { + // headers.set("Range", `bytes=${range[0]}-${range[1]}`); + // } + //// TODO: handle cases when range does not start from multiple of 16 + let { abort, signal } = new AbortController(); + let response = await fetch(file_info.download_url, { headers, signal }); + let body = response.body; + if (body === null) { + return response; + } + let reader = new Chunker(body, 16); // chunk stream to size of multiple of 16 bytes let decrypted_readable_stream = new ReadableStream({ async start(controller) { - const chunk_size = 1310720; - let chunk_number = Math.ceil(file_info.file_size / chunk_size); - let fetched = 0; - let fetch_queue: Promise[] = []; - async function next_fetch() { - if (fetched >= chunk_number) { - return null; + let offset = 0; + while (true) { + let readResult = await reader.read(); + if (readResult.done) { + break; } - let i = fetched; - fetched += 1; - let start = i * chunk_size; - let end; - if (i === chunk_number - 1) { - end = file_info.file_size - 1; - } else { - end = start + chunk_size - 1; - } - let response = await try_fetch(file_info.download_url, { - headers: { Range: `bytes=${start}-${end}` }, - }); - let data = await response.arrayBuffer(); let plain = await decrypt_file_part( file_info.key, - data, + readResult.value, file_info.nonce, file_info.file_id, - start / 16 + offset / 16 ); - return new Uint8Array(plain); - } - fetch_queue.push(next_fetch()); - setTimeout(function () { - // 4 concurrent download - fetch_queue.push(next_fetch()); - fetch_queue.push(next_fetch()); - fetch_queue.push(next_fetch()); - }, 1000); - for (let j = 0; j < chunk_number; j++) { - let chunk = await fetch_queue.shift(); - controller.enqueue(chunk); - fetch_queue.push(next_fetch()); + offset += readResult.value.byteLength; + controller.enqueue(new Uint8Array(plain)); } controller.close(); }, + cancel() { + abort(); + }, }); + // let decrypted_readable_stream = body.pipeThrough( + // new TransformStream({ + // async transform(chunk, controller) { + // let plain = await decrypt_file_part( + // file_info.key, + // chunk, + // file_info.nonce, + // file_info.file_id, + // start / 16 + // ); + // start += chunk.byteLength; + // controller.enqueue(new Uint8Array(plain)); + // }, + // }) + // ); return new Response(decrypted_readable_stream, { headers: { "Content-Length": file_info.file_size,