Skip to content

Commit

Permalink
improve download performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@yuudi
yuudi committed Jul 21, 2023
1 parent 9532cd5 commit 194e17f
Show file tree
Hide file tree
Showing 5 changed files with 135 additions and 77 deletions.
20 changes: 11 additions & 9 deletions assets/homepage.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
///<reference lib="es2021" />

function humanFileSize(bytes: number, si = false, dp = 1) {
const thresh = si ? 1000 : 1024;
if (Math.abs(bytes) < thresh) {
Expand Down Expand Up @@ -35,14 +37,14 @@ async function generate_aes_ctr_keys() {
return {
key: key,
key_base64: btoa(String.fromCharCode.apply(null, key_array))
.replace("+", "-")
.replace("/", "_")
.replace("=", ""),
.replaceAll("+", "-")
.replaceAll("/", "_")
.replaceAll("=", ""),
nonce: nonce_array,
nonce_base64: btoa(String.fromCharCode.apply(null, nonce_array))
.replace("+", "-")
.replace("/", "_")
.replace("=", ""),
.replaceAll("+", "-")
.replaceAll("/", "_")
.replaceAll("=", ""),
};
}

Expand Down Expand Up @@ -77,9 +79,9 @@ async function encrypt_file_name(
new Uint8Array(encrypted_filename_array)
)
)
.replace("+", "-")
.replace("/", "_")
.replace("=", "");
.replaceAll("+", "-")
.replaceAll("/", "_")
.replaceAll("=", "");
return file_id + "." + encrypted_filename_base64;
}

Expand Down
8 changes: 5 additions & 3 deletions assets/receive.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
///<reference lib="es2021" />

function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
Expand Down Expand Up @@ -30,9 +32,9 @@ async function recover_aes_ctr_key(key_base64: string, nonce_base64: string) {
throw new Error("nonce is broken");
}
let original_key_base64 =
key_base64.replace("-", "+").replace("_", "/") + "=";
key_base64.replaceAll("-", "+").replaceAll("_", "/") + "=";
let original_nonce_base64 =
nonce_base64.replace("-", "+").replace("_", "/") + "=";
nonce_base64.replaceAll("-", "+").replaceAll("_", "/") + "=";
let key_array = atob(original_key_base64)
.split("")
.map((c) => c.charCodeAt(0));
Expand Down Expand Up @@ -74,7 +76,7 @@ async function decrypt_file_name(
padding_equals = 4 - padding_equals;
}
let name_encrypted_original_base64 =
name_encrypted.replace("-", "+").replace("_", "/") +
name_encrypted.replaceAll("-", "+").replaceAll("_", "/") +
"=".repeat(padding_equals);
let name_encrypted_array = atob(name_encrypted_original_base64)
.split("")
Expand Down
1 change: 1 addition & 0 deletions minify.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,6 @@ npx tsc ./assets.src/receive.ts --target es2017
npx webpack ./assets.src/receive.js -o ./dist --mode production
mv dist/main.js assets/receive.js
npx tsc ./virtual-downloader.ts --target es2017
sed -i '/export {};/d' ./virtual-downloader.js
npx webpack ./virtual-downloader.js -o ./dist --mode production
mv dist/main.js virtual-downloader.js
4 changes: 4 additions & 0 deletions onesend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,10 @@ func entry() error {
c.Data(200, "text/html", publicIndex)
})
r.GET("/auth.html", func(c *gin.Context) {
if client != nil {
c.Redirect(302, "/")
return
}
c.Header("Cache-Control", "public, max-age=604800")
c.Data(200, "text/html", publicAuth)
})
Expand Down
179 changes: 114 additions & 65 deletions virtual-downloader.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/// <reference lib="webworker" />
/// <reference no-default-lib="true"/>

export type {}; // let typescript shut up
export type {}; // make typescript shut up, this line should be deleted after transpiled
declare let self: ServiceWorkerGlobalScope;

const CACHE_KEY = "v1.0.1";
Expand Down Expand Up @@ -36,22 +36,56 @@ async function decrypt_file_part(
return plain;
}

self.addEventListener("install", function (event) {
event.waitUntil(
(async function () {
let cache = await caches.open(CACHE_KEY);
return cache.addAll([
"/assets/favicon.ico",
"/",
"/assets/homepage.js",
"/assets/homepage.css",
"/s/*",
"/assets/receive.js",
"/assets/receive.css",
]);
})()
);
});
class Chunker {
done = false;
private remaining: Uint8Array | undefined;
remainingSize = 0;
private reader: ReadableStreamDefaultReader<Uint8Array>;

constructor(stream: ReadableStream<Uint8Array>, private size = 16) {
this.reader = stream.getReader();
}

async read(): Promise<
{ done: true; value: undefined } | { done: false; value: Uint8Array }
> {
if (this.done) {
return { done: true, value: undefined };
}
const { done, value } = await this.reader.read();
if (done || value === undefined) {
this.done = true;
if (this.remaining === undefined) {
return { done: true, value: undefined };
} else {
return { done: false, value: this.remaining };
}
}
const inSize = value.byteLength + this.remainingSize;
const remainingSize = inSize % this.size;
const outSize = inSize - remainingSize;
let out: Uint8Array;
if (this.remaining !== undefined) {
out = new Uint8Array(outSize);
out.set(this.remaining);
out.set(
value.slice(0, value.byteLength - remainingSize),
this.remainingSize
);
} else {
out = value.slice(0, value.byteLength - remainingSize);
}

this.remainingSize = remainingSize;
if (remainingSize > 0) {
this.remaining = value.slice(value.byteLength - remainingSize);
} else {
this.remaining = undefined;
}

return { done: false, value: out };
}
}

self.addEventListener("activate", function (event) {
event.waitUntil(self.clients.claim());
Expand All @@ -63,17 +97,6 @@ self.addEventListener("message", function (event) {
}
});

async function try_fetch(input, init, tries = 3) {
try {
return await fetch(input, init);
} catch (e) {
if (tries > 0) {
return try_fetch(input, init, tries - 1);
}
throw e;
}
}

self.addEventListener("fetch", function (event) {
let request = event.request;
let url = new URL(request.url);
Expand All @@ -82,7 +105,7 @@ self.addEventListener("fetch", function (event) {
}
let path = url.pathname;
if (path.startsWith("/s/download")) {
event.respondWith(virtual_downloading_response(path));
event.respondWith(virtual_downloading_response(request));
return;
}
if (path.startsWith("/s/")) {
Expand All @@ -91,7 +114,22 @@ self.addEventListener("fetch", function (event) {
event.respondWith(cached_response(request));
});

async function virtual_downloading_response(path: string) {
function rangeOf(request: Request) {
let range = request.headers.get("Range");
if (range === null) {
return null;
}
let range_match = range.match(/^bytes=(\d+)-(\d+)$/);
if (range_match === null) {
return null;
}
let start = parseInt(range_match[1]);
let end = parseInt(range_match[2]);
return [start, end];
}

async function virtual_downloading_response(request: Request) {
const path = new URL(request.url).pathname;
let path_list = path.split("/");
let file_path = path_list[path_list.length - 1];
let file_info = FilesData[file_path];
Expand All @@ -101,53 +139,64 @@ async function virtual_downloading_response(path: string) {
statusText: "Not Found",
});
}
let headers = new Headers();
// let range = rangeOf(request);
// let start: number;
// if (range !== null) {
// start = range[0];
// } else {
// start = 0;
// }
// if (range !== null) {
// headers.set("Range", `bytes=${range[0]}-${range[1]}`);
// }
//// TODO: handle cases when range does not start from multiple of 16
let { abort, signal } = new AbortController();
let response = await fetch(file_info.download_url, { headers, signal });
let body = response.body;
if (body === null) {
return response;
}
let reader = new Chunker(body, 16); // chunk stream to size of multiple of 16 bytes
let decrypted_readable_stream = new ReadableStream({
async start(controller) {
const chunk_size = 1310720;
let chunk_number = Math.ceil(file_info.file_size / chunk_size);
let fetched = 0;
let fetch_queue: Promise<Uint8Array | null>[] = [];
async function next_fetch() {
if (fetched >= chunk_number) {
return null;
let offset = 0;
while (true) {
let readResult = await reader.read();
if (readResult.done) {
break;
}
let i = fetched;
fetched += 1;
let start = i * chunk_size;
let end;
if (i === chunk_number - 1) {
end = file_info.file_size - 1;
} else {
end = start + chunk_size - 1;
}
let response = await try_fetch(file_info.download_url, {
headers: { Range: `bytes=${start}-${end}` },
});
let data = await response.arrayBuffer();
let plain = await decrypt_file_part(
file_info.key,
data,
readResult.value,
file_info.nonce,
file_info.file_id,
start / 16
offset / 16
);
return new Uint8Array(plain);
}
fetch_queue.push(next_fetch());
setTimeout(function () {
// 4 concurrent download
fetch_queue.push(next_fetch());
fetch_queue.push(next_fetch());
fetch_queue.push(next_fetch());
}, 1000);
for (let j = 0; j < chunk_number; j++) {
let chunk = await fetch_queue.shift();
controller.enqueue(chunk);
fetch_queue.push(next_fetch());
offset += readResult.value.byteLength;
controller.enqueue(new Uint8Array(plain));
}
controller.close();
},
cancel() {
abort();
},
});
// let decrypted_readable_stream = body.pipeThrough(
// new TransformStream({
// async transform(chunk, controller) {
// let plain = await decrypt_file_part(
// file_info.key,
// chunk,
// file_info.nonce,
// file_info.file_id,
// start / 16
// );
// start += chunk.byteLength;
// controller.enqueue(new Uint8Array(plain));
// },
// })
// );
return new Response(decrypted_readable_stream, {
headers: {
"Content-Length": file_info.file_size,
Expand Down

0 comments on commit 194e17f

Please sign in to comment.