Skip to content

Commit

Permalink
feat(cache): debugging testing 307 redirect.
Browse files Browse the repository at this point in the history
  • Loading branch information
@IsakT
IsakT committed Jul 9, 2024
1 parent 45b06fd commit 473085c
Show file tree
Hide file tree
Showing 4 changed files with 206 additions and 72 deletions.
217 changes: 148 additions & 69 deletions lib/interceptor/cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,17 @@ class CacheHandler extends DecoratorHandler {
}

onConnect(abort) {
console.log('onConnect abort')
console.log(abort)

this.#value = null

return this.#handler.onConnect(abort)
}

onHeaders(statusCode, rawHeaders, resume, statusMessage, headers = parseHeaders(rawHeaders)) {
// console.log('onHeaders, headers:')
// console.log(headers)
console.log('onHeaders')
console.log({ statusCode, rawHeaders, resume, statusMessage, headers })

if (statusCode !== 307) {
return this.#handler.onHeaders(statusCode, rawHeaders, resume, statusMessage, headers)
Expand All @@ -36,6 +39,22 @@ class CacheHandler extends DecoratorHandler {
const contentLength = headers['content-length'] ? Number(headers['content-length']) : Infinity
const maxEntrySize = this.#store.maxEntrySize ?? Infinity

console.log({ cacheControl, contentLength, maxEntrySize })

console.log('onHeaders if statement match:')

console.log(
contentLength < maxEntrySize &&
cacheControl &&
cacheControl.public &&
!cacheControl.private &&
!cacheControl['no-store'] &&
!cacheControl['no-cache'] &&
!cacheControl['must-understand'] &&
!cacheControl['must-revalidate'] &&
!cacheControl['proxy-revalidate'],
)

if (
contentLength < maxEntrySize &&
cacheControl &&
Expand All @@ -54,14 +73,16 @@ class CacheHandler extends DecoratorHandler {
? 31556952 // 1 year
: Number(maxAge)

console.log({ ttl, maxAge, cacheControl, contentLength, maxEntrySize })

if (ttl > 0) {
this.#value = {
data: {
statusCode,
statusMessage,
rawHeaders,
rawTrailers: null,
body: [], // Why is the body emptied? When we cache it again it won't have a body.
body: [],
},
size:
(rawHeaders?.reduce((xs, x) => xs + x.length, 0) ?? 0) +
Expand All @@ -70,8 +91,13 @@ class CacheHandler extends DecoratorHandler {
ttl: ttl * 1e3,
}
}

console.log({ thisvalue: this.#value })
}

console.log('onHeaders, finish:')
console.log({ statusCode, rawHeaders, resume, statusMessage, headers })

return this.#handler.onHeaders(statusCode, rawHeaders, resume, statusMessage, headers)
}

Expand All @@ -87,41 +113,48 @@ class CacheHandler extends DecoratorHandler {
}
}
return this.#handler.onData(chunk)
/*
Is 'this.#handler.onData' the previous dispatcher in the chain, e.g. 'redirect'?
And in 'redirect.onData(chunk)' it once again calls 'this.#handler.onData(chunk)'.
Would that be 'responseVerify.onData(chunk)'?
*/
}

onComplete(rawTrailers, opts) {
console.log('onComplete, value: ' + this.#value)
console.log('onComplete, opts:')
console.log(opts)

onComplete(rawTrailers) {
console.log('onComplete this:')
console.log({ thisvalue: this.#value })
console.log({ thisstore: this.#store }) // CacheStore{}
console.log({ thishandler: this.#handler }) // RequestHandler{}
console.log({ thishandlervalue: this.#handler.value })
console.log({ this: this })
if (this.#value) {
this.#value.data.rawTrailers = rawTrailers
this.#value.size += rawTrailers?.reduce((xs, x) => xs + x.length, 0) ?? 0

console.log('OnComplete, cache store is being set to: ')
console.log([this.#key, this.#value.data, { ttl: this.#value.ttl, size: this.#value.size }])
const opts = this.#handler.opts
const entries = this.#handler.entries
console.log('onComplete this:')
console.log({ opts, entries })

const reqHeaders = this.#handler.opts
const resHeaders = parseHeaders(this.#value.data.rawHeaders)

/*
Why are we setting the cache with the same data as the entry we fetched earlier
from the very same cache?
const vary = formatVaryData(resHeaders, reqHeaders)

We have the request data in the `opts` variable, but where is the response data that we need to cache?
Is the response cached somewhere else?
console.log({ vary })

We have the headers we need from the request. But we need the response data to know the vary-header
and we also need it to store the response.
*/
this.#store.set(this.#key, this.#value.data, { ttl: this.#value.ttl, size: this.#value.size })
this.#value.vary = vary

console.log({ entries })

this.#store.set(this.#key, entries.push(this.#value))
}
return this.#handler.onComplete(rawTrailers, opts)
return this.#handler.onComplete(rawTrailers)
}
}

function formatVaryData(resHeaders, reqHeaders) {
return resHeaders.vary
?.split(',')
.map((key) => key.trim().toLowerCase())
.map((key) => [key, reqHeaders[key]])
}

// TODO (fix): Async filesystem cache.
class CacheStore {
constructor({ maxSize = 1024 * 1024, maxEntrySize = 128 * 1024 }) {
Expand All @@ -131,9 +164,6 @@ class CacheStore {
}

set(key, value, opts) {
console.log('setting cache with values:')
console.log({ key, value, opts })

this.cache.set(key, value, opts)
}

Expand All @@ -142,36 +172,29 @@ class CacheStore {
}
}

function makeKey(opts) {
// NOTE: Ignores headers...
// return `${opts.origin}:${opts.method}:${opts.path}`
return `${opts.method}:${opts.path}`
}

function varyHeadersMatchRequest(varyHeaders, requestHeaders) {
// const headersToString = []
// for(const header of cachedRawHeaders){
// headersToString.push(header.toString())
// }

// const varyHeaders = headersToString.reduce((acc, cur, index, arr) => {
// if (index % 2 === 0) {
// acc[cur] = arr[index + 1];
// }
// return acc;
// }, {});

// Early return if `varyHeaders` is null/undefined or an empty object
if (!varyHeaders || Object.keys(varyHeaders).length === 0) {
return true
}
const varyKeys = Object.keys(varyHeaders)
// All vary headers must match request headers, return true/false.
return varyKeys.every((varyKey) => varyHeaders[varyKey] === requestHeaders[varyKey])
}

function findEntryByHeaders(entries, requestHeaders) {
return entries.find((entry) => varyHeadersMatchRequest(entry, requestHeaders))
function findEntryByHeaders(entries, reqHeaders) {
// Sort entries by number of vary headers in descending order, because
// we want to compare the most complex response to the request first.
entries.sort((a, b) => {
const lengthA = a.vary ? a.vary.length : 0
const lengthB = b.vary ? b.vary.length : 0
return lengthB - lengthA
})

console.log('Sort entries')
console.log({ entries })

console.log('reqHeaders')
console.log({ reqHeaders })

return entries?.find(
(entry) =>
entry.vary?.every(([key, val]) => {
console.log(`reqHeaders[${key}] === ${val}`)
console.log({ reqHeadersval: reqHeaders[key] })
return reqHeaders[key] === val
}) ?? true,
)
}

const DEFAULT_CACHE_STORE = new CacheStore({ maxSize: 128 * 1024, maxEntrySize: 1024 })
Expand Down Expand Up @@ -218,23 +241,77 @@ export default (opts) => (dispatch) => (opts, handler) => {
throw new Error(`Cache store not provided.`)
}

let key = makeKey(opts)
let key = `${opts.method}:${opts.path}`
console.log('getting key: ' + key)
let entries = store.get(key)

console.log('Found entries in cache: ')
console.log(entries)

// if key with method:'HEAD' didn't yield results, retry with method:'GET'
if (entries.length === 0 && opts.method === 'HEAD') {
key = makeKey({ ...opts, method: 'GET' })
if (Array.isArray(entries) && entries.length === 0 && opts.method === 'HEAD') {
key = `GET:${opts.path}`
entries = store.get(key)
// value = {data: {headers: {vary: {origin: "www.google.com"}}}
}

// testing
const rawHeaders = [
Buffer.from('Content-Type'),
Buffer.from('application/json'),
Buffer.from('Content-Length'),
Buffer.from('10'),
Buffer.from('Cache-Control'),
Buffer.from('public'),
]
// // cannot get the cache to work inside the test, so I hardcode the entries here
entries = [
{
statusCode: 200,
statusMessage: '',
rawHeaders,
rawTrailers: ['Hello'],
body: ['asd1'],
vary: [
['Accept', 'application/xml'],
['User-Agent', 'Mozilla/5.0'],
],
},
{
statusCode: 200,
statusMessage: '',
rawHeaders,
rawTrailers: ['Hello'],
body: ['asd2'],
vary: [
['Accept', 'application/txt'],
['User-Agent', 'Chrome'],
['origin2', 'www.google.com/images'],
],
},
// {
// statusCode: 200, statusMessage: 'last', rawHeaders, rawTrailers: ['Hello'], body: ['asd3'],
// vary: null },
{
statusCode: 200,
statusMessage: 'first',
rawHeaders,
rawTrailers: ['Hello'],
body: ['asd4'],
vary: [
['Accept', 'application/json'],
['User-Agent', 'Mozilla/5.0'],
['host2', 'www.google.com'],
['origin2', 'www.google.com/images'],
],
},
]

// *testing

// Find an entry that matches the request, if any
const entry = findEntryByHeaders(entries, opts)

console.log('Entry found:')
console.log({ entry })

// handler.value.vary = 'foobar'

if (entry) {
const { statusCode, statusMessage, rawHeaders, rawTrailers, body } = entry
const ac = new AbortController()
Expand All @@ -258,16 +335,18 @@ export default (opts) => (dispatch) => (opts, handler) => {
// TODO (fix): back pressure...
}
}
handler.onComplete(rawTrailers, opts)
handler.onComplete(rawTrailers)
} else {
handler.onComplete([], opts)
handler.onComplete([])
}
} catch (err) {
handler.onError(err)
}

return true
} else {
return dispatch(opts, new CacheHandler({ handler, store, key: makeKey(opts) }))
// handler.opts = opts
// handler.entries = entries
return dispatch(opts, new CacheHandler({ handler, store, key }))
}
}
3 changes: 3 additions & 0 deletions lib/interceptor/proxy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class Handler extends DecoratorHandler {
}

onUpgrade(statusCode, rawHeaders, socket) {
console.log('Proxy onUpgrade')
return this.#handler.onUpgrade(
statusCode,
reduceHeaders(
Expand All @@ -34,6 +35,7 @@ class Handler extends DecoratorHandler {
}

onHeaders(statusCode, rawHeaders, resume, statusMessage) {
console.log('Proxy onHeaders')
return this.#handler.onHeaders(
statusCode,
reduceHeaders(
Expand Down Expand Up @@ -164,6 +166,7 @@ function printIp(address, port) {
}

export default (opts) => (dispatch) => (opts, handler) => {
console.log('Proxy default dispatch')
if (!opts.proxy) {
return dispatch(opts, handler)
}
Expand Down
3 changes: 3 additions & 0 deletions lib/interceptor/redirect.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class Handler extends DecoratorHandler {
}

onConnect(abort) {
console.log('Redirect onConnect')
if (this.#aborted) {
abort(this.#reason)
} else {
Expand All @@ -48,6 +49,7 @@ class Handler extends DecoratorHandler {
}

onHeaders(statusCode, rawHeaders, resume, statusText, headers = parseHeaders(rawHeaders)) {
console.log('Redirect onHeaders')
if (redirectableStatusCodes.indexOf(statusCode) === -1) {
assert(!this.#headersSent)
this.#headersSent = true
Expand Down Expand Up @@ -109,6 +111,7 @@ class Handler extends DecoratorHandler {
}

onData(chunk) {
console.log('Redirect onData')
if (this.#location) {
/*
https://tools.ietf.org/html/rfc7231#section-6.4
Expand Down
Loading

0 comments on commit 473085c

Please sign in to comment.