Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misc adjustments #2

Merged
merged 1 commit into from
May 20, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Misc adjustments
* Rework (some) tests
* Add EnableETagPair option to add a pair of eTags, even if the server does not provide one
* Add CacheKey option
* Remove redundant nil check
* Optionally allow caching for other HTTP methods than GET and HEAD
bep committed May 20, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 95e2c9513b1eadbcf58253c429724a360d026cfa
9 changes: 9 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
module github.com/gohugoio/httpcache

go 1.22.2

require github.com/frankban/quicktest v1.14.6

require (
github.com/google/go-cmp v0.5.9 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
)
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
122 changes: 99 additions & 23 deletions httpcache.go
Original file line number Diff line number Diff line change
@@ -8,7 +8,10 @@ package httpcache
import (
"bufio"
"bytes"
"crypto/md5"
"encoding/hex"
"errors"
"hash"
"io"
"net/http"
"net/http/httputil"
@@ -23,6 +26,15 @@ const (
transparent
// XFromCache is the header added to responses that are returned from the cache
XFromCache = "X-From-Cache"

// xEtags is the prefix for the header with the custom etag pair set in the cached response.
xEtags = "X-Etags-"

// XETag1 is the key for the first eTag value.
XETag1 = xEtags + "1"

// XETag2 is the key for the second eTag value.
XETag2 = xEtags + "2"
)

// A Cache interface is used by the Transport to store and retrieve responses.
@@ -37,7 +49,16 @@ type Cache interface {
}

// cacheKey returns the cache key for req.
func cacheKey(req *http.Request) string {
func (t *Transport) cacheKey(req *http.Request) string {
if t.CacheKey != nil {
return t.CacheKey(req)
}

cacheable := (req.Method != http.MethodHead || req.Method == "HEAD") && req.Header.Get("range") == ""
if !cacheable {
return ""
}

if req.Method == http.MethodGet {
return req.URL.String()
} else {
@@ -47,8 +68,8 @@ func cacheKey(req *http.Request) string {

// cachedResponse returns the cached http.Response for req if present, and nil
// otherwise.
func cachedResponse(c Cache, req *http.Request) (resp *http.Response, err error) {
cachedVal, ok := c.Get(cacheKey(req))
func (t *Transport) cachedResponse(req *http.Request) (resp *http.Response, err error) {
cachedVal, ok := t.Cache.Get(t.cacheKey(req))
if !ok {
return
}
@@ -63,6 +84,12 @@ type memoryCache struct {
items map[string][]byte
}

func (c *memoryCache) Size() int {
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.items)
}

// Get returns the []byte representation of the response and true if present, false if not
func (c *memoryCache) Get(key string) (resp []byte, ok bool) {
c.mu.RLock()
@@ -105,11 +132,21 @@ type Transport struct {
// If true, responses returned from the cache will be given an extra header, X-From-Cache
MarkCachedResponses bool

// if EnableETagPair is true, the Transport will store the pair of eTags in the response header.
// These are stored in the X-Etags-1 and X-Etags-2 headers.
// If these are different, the response has been modified.
// If the server does not return an eTag, the MD5 hash of the response body is used.
EnableETagPair bool

// CacheKey is an optional func that returns the key to use to store the response.
// An empty string signals that this request should not be cached.
CacheKey func(req *http.Request) string

// Around is an optional func.
// If set, the Transport will call Around at the start of RoundTrip
// and defer the returned func until the end of RoundTrip.
// Typically used to implement a lock that is held for the duration of the RoundTrip.
Around func(key string) func()
Around func(req *http.Request, key string) func()
}

// varyMatches will return false unless all of the cached values for the headers listed in Vary
@@ -133,14 +170,18 @@ func varyMatches(cachedResp *http.Response, req *http.Request) bool {
// to give the server a chance to respond with NotModified. If this happens, then the cached Response
// will be returned.
func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
cacheKey := cacheKey(req)
cacheKey := t.cacheKey(req)
if f := t.Around; f != nil {
defer f(cacheKey)()
defer f(req, cacheKey)()
}
cacheable := (req.Method == "GET" || req.Method == "HEAD") && req.Header.Get("range") == ""

var cachedXEtag string

cacheable := cacheKey != ""

var cachedResp *http.Response
if cacheable {
cachedResp, err = cachedResponse(t.Cache, req)
cachedResp, err = t.cachedResponse(req)
} else {
// Need to invalidate an existing value
t.Cache.Delete(cacheKey)
@@ -155,6 +196,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
if t.MarkCachedResponses {
cachedResp.Header.Set(XFromCache, "1")
}
if t.EnableETagPair {
cachedXEtag, _ = getXETags(cachedResp.Header)
}

if varyMatches(cachedResp, req) {
// Can only use cached value if the new request doesn't Vary significantly
@@ -185,15 +229,16 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
}

resp, err = transport.RoundTrip(req)
if err == nil && req.Method == "GET" && resp.StatusCode == http.StatusNotModified {

if err == nil && req.Method != http.MethodHead && resp.StatusCode == http.StatusNotModified {
// Replace the 304 response with the one from cache, but update with some new headers
endToEndHeaders := getEndToEndHeaders(resp.Header)
for _, header := range endToEndHeaders {
cachedResp.Header[header] = resp.Header[header]
}
resp = cachedResp
} else if (err != nil || (cachedResp != nil && resp.StatusCode >= 500)) &&
req.Method == "GET" && canStaleOnError(cachedResp.Header, req.Header) {
} else if (err != nil || resp.StatusCode >= 500) &&
req.Method != http.MethodHead && canStaleOnError(cachedResp.Header, req.Header) {
// In case of transport failure and stale-if-error activated, returns cached content
// when available
return cachedResp, nil
@@ -227,24 +272,51 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
}
}
switch req.Method {
case "GET":
// Delay caching until EOF is reached.
resp.Body = &cachingReadCloser{
R: resp.Body,
case http.MethodHead:
respBytes, err := httputil.DumpResponse(resp, true)
if err == nil {
t.Cache.Set(cacheKey, respBytes)
}
default:
var etagHash hash.Hash
r := resp.Body
if t.EnableETagPair {
if etag := resp.Header.Get("etag"); etag != "" {
resp.Header.Set(XETag1, etag)
resp.Header.Set(XETag2, cachedXEtag)
} else {
etagHash = md5.New()
r = struct {
io.Reader
io.Closer
}{
io.TeeReader(r, etagHash),
resp.Body,
}
}
}

r = &cachingReadCloser{
R: r,
OnEOF: func(r io.Reader) {
if etagHash != nil {
md5Str := hex.EncodeToString(etagHash.Sum(nil))
resp.Header.Set(XETag1, md5Str)
resp.Header.Set(XETag2, cachedXEtag)

}
resp := *resp
resp.Body = io.NopCloser(r)
respBytes, err := httputil.DumpResponse(&resp, true)
if err == nil {
t.Cache.Set(cacheKey, respBytes)
}
},
buf: &bytes.Buffer{},
}
default:
respBytes, err := httputil.DumpResponse(resp, true)
if err == nil {
t.Cache.Set(cacheKey, respBytes)
}
// Delay caching until EOF is reached.
resp.Body = r

}
} else {
t.Cache.Delete(cacheKey)
@@ -278,6 +350,10 @@ type timer interface {

var clock timer = &realClock{}

func getXETags(h http.Header) (string, string) {
return h.Get(XETag1), h.Get(XETag2)
}

// getFreshness will return one of fresh/stale/transparent based on the cache-control
// values of the request and the response
//
@@ -522,7 +598,7 @@ type cachingReadCloser struct {
// OnEOF is called with a copy of the content of R when EOF is reached.
OnEOF func(io.Reader)

buf bytes.Buffer // buf stores a copy of the content of R.
buf *bytes.Buffer // buf stores a copy of the content of R.
}

// Read reads the next len(p) bytes from R or until R is drained. The
@@ -533,7 +609,7 @@ func (r *cachingReadCloser) Read(p []byte) (n int, err error) {
n, err = r.R.Read(p)
r.buf.Write(p[:n])
if err == io.EOF {
r.OnEOF(bytes.NewReader(r.buf.Bytes()))
r.OnEOF(r.buf)
}
return n, err
}
@@ -545,6 +621,6 @@ func (r *cachingReadCloser) Close() error {
// newMemoryCacheTransport returns a new Transport using the in-memory cache implementation
func newMemoryCacheTransport() *Transport {
c := newMemoryCache()
t := &Transport{Cache: c, MarkCachedResponses: true}
t := &Transport{Cache: c}
return t
}
Loading